net/ipv6/tcp_ipv6.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  *      TCP over IPv6
   4  *      Linux INET6 implementation
   5  *
   6  *      Authors:
   7  *      Pedro Roque             <roque@di.fc.ul.pt>
   8  *
   9  *      Based on:
  10  *      linux/net/ipv4/tcp.c
  11  *      linux/net/ipv4/tcp_input.c
  12  *      linux/net/ipv4/tcp_output.c
  13  *
  14  *      Fixes:
  15  *      Hideaki YOSHIFUJI       :       sin6_scope_id support
  16  *      YOSHIFUJI Hideaki @USAGI and:   Support IPV6_V6ONLY socket option, which
  17  *      Alexey Kuznetsov                allow both IPv4 and IPv6 sockets to bind
  18  *                                      a single port at the same time.
  19  *      YOSHIFUJI Hideaki @USAGI:       convert /proc/net/tcp6 to seq_file.
  20  */
  21
  22 #include <linux/bottom_half.h>
  23 #include <linux/module.h>
  24 #include <linux/errno.h>
  25 #include <linux/types.h>
  26 #include <linux/socket.h>
  27 #include <linux/sockios.h>
  28 #include <linux/net.h>
  29 #include <linux/jiffies.h>
  30 #include <linux/in.h>
  31 #include <linux/in6.h>
  32 #include <linux/netdevice.h>
  33 #include <linux/init.h>
  34 #include <linux/jhash.h>
  35 #include <linux/ipsec.h>
  36 #include <linux/times.h>
  37 #include <linux/slab.h>
  38 #include <linux/uaccess.h>
  39 #include <linux/ipv6.h>
  40 #include <linux/icmpv6.h>
  41 #include <linux/random.h>
  42 #include <linux/indirect_call_wrapper.h>
  43
  44 #include <net/tcp.h>
  45 #include <net/ndisc.h>
  46 #include <net/inet6_hashtables.h>
  47 #include <net/inet6_connection_sock.h>
  48 #include <net/ipv6.h>
  49 #include <net/transp_v6.h>
  50 #include <net/addrconf.h>
  51 #include <net/ip6_route.h>
  52 #include <net/ip6_checksum.h>
  53 #include <net/inet_ecn.h>
  54 #include <net/protocol.h>
  55 #include <net/xfrm.h>
  56 #include <net/snmp.h>
  57 #include <net/dsfield.h>
  58 #include <net/timewait_sock.h>
  59 #include <net/inet_common.h>
  60 #include <net/secure_seq.h>
  61 #include <net/busy_poll.h>
  62
  63 #include <linux/proc_fs.h>
  64 #include <linux/seq_file.h>
  65
  66 #include <crypto/hash.h>
  67 #include <linux/scatterlist.h>
  68
  69 #include <trace/events/tcp.h>
  70
  71 static void     tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb);
  72 static void     tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
  73                                       struct request_sock *req);
  74
  75 static int      tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb);
  76
  77 static const struct inet_connection_sock_af_ops ipv6_mapped;
  78 const struct inet_connection_sock_af_ops ipv6_specific;
  79 #ifdef CONFIG_TCP_MD5SIG
  80 static const struct tcp_sock_af_ops tcp_sock_ipv6_specific;
  81 static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific;
  82 #else
  83 static struct tcp_md5sig_key *tcp_v6_md5_do_lookup(const struct sock *sk,
  84                                                    const struct in6_addr *addr,
  85                                                    int l3index)
  86 {
  87         return NULL;
  88 }
  89 #endif
  90
  91 /* Helper returning the inet6 address from a given tcp socket.
  92  * It can be used in TCP stack instead of inet6_sk(sk).
  93  * This avoids a dereference and allow compiler optimizations.
  94  * It is a specialized version of inet6_sk_generic().
  95  */
  96 static struct ipv6_pinfo *tcp_inet6_sk(const struct sock *sk)
  97 {
  98         unsigned int offset = sizeof(struct tcp6_sock) - sizeof(struct ipv6_pinfo);
  99
 100         return (struct ipv6_pinfo *)(((u8 *)sk) + offset);
 101 }
 102
 103 static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
 104 {
 105         struct dst_entry *dst = skb_dst(skb);
 106
 107         if (dst && dst_hold_safe(dst)) {
 108                 const struct rt6_info *rt = (const struct rt6_info *)dst;
 109
 110                 sk->sk_rx_dst = dst;
 111                 inet_sk(sk)->rx_dst_ifindex = skb->skb_iif;
 112                 tcp_inet6_sk(sk)->rx_dst_cookie = rt6_get_cookie(rt);
 113         }
 114 }
 115
 116 static u32 tcp_v6_init_seq(const struct sk_buff *skb)
 117 {
 118         return secure_tcpv6_seq(ipv6_hdr(skb)->daddr.s6_addr32,
 119                                 ipv6_hdr(skb)->saddr.s6_addr32,
 120                                 tcp_hdr(skb)->dest,
 121                                 tcp_hdr(skb)->source);
 122 }
 123
 124 static u32 tcp_v6_init_ts_off(const struct net *net, const struct sk_buff *skb)
 125 {
 126         return secure_tcpv6_ts_off(net, ipv6_hdr(skb)->daddr.s6_addr32,
 127                                    ipv6_hdr(skb)->saddr.s6_addr32);
 128 }
 129
 130 static int tcp_v6_pre_connect(struct sock *sk, struct sockaddr *uaddr,
 131                               int addr_len)
 132 {
 133         /* This check is replicated from tcp_v6_connect() and intended to
 134          * prevent BPF program called below from accessing bytes that are out
 135          * of the bound specified by user in addr_len.
 136          */
 137         if (addr_len < SIN6_LEN_RFC2133)
 138                 return -EINVAL;
 139
 140         sock_owned_by_me(sk);
 141
 142         return BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr);
 143 }
 144
 145 static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
 146                           int addr_len)
 147 {
 148         struct sockaddr_in6 *usin = (struct sockaddr_in6 *) uaddr;
 149         struct inet_sock *inet = inet_sk(sk);
 150         struct inet_connection_sock *icsk = inet_csk(sk);
 151         struct ipv6_pinfo *np = tcp_inet6_sk(sk);
 152         struct tcp_sock *tp = tcp_sk(sk);
 153         struct in6_addr *saddr = NULL, *final_p, final;
 154         struct ipv6_txoptions *opt;
 155         struct flowi6 fl6;
 156         struct dst_entry *dst;
 157         int addr_type;
 158         int err;
 159         struct inet_timewait_death_row *tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;
 160
 161         if (addr_len < SIN6_LEN_RFC2133)
 162                 return -EINVAL;
 163
 164         if (usin->sin6_family != AF_INET6)
 165                 return -EAFNOSUPPORT;
 166
 167         memset(&fl6, 0, sizeof(fl6));
 168
 169         if (np->sndflow) {
 170                 fl6.flowlabel = usin->sin6_flowinfo&IPV6_FLOWINFO_MASK;
 171                 IP6_ECN_flow_init(fl6.flowlabel);
 172                 if (fl6.flowlabel&IPV6_FLOWLABEL_MASK) {
 173                         struct ip6_flowlabel *flowlabel;
 174                         flowlabel = fl6_sock_lookup(sk, fl6.flowlabel);
 175                         if (IS_ERR(flowlabel))
 176                                 return -EINVAL;
 177                         fl6_sock_release(flowlabel);
 178                 }
 179         }
 180
 181         /*
 182          *      connect() to INADDR_ANY means loopback (BSD'ism).
 183          */
 184
 185         if (ipv6_addr_any(&usin->sin6_addr)) {
 186                 if (ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr))
 187                         ipv6_addr_set_v4mapped(htonl(INADDR_LOOPBACK),
 188                                                &usin->sin6_addr);
 189                 else
 190                         usin->sin6_addr = in6addr_loopback;
 191         }
 192
 193         addr_type = ipv6_addr_type(&usin->sin6_addr);
 194
 195         if (addr_type & IPV6_ADDR_MULTICAST)
 196                 return -ENETUNREACH;
 197
 198         if (addr_type&IPV6_ADDR_LINKLOCAL) {
 199                 if (addr_len >= sizeof(struct sockaddr_in6) &&
 200                     usin->sin6_scope_id) {
 201                         /* If interface is set while binding, indices
 202                          * must coincide.
 203                          */
 204                         if (!sk_dev_equal_l3scope(sk, usin->sin6_scope_id))
 205                                 return -EINVAL;
 206
 207                         sk->sk_bound_dev_if = usin->sin6_scope_id;
 208                 }
 209
 210                 /* Connect to link-local address requires an interface */
 211                 if (!sk->sk_bound_dev_if)
 212                         return -EINVAL;
 213         }
 214
 215         if (tp->rx_opt.ts_recent_stamp &&
 216             !ipv6_addr_equal(&sk->sk_v6_daddr, &usin->sin6_addr)) {
 217                 tp->rx_opt.ts_recent = 0;
 218                 tp->rx_opt.ts_recent_stamp = 0;
 219                 WRITE_ONCE(tp->write_seq, 0);
 220         }
 221
 222         sk->sk_v6_daddr = usin->sin6_addr;
 223         np->flow_label = fl6.flowlabel;
 224
 225         /*
 226          *      TCP over IPv4
 227          */
 228
 229         if (addr_type & IPV6_ADDR_MAPPED) {
 230                 u32 exthdrlen = icsk->icsk_ext_hdr_len;
 231                 struct sockaddr_in sin;
 232
 233                 if (__ipv6_only_sock(sk))
 234                         return -ENETUNREACH;
 235
 236                 sin.sin_family = AF_INET;
 237                 sin.sin_port = usin->sin6_port;
 238                 sin.sin_addr.s_addr = usin->sin6_addr.s6_addr32[3];
 239
 240                 icsk->icsk_af_ops = &ipv6_mapped;
 241                 if (sk_is_mptcp(sk))
 242                         mptcpv6_handle_mapped(sk, true);
 243                 sk->sk_backlog_rcv = tcp_v4_do_rcv;
 244 #ifdef CONFIG_TCP_MD5SIG
 245                 tp->af_specific = &tcp_sock_ipv6_mapped_specific;
 246 #endif
 247
 248                 err = tcp_v4_connect(sk, (struct sockaddr *)&sin, sizeof(sin));
 249
 250                 if (err) {
 251                         icsk->icsk_ext_hdr_len = exthdrlen;
 252                         icsk->icsk_af_ops = &ipv6_specific;
 253                         if (sk_is_mptcp(sk))
 254                                 mptcpv6_handle_mapped(sk, false);
 255                         sk->sk_backlog_rcv = tcp_v6_do_rcv;
 256 #ifdef CONFIG_TCP_MD5SIG
 257                         tp->af_specific = &tcp_sock_ipv6_specific;
 258 #endif
 259                         goto failure;
 260                 }
 261                 np->saddr = sk->sk_v6_rcv_saddr;
 262
 263                 return err;
 264         }
 265
 266         if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr))
 267                 saddr = &sk->sk_v6_rcv_saddr;
 268
 269         fl6.flowi6_proto = IPPROTO_TCP;
 270         fl6.daddr = sk->sk_v6_daddr;
 271         fl6.saddr = saddr ? *saddr : np->saddr;
 272         fl6.flowi6_oif = sk->sk_bound_dev_if;
 273         fl6.flowi6_mark = sk->sk_mark;
 274         fl6.fl6_dport = usin->sin6_port;
 275         fl6.fl6_sport = inet->inet_sport;
 276         fl6.flowi6_uid = sk->sk_uid;
 277
 278         opt = rcu_dereference_protected(np->opt, lockdep_sock_is_held(sk));
 279         final_p = fl6_update_dst(&fl6, opt, &final);
 280
 281         security_sk_classify_flow(sk, flowi6_to_flowi(&fl6));
 282
 283         dst = ip6_dst_lookup_flow(sock_net(sk), sk, &fl6, final_p);
 284         if (IS_ERR(dst)) {
 285                 err = PTR_ERR(dst);
 286                 goto failure;
 287         }
 288
 289         if (!saddr) {
 290                 saddr = &fl6.saddr;
 291                 sk->sk_v6_rcv_saddr = *saddr;
 292         }
 293
 294         /* set the source address */
 295         np->saddr = *saddr;
 296         inet->inet_rcv_saddr = LOOPBACK4_IPV6;
 297
 298         sk->sk_gso_type = SKB_GSO_TCPV6;
 299         ip6_dst_store(sk, dst, NULL, NULL);
 300
 301         icsk->icsk_ext_hdr_len = 0;
 302         if (opt)
 303                 icsk->icsk_ext_hdr_len = opt->opt_flen +
 304                                          opt->opt_nflen;
 305
 306         tp->rx_opt.mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) - sizeof(struct ipv6hdr);
 307
 308         inet->inet_dport = usin->sin6_port;
 309
 310         tcp_set_state(sk, TCP_SYN_SENT);
 311         err = inet6_hash_connect(tcp_death_row, sk);
 312         if (err)
 313                 goto late_failure;
 314
 315         sk_set_txhash(sk);
 316
 317         if (likely(!tp->repair)) {
 318                 if (!tp->write_seq)
 319                         WRITE_ONCE(tp->write_seq,
 320                                    secure_tcpv6_seq(np->saddr.s6_addr32,
 321                                                     sk->sk_v6_daddr.s6_addr32,
 322                                                     inet->inet_sport,
 323                                                     inet->inet_dport));
 324                 tp->tsoffset = secure_tcpv6_ts_off(sock_net(sk),
 325                                                    np->saddr.s6_addr32,
 326                                                    sk->sk_v6_daddr.s6_addr32);
 327         }
 328
 329         if (tcp_fastopen_defer_connect(sk, &err))
 330                 return err;
 331         if (err)
 332                 goto late_failure;
 333
 334         err = tcp_connect(sk);
 335         if (err)
 336                 goto late_failure;
 337
 338         return 0;
 339
 340 late_failure:
 341         tcp_set_state(sk, TCP_CLOSE);
 342 failure:
 343         inet->inet_dport = 0;
 344         sk->sk_route_caps = 0;
 345         return err;
 346 }
 347
 348 static void tcp_v6_mtu_reduced(struct sock *sk)
 349 {
 350         struct dst_entry *dst;
 351
 352         if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
 353                 return;
 354
 355         dst = inet6_csk_update_pmtu(sk, tcp_sk(sk)->mtu_info);
 356         if (!dst)
 357                 return;
 358
 359         if (inet_csk(sk)->icsk_pmtu_cookie > dst_mtu(dst)) {
 360                 tcp_sync_mss(sk, dst_mtu(dst));
 361                 tcp_simple_retransmit(sk);
 362         }
 363 }
 364
 365 static int tcp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
 366                 u8 type, u8 code, int offset, __be32 info)
 367 {
 368         const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data;
 369         const struct tcphdr *th = (struct tcphdr *)(skb->data+offset);
 370         struct net *net = dev_net(skb->dev);
 371         struct request_sock *fastopen;
 372         struct ipv6_pinfo *np;
 373         struct tcp_sock *tp;
 374         __u32 seq, snd_una;
 375         struct sock *sk;
 376         bool fatal;
 377         int err;
 378
 379         sk = __inet6_lookup_established(net, &tcp_hashinfo,
 380                                         &hdr->daddr, th->dest,
 381                                         &hdr->saddr, ntohs(th->source),
 382                                         skb->dev->ifindex, inet6_sdif(skb));
 383
 384         if (!sk) {
 385                 __ICMP6_INC_STATS(net, __in6_dev_get(skb->dev),
 386                                   ICMP6_MIB_INERRORS);
 387                 return -ENOENT;
 388         }
 389
 390         if (sk->sk_state == TCP_TIME_WAIT) {
 391                 inet_twsk_put(inet_twsk(sk));
 392                 return 0;
 393         }
 394         seq = ntohl(th->seq);
 395         fatal = icmpv6_err_convert(type, code, &err);
 396         if (sk->sk_state == TCP_NEW_SYN_RECV) {
 397                 tcp_req_err(sk, seq, fatal);
 398                 return 0;
 399         }
 400
 401         bh_lock_sock(sk);
 402         if (sock_owned_by_user(sk) && type != ICMPV6_PKT_TOOBIG)
 403                 __NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS);
 404
 405         if (sk->sk_state == TCP_CLOSE)
 406                 goto out;
 407
 408         if (ipv6_hdr(skb)->hop_limit < tcp_inet6_sk(sk)->min_hopcount) {
 409                 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
 410                 goto out;
 411         }
 412
 413         tp = tcp_sk(sk);
 414         /* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
 415         fastopen = rcu_dereference(tp->fastopen_rsk);
 416         snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;
 417         if (sk->sk_state != TCP_LISTEN &&
 418             !between(seq, snd_una, tp->snd_nxt)) {
 419                 __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
 420                 goto out;
 421         }
 422
 423         np = tcp_inet6_sk(sk);
 424
 425         if (type == NDISC_REDIRECT) {
 426                 if (!sock_owned_by_user(sk)) {
 427                         struct dst_entry *dst = __sk_dst_check(sk, np->dst_cookie);
 428
 429                         if (dst)
 430                                 dst->ops->redirect(dst, sk, skb);
 431                 }
 432                 goto out;
 433         }
 434
 435         if (type == ICMPV6_PKT_TOOBIG) {
 436                 /* We are not interested in TCP_LISTEN and open_requests
 437                  * (SYN-ACKs send out by Linux are always <576bytes so
 438                  * they should go through unfragmented).
 439                  */
 440                 if (sk->sk_state == TCP_LISTEN)
 441                         goto out;
 442
 443                 if (!ip6_sk_accept_pmtu(sk))
 444                         goto out;
 445
 446                 tp->mtu_info = ntohl(info);
 447                 if (!sock_owned_by_user(sk))
 448                         tcp_v6_mtu_reduced(sk);
 449                 else if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED,
 450                                            &sk->sk_tsq_flags))
 451                         sock_hold(sk);
 452                 goto out;
 453         }
 454
 455
 456         /* Might be for an request_sock */
 457         switch (sk->sk_state) {
 458         case TCP_SYN_SENT:
 459         case TCP_SYN_RECV:
 460                 /* Only in fast or simultaneous open. If a fast open socket is
 461                  * is already accepted it is treated as a connected one below.
 462                  */
 463                 if (fastopen && !fastopen->sk)
 464                         break;
 465
 466                 ipv6_icmp_error(sk, skb, err, th->dest, ntohl(info), (u8 *)th);
 467
 468                 if (!sock_owned_by_user(sk)) {
 469                         sk->sk_err = err;
 470                         sk->sk_error_report(sk);                /* Wake people up to see the error (see connect in sock.c) */
 471
 472                         tcp_done(sk);
 473                 } else
 474                         sk->sk_err_soft = err;
 475                 goto out;
 476         }
 477
 478         if (!sock_owned_by_user(sk) && np->recverr) {
 479                 sk->sk_err = err;
 480                 sk->sk_error_report(sk);
 481         } else
 482                 sk->sk_err_soft = err;
 483
 484 out:
 485         bh_unlock_sock(sk);
 486         sock_put(sk);
 487         return 0;
 488 }
 489
 490
 491 static int tcp_v6_send_synack(const struct sock *sk, struct dst_entry *dst,
 492                               struct flowi *fl,
 493                               struct request_sock *req,
 494                               struct tcp_fastopen_cookie *foc,
 495                               enum tcp_synack_type synack_type)
 496 {
 497         struct inet_request_sock *ireq = inet_rsk(req);
 498         struct ipv6_pinfo *np = tcp_inet6_sk(sk);
 499         struct ipv6_txoptions *opt;
 500         struct flowi6 *fl6 = &fl->u.ip6;
 501         struct sk_buff *skb;
 502         int err = -ENOMEM;
 503
 504         /* First, grab a route. */
 505         if (!dst && (dst = inet6_csk_route_req(sk, fl6, req,
 506                                                IPPROTO_TCP)) == NULL)
 507                 goto done;
 508
 509         skb = tcp_make_synack(sk, dst, req, foc, synack_type);
 510
 511         if (skb) {
 512                 __tcp_v6_send_check(skb, &ireq->ir_v6_loc_addr,
 513                                     &ireq->ir_v6_rmt_addr);
 514
 515                 fl6->daddr = ireq->ir_v6_rmt_addr;
 516                 if (np->repflow && ireq->pktopts)
 517                         fl6->flowlabel = ip6_flowlabel(ipv6_hdr(ireq->pktopts));
 518
 519                 rcu_read_lock();
 520                 opt = ireq->ipv6_opt;
 521                 if (!opt)
 522                         opt = rcu_dereference(np->opt);
 523                 err = ip6_xmit(sk, skb, fl6, sk->sk_mark, opt, np->tclass,
 524                                sk->sk_priority);
 525                 rcu_read_unlock();
 526                 err = net_xmit_eval(err);
 527         }
 528
 529 done:
 530         return err;
 531 }
 532
 533
 534 static void tcp_v6_reqsk_destructor(struct request_sock *req)
 535 {
 536         kfree(inet_rsk(req)->ipv6_opt);
 537         kfree_skb(inet_rsk(req)->pktopts);
 538 }
 539
 540 #ifdef CONFIG_TCP_MD5SIG
 541 static struct tcp_md5sig_key *tcp_v6_md5_do_lookup(const struct sock *sk,
 542                                                    const struct in6_addr *addr,
 543                                                    int l3index)
 544 {
 545         return tcp_md5_do_lookup(sk, l3index,
 546                                  (union tcp_md5_addr *)addr, AF_INET6);
 547 }
 548
 549 static struct tcp_md5sig_key *tcp_v6_md5_lookup(const struct sock *sk,
 550                                                 const struct sock *addr_sk)
 551 {
 552         int l3index;
 553
 554         l3index = l3mdev_master_ifindex_by_index(sock_net(sk),
 555                                                  addr_sk->sk_bound_dev_if);
 556         return tcp_v6_md5_do_lookup(sk, &addr_sk->sk_v6_daddr,
 557                                     l3index);
 558 }
 559
 560 static int tcp_v6_parse_md5_keys(struct sock *sk, int optname,
 561                                  char __user *optval, int optlen)
 562 {
 563         struct tcp_md5sig cmd;
 564         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&cmd.tcpm_addr;
 565         int l3index = 0;
 566         u8 prefixlen;
 567
 568         if (optlen < sizeof(cmd))
 569                 return -EINVAL;
 570
 571         if (copy_from_user(&cmd, optval, sizeof(cmd)))
 572                 return -EFAULT;
 573
 574         if (sin6->sin6_family != AF_INET6)
 575                 return -EINVAL;
 576
 577         if (optname == TCP_MD5SIG_EXT &&
 578             cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) {
 579                 prefixlen = cmd.tcpm_prefixlen;
 580                 if (prefixlen > 128 || (ipv6_addr_v4mapped(&sin6->sin6_addr) &&
 581                                         prefixlen > 32))
 582                         return -EINVAL;
 583         } else {
 584                 prefixlen = ipv6_addr_v4mapped(&sin6->sin6_addr) ? 32 : 128;
 585         }
 586
 587         if (optname == TCP_MD5SIG_EXT &&
 588             cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) {
 589                 struct net_device *dev;
 590
 591                 rcu_read_lock();
 592                 dev = dev_get_by_index_rcu(sock_net(sk), cmd.tcpm_ifindex);
 593                 if (dev && netif_is_l3_master(dev))
 594                         l3index = dev->ifindex;
 595                 rcu_read_unlock();
 596
 597                 /* ok to reference set/not set outside of rcu;
 598                  * right now device MUST be an L3 master
 599                  */
 600                 if (!dev || !l3index)
 601                         return -EINVAL;
 602         }
 603
 604         if (!cmd.tcpm_keylen) {
 605                 if (ipv6_addr_v4mapped(&sin6->sin6_addr))
 606                         return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3],
 607                                               AF_INET, prefixlen,
 608                                               l3index);
 609                 return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr,
 610                                       AF_INET6, prefixlen, l3index);
 611         }
 612
 613         if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
 614                 return -EINVAL;
 615
 616         if (ipv6_addr_v4mapped(&sin6->sin6_addr))
 617                 return tcp_md5_do_add(sk, (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3],
 618                                       AF_INET, prefixlen, l3index,
 619                                       cmd.tcpm_key, cmd.tcpm_keylen,
 620                                       GFP_KERNEL);
 621
 622         return tcp_md5_do_add(sk, (union tcp_md5_addr *)&sin6->sin6_addr,
 623                               AF_INET6, prefixlen, l3index,
 624                               cmd.tcpm_key, cmd.tcpm_keylen, GFP_KERNEL);
 625 }
 626
 627 static int tcp_v6_md5_hash_headers(struct tcp_md5sig_pool *hp,
 628                                    const struct in6_addr *daddr,
 629                                    const struct in6_addr *saddr,
 630                                    const struct tcphdr *th, int nbytes)
 631 {
 632         struct tcp6_pseudohdr *bp;
 633         struct scatterlist sg;
 634         struct tcphdr *_th;
 635
 636         bp = hp->scratch;
 637         /* 1. TCP pseudo-header (RFC2460) */
 638         bp->saddr = *saddr;
 639         bp->daddr = *daddr;
 640         bp->protocol = cpu_to_be32(IPPROTO_TCP);
 641         bp->len = cpu_to_be32(nbytes);
 642
 643         _th = (struct tcphdr *)(bp + 1);
 644         memcpy(_th, th, sizeof(*th));
 645         _th->check = 0;
 646
 647         sg_init_one(&sg, bp, sizeof(*bp) + sizeof(*th));
 648         ahash_request_set_crypt(hp->md5_req, &sg, NULL,
 649                                 sizeof(*bp) + sizeof(*th));
 650         return crypto_ahash_update(hp->md5_req);
 651 }
 652
 653 static int tcp_v6_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
 654                                const struct in6_addr *daddr, struct in6_addr *saddr,
 655                                const struct tcphdr *th)
 656 {
 657         struct tcp_md5sig_pool *hp;
 658         struct ahash_request *req;
 659
 660         hp = tcp_get_md5sig_pool();
 661         if (!hp)
 662                 goto clear_hash_noput;
 663         req = hp->md5_req;
 664
 665         if (crypto_ahash_init(req))
 666                 goto clear_hash;
 667         if (tcp_v6_md5_hash_headers(hp, daddr, saddr, th, th->doff << 2))
 668                 goto clear_hash;
 669         if (tcp_md5_hash_key(hp, key))
 670                 goto clear_hash;
 671         ahash_request_set_crypt(req, NULL, md5_hash, 0);
 672         if (crypto_ahash_final(req))
 673                 goto clear_hash;
 674
 675         tcp_put_md5sig_pool();
 676         return 0;
 677
 678 clear_hash:
 679         tcp_put_md5sig_pool();
 680 clear_hash_noput:
 681         memset(md5_hash, 0, 16);
 682         return 1;
 683 }
 684
 685 static int tcp_v6_md5_hash_skb(char *md5_hash,
 686                                const struct tcp_md5sig_key *key,
 687                                const struct sock *sk,
 688                                const struct sk_buff *skb)
 689 {
 690         const struct in6_addr *saddr, *daddr;
 691         struct tcp_md5sig_pool *hp;
 692         struct ahash_request *req;
 693         const struct tcphdr *th = tcp_hdr(skb);
 694
 695         if (sk) { /* valid for establish/request sockets */
 696                 saddr = &sk->sk_v6_rcv_saddr;
 697                 daddr = &sk->sk_v6_daddr;
 698         } else {
 699                 const struct ipv6hdr *ip6h = ipv6_hdr(skb);
 700                 saddr = &ip6h->saddr;
 701                 daddr = &ip6h->daddr;
 702         }
 703
 704         hp = tcp_get_md5sig_pool();
 705         if (!hp)
 706                 goto clear_hash_noput;
 707         req = hp->md5_req;
 708
 709         if (crypto_ahash_init(req))
 710                 goto clear_hash;
 711
 712         if (tcp_v6_md5_hash_headers(hp, daddr, saddr, th, skb->len))
 713                 goto clear_hash;
 714         if (tcp_md5_hash_skb_data(hp, skb, th->doff << 2))
 715                 goto clear_hash;
 716         if (tcp_md5_hash_key(hp, key))
 717                 goto clear_hash;
 718         ahash_request_set_crypt(req, NULL, md5_hash, 0);
 719         if (crypto_ahash_final(req))
 720                 goto clear_hash;
 721
 722         tcp_put_md5sig_pool();
 723         return 0;
 724
 725 clear_hash:
 726         tcp_put_md5sig_pool();
 727 clear_hash_noput:
 728         memset(md5_hash, 0, 16);
 729         return 1;
 730 }
 731
 732 #endif
 733
 734 static bool tcp_v6_inbound_md5_hash(const struct sock *sk,
 735                                     const struct sk_buff *skb,
 736                                     int dif, int sdif)
 737 {
 738 #ifdef CONFIG_TCP_MD5SIG
 739         const __u8 *hash_location = NULL;
 740         struct tcp_md5sig_key *hash_expected;
 741         const struct ipv6hdr *ip6h = ipv6_hdr(skb);
 742         const struct tcphdr *th = tcp_hdr(skb);
 743         int genhash, l3index;
 744         u8 newhash[16];
 745
 746         /* sdif set, means packet ingressed via a device
 747          * in an L3 domain and dif is set to the l3mdev
 748          */
 749         l3index = sdif ? dif : 0;
 750
 751         hash_expected = tcp_v6_md5_do_lookup(sk, &ip6h->saddr, l3index);
 752         hash_location = tcp_parse_md5sig_option(th);
 753
 754         /* We've parsed the options - do we have a hash? */
 755         if (!hash_expected && !hash_location)
 756                 return false;
 757
 758         if (hash_expected && !hash_location) {
 759                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5NOTFOUND);
 760                 return true;
 761         }
 762
 763         if (!hash_expected && hash_location) {
 764                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5UNEXPECTED);
 765                 return true;
 766         }
 767
 768         /* check the signature */
 769         genhash = tcp_v6_md5_hash_skb(newhash,
 770                                       hash_expected,
 771                                       NULL, skb);
 772
 773         if (genhash || memcmp(hash_location, newhash, 16) != 0) {
 774                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5FAILURE);
 775                 net_info_ratelimited("MD5 Hash %s for [%pI6c]:%u->[%pI6c]:%u L3 index %d\n",
 776                                      genhash ? "failed" : "mismatch",
 777                                      &ip6h->saddr, ntohs(th->source),
 778                                      &ip6h->daddr, ntohs(th->dest), l3index);
 779                 return true;
 780         }
 781 #endif
 782         return false;
 783 }
 784
 785 static void tcp_v6_init_req(struct request_sock *req,
 786                             const struct sock *sk_listener,
 787                             struct sk_buff *skb)
 788 {
 789         bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags);
 790         struct inet_request_sock *ireq = inet_rsk(req);
 791         const struct ipv6_pinfo *np = tcp_inet6_sk(sk_listener);
 792
 793         ireq->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr;
 794         ireq->ir_v6_loc_addr = ipv6_hdr(skb)->daddr;
 795
 796         /* So that link locals have meaning */
 797         if ((!sk_listener->sk_bound_dev_if || l3_slave) &&
 798             ipv6_addr_type(&ireq->ir_v6_rmt_addr) & IPV6_ADDR_LINKLOCAL)
 799                 ireq->ir_iif = tcp_v6_iif(skb);
 800
 801         if (!TCP_SKB_CB(skb)->tcp_tw_isn &&
 802             (ipv6_opt_accepted(sk_listener, skb, &TCP_SKB_CB(skb)->header.h6) ||
 803              np->rxopt.bits.rxinfo ||
 804              np->rxopt.bits.rxoinfo || np->rxopt.bits.rxhlim ||
 805              np->rxopt.bits.rxohlim || np->repflow)) {
 806                 refcount_inc(&skb->users);
 807                 ireq->pktopts = skb;
 808         }
 809 }
 810
 811 static struct dst_entry *tcp_v6_route_req(const struct sock *sk,
 812                                           struct flowi *fl,
 813                                           const struct request_sock *req)
 814 {
 815         return inet6_csk_route_req(sk, &fl->u.ip6, req, IPPROTO_TCP);
 816 }
 817
 818 struct request_sock_ops tcp6_request_sock_ops __read_mostly = {
 819         .family         =       AF_INET6,
 820         .obj_size       =       sizeof(struct tcp6_request_sock),
 821         .rtx_syn_ack    =       tcp_rtx_synack,
 822         .send_ack       =       tcp_v6_reqsk_send_ack,
 823         .destructor     =       tcp_v6_reqsk_destructor,
 824         .send_reset     =       tcp_v6_send_reset,
 825         .syn_ack_timeout =      tcp_syn_ack_timeout,
 826 };
 827
 828 const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops = {
 829         .mss_clamp      =       IPV6_MIN_MTU - sizeof(struct tcphdr) -
 830                                 sizeof(struct ipv6hdr),
 831 #ifdef CONFIG_TCP_MD5SIG
 832         .req_md5_lookup =       tcp_v6_md5_lookup,
 833         .calc_md5_hash  =       tcp_v6_md5_hash_skb,
 834 #endif
 835         .init_req       =       tcp_v6_init_req,
 836 #ifdef CONFIG_SYN_COOKIES
 837         .cookie_init_seq =      cookie_v6_init_sequence,
 838 #endif
 839         .route_req      =       tcp_v6_route_req,
 840         .init_seq       =       tcp_v6_init_seq,
 841         .init_ts_off    =       tcp_v6_init_ts_off,
 842         .send_synack    =       tcp_v6_send_synack,
 843 };
 844
 845 static void tcp_v6_send_response(const struct sock *sk, struct sk_buff *skb, u32 seq,
 846                                  u32 ack, u32 win, u32 tsval, u32 tsecr,
 847                                  int oif, struct tcp_md5sig_key *key, int rst,
 848                                  u8 tclass, __be32 label, u32 priority)
 849 {
 850         const struct tcphdr *th = tcp_hdr(skb);
 851         struct tcphdr *t1;
 852         struct sk_buff *buff;
 853         struct flowi6 fl6;
 854         struct net *net = sk ? sock_net(sk) : dev_net(skb_dst(skb)->dev);
 855         struct sock *ctl_sk = net->ipv6.tcp_sk;
 856         unsigned int tot_len = sizeof(struct tcphdr);
 857         struct dst_entry *dst;
 858         __be32 *topt;
 859         __u32 mark = 0;
 860
 861         if (tsecr)
 862                 tot_len += TCPOLEN_TSTAMP_ALIGNED;
 863 #ifdef CONFIG_TCP_MD5SIG
 864         if (key)
 865                 tot_len += TCPOLEN_MD5SIG_ALIGNED;
 866 #endif
 867
 868         buff = alloc_skb(MAX_HEADER + sizeof(struct ipv6hdr) + tot_len,
 869                          GFP_ATOMIC);
 870         if (!buff)
 871                 return;
 872
 873         skb_reserve(buff, MAX_HEADER + sizeof(struct ipv6hdr) + tot_len);
 874
 875         t1 = skb_push(buff, tot_len);
 876         skb_reset_transport_header(buff);
 877
 878         /* Swap the send and the receive. */
 879         memset(t1, 0, sizeof(*t1));
 880         t1->dest = th->source;
 881         t1->source = th->dest;
 882         t1->doff = tot_len / 4;
 883         t1->seq = htonl(seq);
 884         t1->ack_seq = htonl(ack);
 885         t1->ack = !rst || !th->ack;
 886         t1->rst = rst;
 887         t1->window = htons(win);
 888
 889         topt = (__be32 *)(t1 + 1);
 890
 891         if (tsecr) {
 892                 *topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
 893                                 (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP);
 894                 *topt++ = htonl(tsval);
 895                 *topt++ = htonl(tsecr);
 896         }
 897
 898 #ifdef CONFIG_TCP_MD5SIG
 899         if (key) {
 900                 *topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
 901                                 (TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG);
 902                 tcp_v6_md5_hash_hdr((__u8 *)topt, key,
 903                                     &ipv6_hdr(skb)->saddr,
 904                                     &ipv6_hdr(skb)->daddr, t1);
 905         }
 906 #endif
 907
 908         memset(&fl6, 0, sizeof(fl6));
 909         fl6.daddr = ipv6_hdr(skb)->saddr;
 910         fl6.saddr = ipv6_hdr(skb)->daddr;
 911         fl6.flowlabel = label;
 912
 913         buff->ip_summed = CHECKSUM_PARTIAL;
 914         buff->csum = 0;
 915
 916         __tcp_v6_send_check(buff, &fl6.saddr, &fl6.daddr);
 917
 918         fl6.flowi6_proto = IPPROTO_TCP;
 919         if (rt6_need_strict(&fl6.daddr) && !oif)
 920                 fl6.flowi6_oif = tcp_v6_iif(skb);
 921         else {
 922                 if (!oif && netif_index_is_l3_master(net, skb->skb_iif))
 923                         oif = skb->skb_iif;
 924
 925                 fl6.flowi6_oif = oif;
 926         }
 927
 928         if (sk) {
 929                 if (sk->sk_state == TCP_TIME_WAIT) {
 930                         mark = inet_twsk(sk)->tw_mark;
 931                         /* autoflowlabel relies on buff->hash */
 932                         skb_set_hash(buff, inet_twsk(sk)->tw_txhash,
 933                                      PKT_HASH_TYPE_L4);
 934                 } else {
 935                         mark = sk->sk_mark;
 936                 }
 937                 buff->tstamp = tcp_transmit_time(sk);
 938         }
 939         fl6.flowi6_mark = IP6_REPLY_MARK(net, skb->mark) ?: mark;
 940         fl6.fl6_dport = t1->dest;
 941         fl6.fl6_sport = t1->source;
 942         fl6.flowi6_uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL);
 943         security_skb_classify_flow(skb, flowi6_to_flowi(&fl6));
 944
 945         /* Pass a socket to ip6_dst_lookup either it is for RST
 946          * Underlying function will use this to retrieve the network
 947          * namespace
 948          */
 949         dst = ip6_dst_lookup_flow(sock_net(ctl_sk), ctl_sk, &fl6, NULL);
 950         if (!IS_ERR(dst)) {
 951                 skb_dst_set(buff, dst);
 952                 ip6_xmit(ctl_sk, buff, &fl6, fl6.flowi6_mark, NULL, tclass,
 953                          priority);
 954                 TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
 955                 if (rst)
 956                         TCP_INC_STATS(net, TCP_MIB_OUTRSTS);
 957                 return;
 958         }
 959
 960         kfree_skb(buff);
 961 }
 962
 963 static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb)
 964 {
 965         const struct tcphdr *th = tcp_hdr(skb);
 966         struct ipv6hdr *ipv6h = ipv6_hdr(skb);
 967         u32 seq = 0, ack_seq = 0;
 968         struct tcp_md5sig_key *key = NULL;
 969 #ifdef CONFIG_TCP_MD5SIG
 970         const __u8 *hash_location = NULL;
 971         unsigned char newhash[16];
 972         int genhash;
 973         struct sock *sk1 = NULL;
 974 #endif
 975         __be32 label = 0;
 976         u32 priority = 0;
 977         struct net *net;
 978         int oif = 0;
 979
 980         if (th->rst)
 981                 return;
 982
 983         /* If sk not NULL, it means we did a successful lookup and incoming
 984          * route had to be correct. prequeue might have dropped our dst.
 985          */
 986         if (!sk && !ipv6_unicast_destination(skb))
 987                 return;
 988
 989         net = sk ? sock_net(sk) : dev_net(skb_dst(skb)->dev);
 990 #ifdef CONFIG_TCP_MD5SIG
 991         rcu_read_lock();
 992         hash_location = tcp_parse_md5sig_option(th);
 993         if (sk && sk_fullsock(sk)) {
 994                 int l3index;
 995
 996                 /* sdif set, means packet ingressed via a device
 997                  * in an L3 domain and inet_iif is set to it.
 998                  */
 999                 l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1000                 key = tcp_v6_md5_do_lookup(sk, &ipv6h->saddr, l3index);
1001         } else if (hash_location) {
1002                 int dif = tcp_v6_iif_l3_slave(skb);
1003                 int sdif = tcp_v6_sdif(skb);
1004                 int l3index;
1005
1006                 /*
1007                  * active side is lost. Try to find listening socket through
1008                  * source port, and then find md5 key through listening socket.
1009                  * we are not loose security here:
1010                  * Incoming packet is checked with md5 hash with finding key,
1011                  * no RST generated if md5 hash doesn't match.
1012                  */
1013                 sk1 = inet6_lookup_listener(net,
1014                                            &tcp_hashinfo, NULL, 0,
1015                                            &ipv6h->saddr,
1016                                            th->source, &ipv6h->daddr,
1017                                            ntohs(th->source), dif, sdif);
1018                 if (!sk1)
1019                         goto out;
1020
1021                 /* sdif set, means packet ingressed via a device
1022                  * in an L3 domain and dif is set to it.
1023                  */
1024                 l3index = tcp_v6_sdif(skb) ? dif : 0;
1025
1026                 key = tcp_v6_md5_do_lookup(sk1, &ipv6h->saddr, l3index);
1027                 if (!key)
1028                         goto out;
1029
1030                 genhash = tcp_v6_md5_hash_skb(newhash, key, NULL, skb);
1031                 if (genhash || memcmp(hash_location, newhash, 16) != 0)
1032                         goto out;
1033         }
1034 #endif
1035
1036         if (th->ack)
1037                 seq = ntohl(th->ack_seq);
1038         else
1039                 ack_seq = ntohl(th->seq) + th->syn + th->fin + skb->len -
1040                           (th->doff << 2);
1041
1042         if (sk) {
1043                 oif = sk->sk_bound_dev_if;
1044                 if (sk_fullsock(sk)) {
1045                         const struct ipv6_pinfo *np = tcp_inet6_sk(sk);
1046
1047                         trace_tcp_send_reset(sk, skb);
1048                         if (np->repflow)
1049                                 label = ip6_flowlabel(ipv6h);
1050                         priority = sk->sk_priority;
1051                 }
1052                 if (sk->sk_state == TCP_TIME_WAIT) {
1053                         label = cpu_to_be32(inet_twsk(sk)->tw_flowlabel);
1054                         priority = inet_twsk(sk)->tw_priority;
1055                 }
1056         } else {
1057                 if (net->ipv6.sysctl.flowlabel_reflect & FLOWLABEL_REFLECT_TCP_RESET)
1058                         label = ip6_flowlabel(ipv6h);
1059         }
1060
1061         tcp_v6_send_response(sk, skb, seq, ack_seq, 0, 0, 0, oif, key, 1, 0,
1062                              label, priority);
1063
1064 #ifdef CONFIG_TCP_MD5SIG
1065 out:
1066         rcu_read_unlock();
1067 #endif
1068 }
1069
1070 static void tcp_v6_send_ack(const struct sock *sk, struct sk_buff *skb, u32 seq,
1071                             u32 ack, u32 win, u32 tsval, u32 tsecr, int oif,
1072                             struct tcp_md5sig_key *key, u8 tclass,
1073                             __be32 label, u32 priority)
1074 {
1075         tcp_v6_send_response(sk, skb, seq, ack, win, tsval, tsecr, oif, key, 0,
1076                              tclass, label, priority);
1077 }
1078
1079 static void tcp_v6_timewait_ack(struct sock *sk, struct sk_buff *skb)
1080 {
1081         struct inet_timewait_sock *tw = inet_twsk(sk);
1082         struct tcp_timewait_sock *tcptw = tcp_twsk(sk);
1083
1084         tcp_v6_send_ack(sk, skb, tcptw->tw_snd_nxt, tcptw->tw_rcv_nxt,
1085                         tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale,
1086                         tcp_time_stamp_raw() + tcptw->tw_ts_offset,
1087                         tcptw->tw_ts_recent, tw->tw_bound_dev_if, tcp_twsk_md5_key(tcptw),
1088                         tw->tw_tclass, cpu_to_be32(tw->tw_flowlabel), tw->tw_priority);
1089
1090         inet_twsk_put(tw);
1091 }
1092
1093 static void tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
1094                                   struct request_sock *req)
1095 {
1096         int l3index;
1097
1098         l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1099
1100         /* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV
1101          * sk->sk_state == TCP_SYN_RECV -> for Fast Open.
1102          */
1103         /* RFC 7323 2.3
1104          * The window field (SEG.WND) of every outgoing segment, with the
1105          * exception of <SYN> segments, MUST be right-shifted by
1106          * Rcv.Wind.Shift bits:
1107          */
1108         tcp_v6_send_ack(sk, skb, (sk->sk_state == TCP_LISTEN) ?
1109                         tcp_rsk(req)->snt_isn + 1 : tcp_sk(sk)->snd_nxt,
1110                         tcp_rsk(req)->rcv_nxt,
1111                         req->rsk_rcv_wnd >> inet_rsk(req)->rcv_wscale,
1112                         tcp_time_stamp_raw() + tcp_rsk(req)->ts_off,
1113                         req->ts_recent, sk->sk_bound_dev_if,
1114                         tcp_v6_md5_do_lookup(sk, &ipv6_hdr(skb)->saddr, l3index),
1115                         0, 0, sk->sk_priority);
1116 }
1117
1118
1119 static struct sock *tcp_v6_cookie_check(struct sock *sk, struct sk_buff *skb)
1120 {
1121 #ifdef CONFIG_SYN_COOKIES
1122         const struct tcphdr *th = tcp_hdr(skb);
1123
1124         if (!th->syn)
1125                 sk = cookie_v6_check(sk, skb);
1126 #endif
1127         return sk;
1128 }
1129
1130 u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph,
1131                          struct tcphdr *th, u32 *cookie)
1132 {
1133         u16 mss = 0;
1134 #ifdef CONFIG_SYN_COOKIES
1135         mss = tcp_get_syncookie_mss(&tcp6_request_sock_ops,
1136                                     &tcp_request_sock_ipv6_ops, sk, th);
1137         if (mss) {
1138                 *cookie = __cookie_v6_init_sequence(iph, th, &mss);
1139                 tcp_synq_overflow(sk);
1140         }
1141 #endif
1142         return mss;
1143 }
1144
1145 static int tcp_v6_conn_request(struct sock *sk, struct sk_buff *skb)
1146 {
1147         if (skb->protocol == htons(ETH_P_IP))
1148                 return tcp_v4_conn_request(sk, skb);
1149
1150         if (!ipv6_unicast_destination(skb))
1151                 goto drop;
1152
1153         return tcp_conn_request(&tcp6_request_sock_ops,
1154                                 &tcp_request_sock_ipv6_ops, sk, skb);
1155
1156 drop:
1157         tcp_listendrop(sk);
1158         return 0; /* don't send reset */
1159 }
1160
1161 static void tcp_v6_restore_cb(struct sk_buff *skb)
1162 {
1163         /* We need to move header back to the beginning if xfrm6_policy_check()
1164          * and tcp_v6_fill_cb() are going to be called again.
1165          * ip6_datagram_recv_specific_ctl() also expects IP6CB to be there.
1166          */
1167         memmove(IP6CB(skb), &TCP_SKB_CB(skb)->header.h6,
1168                 sizeof(struct inet6_skb_parm));
1169 }
1170
1171 static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
1172                                          struct request_sock *req,
1173                                          struct dst_entry *dst,
1174                                          struct request_sock *req_unhash,
1175                                          bool *own_req)
1176 {
1177         struct inet_request_sock *ireq;
1178         struct ipv6_pinfo *newnp;
1179         const struct ipv6_pinfo *np = tcp_inet6_sk(sk);
1180         struct ipv6_txoptions *opt;
1181         struct inet_sock *newinet;
1182         struct tcp_sock *newtp;
1183         struct sock *newsk;
1184 #ifdef CONFIG_TCP_MD5SIG
1185         struct tcp_md5sig_key *key;
1186         int l3index;
1187 #endif
1188         struct flowi6 fl6;
1189
1190         if (skb->protocol == htons(ETH_P_IP)) {
1191                 /*
1192                  *      v6 mapped
1193                  */
1194
1195                 newsk = tcp_v4_syn_recv_sock(sk, skb, req, dst,
1196                                              req_unhash, own_req);
1197
1198                 if (!newsk)
1199                         return NULL;
1200
1201                 inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk);
1202
1203                 newinet = inet_sk(newsk);
1204                 newnp = tcp_inet6_sk(newsk);
1205                 newtp = tcp_sk(newsk);
1206
1207                 memcpy(newnp, np, sizeof(struct ipv6_pinfo));
1208
1209                 newnp->saddr = newsk->sk_v6_rcv_saddr;
1210
1211                 inet_csk(newsk)->icsk_af_ops = &ipv6_mapped;
1212                 if (sk_is_mptcp(newsk))
1213                         mptcpv6_handle_mapped(newsk, true);
1214                 newsk->sk_backlog_rcv = tcp_v4_do_rcv;
1215 #ifdef CONFIG_TCP_MD5SIG
1216                 newtp->af_specific = &tcp_sock_ipv6_mapped_specific;
1217 #endif
1218
1219                 newnp->ipv6_mc_list = NULL;
1220                 newnp->ipv6_ac_list = NULL;
1221                 newnp->ipv6_fl_list = NULL;
1222                 newnp->pktoptions  = NULL;
1223                 newnp->opt         = NULL;
1224                 newnp->mcast_oif   = inet_iif(skb);
1225                 newnp->mcast_hops  = ip_hdr(skb)->ttl;
1226                 newnp->rcv_flowinfo = 0;
1227                 if (np->repflow)
1228                         newnp->flow_label = 0;
1229
1230                 /*
1231                  * No need to charge this sock to the relevant IPv6 refcnt debug socks count
1232                  * here, tcp_create_openreq_child now does this for us, see the comment in
1233                  * that function for the gory details. -acme
1234                  */
1235
1236                 /* It is tricky place. Until this moment IPv4 tcp
1237                    worked with IPv6 icsk.icsk_af_ops.
1238                    Sync it now.
1239                  */
1240                 tcp_sync_mss(newsk, inet_csk(newsk)->icsk_pmtu_cookie);
1241
1242                 return newsk;
1243         }
1244
1245         ireq = inet_rsk(req);
1246
1247         if (sk_acceptq_is_full(sk))
1248                 goto out_overflow;
1249
1250         if (!dst) {
1251                 dst = inet6_csk_route_req(sk, &fl6, req, IPPROTO_TCP);
1252                 if (!dst)
1253                         goto out;
1254         }
1255
1256         newsk = tcp_create_openreq_child(sk, req, skb);
1257         if (!newsk)
1258                 goto out_nonewsk;
1259
1260         /*
1261          * No need to charge this sock to the relevant IPv6 refcnt debug socks
1262          * count here, tcp_create_openreq_child now does this for us, see the
1263          * comment in that function for the gory details. -acme
1264          */
1265
1266         newsk->sk_gso_type = SKB_GSO_TCPV6;
1267         ip6_dst_store(newsk, dst, NULL, NULL);
1268         inet6_sk_rx_dst_set(newsk, skb);
1269
1270         inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk);
1271
1272         newtp = tcp_sk(newsk);
1273         newinet = inet_sk(newsk);
1274         newnp = tcp_inet6_sk(newsk);
1275
1276         memcpy(newnp, np, sizeof(struct ipv6_pinfo));
1277
1278         newsk->sk_v6_daddr = ireq->ir_v6_rmt_addr;
1279         newnp->saddr = ireq->ir_v6_loc_addr;
1280         newsk->sk_v6_rcv_saddr = ireq->ir_v6_loc_addr;
1281         newsk->sk_bound_dev_if = ireq->ir_iif;
1282
1283         /* Now IPv6 options...
1284
1285            First: no IPv4 options.
1286          */
1287         newinet->inet_opt = NULL;
1288         newnp->ipv6_mc_list = NULL;
1289         newnp->ipv6_ac_list = NULL;
1290         newnp->ipv6_fl_list = NULL;
1291
1292         /* Clone RX bits */
1293         newnp->rxopt.all = np->rxopt.all;
1294
1295         newnp->pktoptions = NULL;
1296         newnp->opt        = NULL;
1297         newnp->mcast_oif  = tcp_v6_iif(skb);
1298         newnp->mcast_hops = ipv6_hdr(skb)->hop_limit;
1299         newnp->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(skb));
1300         if (np->repflow)
1301                 newnp->flow_label = ip6_flowlabel(ipv6_hdr(skb));
1302
1303         /* Clone native IPv6 options from listening socket (if any)
1304
1305            Yes, keeping reference count would be much more clever,
1306            but we make one more one thing there: reattach optmem
1307            to newsk.
1308          */
1309         opt = ireq->ipv6_opt;
1310         if (!opt)
1311                 opt = rcu_dereference(np->opt);
1312         if (opt) {
1313                 opt = ipv6_dup_options(newsk, opt);
1314                 RCU_INIT_POINTER(newnp->opt, opt);
1315         }
1316         inet_csk(newsk)->icsk_ext_hdr_len = 0;
1317         if (opt)
1318                 inet_csk(newsk)->icsk_ext_hdr_len = opt->opt_nflen +
1319                                                     opt->opt_flen;
1320
1321         tcp_ca_openreq_child(newsk, dst);
1322
1323         tcp_sync_mss(newsk, dst_mtu(dst));
1324         newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst));
1325
1326         tcp_initialize_rcv_mss(newsk);
1327
1328         newinet->inet_daddr = newinet->inet_saddr = LOOPBACK4_IPV6;
1329         newinet->inet_rcv_saddr = LOOPBACK4_IPV6;
1330
1331 #ifdef CONFIG_TCP_MD5SIG
1332         l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif);
1333
1334         /* Copy over the MD5 key from the original socket */
1335         key = tcp_v6_md5_do_lookup(sk, &newsk->sk_v6_daddr, l3index);
1336         if (key) {
1337                 /* We're using one, so create a matching key
1338                  * on the newsk structure. If we fail to get
1339                  * memory, then we end up not copying the key
1340                  * across. Shucks.
1341                  */
1342                 tcp_md5_do_add(newsk, (union tcp_md5_addr *)&newsk->sk_v6_daddr,
1343                                AF_INET6, 128, l3index, key->key, key->keylen,
1344                                sk_gfp_mask(sk, GFP_ATOMIC));
1345         }
1346 #endif
1347
1348         if (__inet_inherit_port(sk, newsk) < 0) {
1349                 inet_csk_prepare_forced_close(newsk);
1350                 tcp_done(newsk);
1351                 goto out;
1352         }
1353         *own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash));
1354         if (*own_req) {
1355                 tcp_move_syn(newtp, req);
1356
1357                 /* Clone pktoptions received with SYN, if we own the req */
1358                 if (ireq->pktopts) {
1359                         newnp->pktoptions = skb_clone(ireq->pktopts,
1360                                                       sk_gfp_mask(sk, GFP_ATOMIC));
1361                         consume_skb(ireq->pktopts);
1362                         ireq->pktopts = NULL;
1363                         if (newnp->pktoptions) {
1364                                 tcp_v6_restore_cb(newnp->pktoptions);
1365                                 skb_set_owner_r(newnp->pktoptions, newsk);
1366                         }
1367                 }
1368         }
1369
1370         return newsk;
1371
1372 out_overflow:
1373         __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
1374 out_nonewsk:
1375         dst_release(dst);
1376 out:
1377         tcp_listendrop(sk);
1378         return NULL;
1379 }
1380
1381 /* The socket must have it's spinlock held when we get
1382  * here, unless it is a TCP_LISTEN socket.
1383  *
1384  * We have a potential double-lock case here, so even when
1385  * doing backlog processing we use the BH locking scheme.
1386  * This is because we cannot sleep with the original spinlock
1387  * held.
1388  */
1389 static int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)
1390 {
1391         struct ipv6_pinfo *np = tcp_inet6_sk(sk);
1392         struct sk_buff *opt_skb = NULL;
1393         struct tcp_sock *tp;
1394
1395         /* Imagine: socket is IPv6. IPv4 packet arrives,
1396            goes to IPv4 receive handler and backlogged.
1397            From backlog it always goes here. Kerboom...
1398            Fortunately, tcp_rcv_established and rcv_established
1399            handle them correctly, but it is not case with
1400            tcp_v6_hnd_req and tcp_v6_send_reset().   --ANK
1401          */
1402
1403         if (skb->protocol == htons(ETH_P_IP))
1404                 return tcp_v4_do_rcv(sk, skb);
1405
1406         /*
1407          *      socket locking is here for SMP purposes as backlog rcv
1408          *      is currently called with bh processing disabled.
1409          */
1410
1411         /* Do Stevens' IPV6_PKTOPTIONS.
1412
1413            Yes, guys, it is the only place in our code, where we
1414            may make it not affecting IPv4.
1415            The rest of code is protocol independent,
1416            and I do not like idea to uglify IPv4.
1417
1418            Actually, all the idea behind IPV6_PKTOPTIONS
1419            looks not very well thought. For now we latch
1420            options, received in the last packet, enqueued
1421            by tcp. Feel free to propose better solution.
1422                                                --ANK (980728)
1423          */
1424         if (np->rxopt.all)
1425                 opt_skb = skb_clone(skb, sk_gfp_mask(sk, GFP_ATOMIC));
1426
1427         if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
1428                 struct dst_entry *dst = sk->sk_rx_dst;
1429
1430                 sock_rps_save_rxhash(sk, skb);
1431                 sk_mark_napi_id(sk, skb);
1432                 if (dst) {
1433                         if (inet_sk(sk)->rx_dst_ifindex != skb->skb_iif ||
1434                             dst->ops->check(dst, np->rx_dst_cookie) == NULL) {
1435                                 dst_release(dst);
1436                                 sk->sk_rx_dst = NULL;
1437                         }
1438                 }
1439
1440                 tcp_rcv_established(sk, skb);
1441                 if (opt_skb)
1442                         goto ipv6_pktoptions;
1443                 return 0;
1444         }
1445
1446         if (tcp_checksum_complete(skb))
1447                 goto csum_err;
1448
1449         if (sk->sk_state == TCP_LISTEN) {
1450                 struct sock *nsk = tcp_v6_cookie_check(sk, skb);
1451
1452                 if (!nsk)
1453                         goto discard;
1454
1455                 if (nsk != sk) {
1456                         if (tcp_child_process(sk, nsk, skb))
1457                                 goto reset;
1458                         if (opt_skb)
1459                                 __kfree_skb(opt_skb);
1460                         return 0;
1461                 }
1462         } else
1463                 sock_rps_save_rxhash(sk, skb);
1464
1465         if (tcp_rcv_state_process(sk, skb))
1466                 goto reset;
1467         if (opt_skb)
1468                 goto ipv6_pktoptions;
1469         return 0;
1470
1471 reset:
1472         tcp_v6_send_reset(sk, skb);
1473 discard:
1474         if (opt_skb)
1475                 __kfree_skb(opt_skb);
1476         kfree_skb(skb);
1477         return 0;
1478 csum_err:
1479         TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1480         TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1481         goto discard;
1482
1483
1484 ipv6_pktoptions:
1485         /* Do you ask, what is it?
1486
1487            1. skb was enqueued by tcp.
1488            2. skb is added to tail of read queue, rather than out of order.
1489            3. socket is not in passive state.
1490            4. Finally, it really contains options, which user wants to receive.
1491          */
1492         tp = tcp_sk(sk);
1493         if (TCP_SKB_CB(opt_skb)->end_seq == tp->rcv_nxt &&
1494             !((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) {
1495                 if (np->rxopt.bits.rxinfo || np->rxopt.bits.rxoinfo)
1496                         np->mcast_oif = tcp_v6_iif(opt_skb);
1497                 if (np->rxopt.bits.rxhlim || np->rxopt.bits.rxohlim)
1498                         np->mcast_hops = ipv6_hdr(opt_skb)->hop_limit;
1499                 if (np->rxopt.bits.rxflow || np->rxopt.bits.rxtclass)
1500                         np->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(opt_skb));
1501                 if (np->repflow)
1502                         np->flow_label = ip6_flowlabel(ipv6_hdr(opt_skb));
1503                 if (ipv6_opt_accepted(sk, opt_skb, &TCP_SKB_CB(opt_skb)->header.h6)) {
1504                         skb_set_owner_r(opt_skb, sk);
1505                         tcp_v6_restore_cb(opt_skb);
1506                         opt_skb = xchg(&np->pktoptions, opt_skb);
1507                 } else {
1508                         __kfree_skb(opt_skb);
1509                         opt_skb = xchg(&np->pktoptions, NULL);
1510                 }
1511         }
1512
1513         kfree_skb(opt_skb);
1514         return 0;
1515 }
1516
1517 static void tcp_v6_fill_cb(struct sk_buff *skb, const struct ipv6hdr *hdr,
1518                            const struct tcphdr *th)
1519 {
1520         /* This is tricky: we move IP6CB at its correct location into
1521          * TCP_SKB_CB(). It must be done after xfrm6_policy_check(), because
1522          * _decode_session6() uses IP6CB().
1523          * barrier() makes sure compiler won't play aliasing games.
1524          */
1525         memmove(&TCP_SKB_CB(skb)->header.h6, IP6CB(skb),
1526                 sizeof(struct inet6_skb_parm));
1527         barrier();
1528
1529         TCP_SKB_CB(skb)->seq = ntohl(th->seq);
1530         TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
1531                                     skb->len - th->doff*4);
1532         TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
1533         TCP_SKB_CB(skb)->tcp_flags = tcp_flag_byte(th);
1534         TCP_SKB_CB(skb)->tcp_tw_isn = 0;
1535         TCP_SKB_CB(skb)->ip_dsfield = ipv6_get_dsfield(hdr);
1536         TCP_SKB_CB(skb)->sacked = 0;
1537         TCP_SKB_CB(skb)->has_rxtstamp =
1538                         skb->tstamp || skb_hwtstamps(skb)->hwtstamp;
1539 }
1540
1541 INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb)
1542 {
1543         struct sk_buff *skb_to_free;
1544         int sdif = inet6_sdif(skb);
1545         int dif = inet6_iif(skb);
1546         const struct tcphdr *th;
1547         const struct ipv6hdr *hdr;
1548         bool refcounted;
1549         struct sock *sk;
1550         int ret;
1551         struct net *net = dev_net(skb->dev);
1552
1553         if (skb->pkt_type != PACKET_HOST)
1554                 goto discard_it;
1555
1556         /*
1557          *      Count it even if it's bad.
1558          */
1559         __TCP_INC_STATS(net, TCP_MIB_INSEGS);
1560
1561         if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
1562                 goto discard_it;
1563
1564         th = (const struct tcphdr *)skb->data;
1565
1566         if (unlikely(th->doff < sizeof(struct tcphdr)/4))
1567                 goto bad_packet;
1568         if (!pskb_may_pull(skb, th->doff*4))
1569                 goto discard_it;
1570
1571         if (skb_checksum_init(skb, IPPROTO_TCP, ip6_compute_pseudo))
1572                 goto csum_error;
1573
1574         th = (const struct tcphdr *)skb->data;
1575         hdr = ipv6_hdr(skb);
1576
1577 lookup:
1578         sk = __inet6_lookup_skb(&tcp_hashinfo, skb, __tcp_hdrlen(th),
1579                                 th->source, th->dest, inet6_iif(skb), sdif,
1580                                 &refcounted);
1581         if (!sk)
1582                 goto no_tcp_socket;
1583
1584 process:
1585         if (sk->sk_state == TCP_TIME_WAIT)
1586                 goto do_time_wait;
1587
1588         if (sk->sk_state == TCP_NEW_SYN_RECV) {
1589                 struct request_sock *req = inet_reqsk(sk);
1590                 bool req_stolen = false;
1591                 struct sock *nsk;
1592
1593                 sk = req->rsk_listener;
1594                 if (tcp_v6_inbound_md5_hash(sk, skb, dif, sdif)) {
1595                         sk_drops_add(sk, skb);
1596                         reqsk_put(req);
1597                         goto discard_it;
1598                 }
1599                 if (tcp_checksum_complete(skb)) {
1600                         reqsk_put(req);
1601                         goto csum_error;
1602                 }
1603                 if (unlikely(sk->sk_state != TCP_LISTEN)) {
1604                         inet_csk_reqsk_queue_drop_and_put(sk, req);
1605                         goto lookup;
1606                 }
1607                 sock_hold(sk);
1608                 refcounted = true;
1609                 nsk = NULL;
1610                 if (!tcp_filter(sk, skb)) {
1611                         th = (const struct tcphdr *)skb->data;
1612                         hdr = ipv6_hdr(skb);
1613                         tcp_v6_fill_cb(skb, hdr, th);
1614                         nsk = tcp_check_req(sk, skb, req, false, &req_stolen);
1615                 }
1616                 if (!nsk) {
1617                         reqsk_put(req);
1618                         if (req_stolen) {
1619                                 /* Another cpu got exclusive access to req
1620                                  * and created a full blown socket.
1621                                  * Try to feed this packet to this socket
1622                                  * instead of discarding it.
1623                                  */
1624                                 tcp_v6_restore_cb(skb);
1625                                 sock_put(sk);
1626                                 goto lookup;
1627                         }
1628                         goto discard_and_relse;
1629                 }
1630                 if (nsk == sk) {
1631                         reqsk_put(req);
1632                         tcp_v6_restore_cb(skb);
1633                 } else if (tcp_child_process(sk, nsk, skb)) {
1634                         tcp_v6_send_reset(nsk, skb);
1635                         goto discard_and_relse;
1636                 } else {
1637                         sock_put(sk);
1638                         return 0;
1639                 }
1640         }
1641         if (hdr->hop_limit < tcp_inet6_sk(sk)->min_hopcount) {
1642                 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
1643                 goto discard_and_relse;
1644         }
1645
1646         if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb))
1647                 goto discard_and_relse;
1648
1649         if (tcp_v6_inbound_md5_hash(sk, skb, dif, sdif))
1650                 goto discard_and_relse;
1651
1652         if (tcp_filter(sk, skb))
1653                 goto discard_and_relse;
1654         th = (const struct tcphdr *)skb->data;
1655         hdr = ipv6_hdr(skb);
1656         tcp_v6_fill_cb(skb, hdr, th);
1657
1658         skb->dev = NULL;
1659
1660         if (sk->sk_state == TCP_LISTEN) {
1661                 ret = tcp_v6_do_rcv(sk, skb);
1662                 goto put_and_return;
1663         }
1664
1665         sk_incoming_cpu_update(sk);
1666
1667         bh_lock_sock_nested(sk);
1668         tcp_segs_in(tcp_sk(sk), skb);
1669         ret = 0;
1670         if (!sock_owned_by_user(sk)) {
1671                 skb_to_free = sk->sk_rx_skb_cache;
1672                 sk->sk_rx_skb_cache = NULL;
1673                 ret = tcp_v6_do_rcv(sk, skb);
1674         } else {
1675                 if (tcp_add_backlog(sk, skb))
1676                         goto discard_and_relse;
1677                 skb_to_free = NULL;
1678         }
1679         bh_unlock_sock(sk);
1680         if (skb_to_free)
1681                 __kfree_skb(skb_to_free);
1682 put_and_return:
1683         if (refcounted)
1684                 sock_put(sk);
1685         return ret ? -1 : 0;
1686
1687 no_tcp_socket:
1688         if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb))
1689                 goto discard_it;
1690
1691         tcp_v6_fill_cb(skb, hdr, th);
1692
1693         if (tcp_checksum_complete(skb)) {
1694 csum_error:
1695                 __TCP_INC_STATS(net, TCP_MIB_CSUMERRORS);
1696 bad_packet:
1697                 __TCP_INC_STATS(net, TCP_MIB_INERRS);
1698         } else {
1699                 tcp_v6_send_reset(NULL, skb);
1700         }
1701
1702 discard_it:
1703         kfree_skb(skb);
1704         return 0;
1705
1706 discard_and_relse:
1707         sk_drops_add(sk, skb);
1708         if (refcounted)
1709                 sock_put(sk);
1710         goto discard_it;
1711
1712 do_time_wait:
1713         if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb)) {
1714                 inet_twsk_put(inet_twsk(sk));
1715                 goto discard_it;
1716         }
1717
1718         tcp_v6_fill_cb(skb, hdr, th);
1719
1720         if (tcp_checksum_complete(skb)) {
1721                 inet_twsk_put(inet_twsk(sk));
1722                 goto csum_error;
1723         }
1724
1725         switch (tcp_timewait_state_process(inet_twsk(sk), skb, th)) {
1726         case TCP_TW_SYN:
1727         {
1728                 struct sock *sk2;
1729
1730                 sk2 = inet6_lookup_listener(dev_net(skb->dev), &tcp_hashinfo,
1731                                             skb, __tcp_hdrlen(th),
1732                                             &ipv6_hdr(skb)->saddr, th->source,
1733                                             &ipv6_hdr(skb)->daddr,
1734                                             ntohs(th->dest),
1735                                             tcp_v6_iif_l3_slave(skb),
1736                                             sdif);
1737                 if (sk2) {
1738                         struct inet_timewait_sock *tw = inet_twsk(sk);
1739                         inet_twsk_deschedule_put(tw);
1740                         sk = sk2;
1741                         tcp_v6_restore_cb(skb);
1742                         refcounted = false;
1743                         goto process;
1744                 }
1745         }
1746                 /* to ACK */
1747                 fallthrough;
1748         case TCP_TW_ACK:
1749                 tcp_v6_timewait_ack(sk, skb);
1750                 break;
1751         case TCP_TW_RST:
1752                 tcp_v6_send_reset(sk, skb);
1753                 inet_twsk_deschedule_put(inet_twsk(sk));
1754                 goto discard_it;
1755         case TCP_TW_SUCCESS:
1756                 ;
1757         }
1758         goto discard_it;
1759 }
1760
1761 INDIRECT_CALLABLE_SCOPE void tcp_v6_early_demux(struct sk_buff *skb)
1762 {
1763         const struct ipv6hdr *hdr;
1764         const struct tcphdr *th;
1765         struct sock *sk;
1766
1767         if (skb->pkt_type != PACKET_HOST)
1768                 return;
1769
1770         if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct tcphdr)))
1771                 return;
1772
1773         hdr = ipv6_hdr(skb);
1774         th = tcp_hdr(skb);
1775
1776         if (th->doff < sizeof(struct tcphdr) / 4)
1777                 return;
1778
1779         /* Note : We use inet6_iif() here, not tcp_v6_iif() */
1780         sk = __inet6_lookup_established(dev_net(skb->dev), &tcp_hashinfo,
1781                                         &hdr->saddr, th->source,
1782                                         &hdr->daddr, ntohs(th->dest),
1783                                         inet6_iif(skb), inet6_sdif(skb));
1784         if (sk) {
1785                 skb->sk = sk;
1786                 skb->destructor = sock_edemux;
1787                 if (sk_fullsock(sk)) {
1788                         struct dst_entry *dst = READ_ONCE(sk->sk_rx_dst);
1789
1790                         if (dst)
1791                                 dst = dst_check(dst, tcp_inet6_sk(sk)->rx_dst_cookie);
1792                         if (dst &&
1793                             inet_sk(sk)->rx_dst_ifindex == skb->skb_iif)
1794                                 skb_dst_set_noref(skb, dst);
1795                 }
1796         }
1797 }
1798
1799 static struct timewait_sock_ops tcp6_timewait_sock_ops = {
1800         .twsk_obj_size  = sizeof(struct tcp6_timewait_sock),
1801         .twsk_unique    = tcp_twsk_unique,
1802         .twsk_destructor = tcp_twsk_destructor,
1803 };
1804
1805 const struct inet_connection_sock_af_ops ipv6_specific = {
1806         .queue_xmit        = inet6_csk_xmit,
1807         .send_check        = tcp_v6_send_check,
1808         .rebuild_header    = inet6_sk_rebuild_header,
1809         .sk_rx_dst_set     = inet6_sk_rx_dst_set,
1810         .conn_request      = tcp_v6_conn_request,
1811         .syn_recv_sock     = tcp_v6_syn_recv_sock,
1812         .net_header_len    = sizeof(struct ipv6hdr),
1813         .net_frag_header_len = sizeof(struct frag_hdr),
1814         .setsockopt        = ipv6_setsockopt,
1815         .getsockopt        = ipv6_getsockopt,
1816         .addr2sockaddr     = inet6_csk_addr2sockaddr,
1817         .sockaddr_len      = sizeof(struct sockaddr_in6),
1818 #ifdef CONFIG_COMPAT
1819         .compat_setsockopt = compat_ipv6_setsockopt,
1820         .compat_getsockopt = compat_ipv6_getsockopt,
1821 #endif
1822         .mtu_reduced       = tcp_v6_mtu_reduced,
1823 };
1824
1825 #ifdef CONFIG_TCP_MD5SIG
1826 static const struct tcp_sock_af_ops tcp_sock_ipv6_specific = {
1827         .md5_lookup     =       tcp_v6_md5_lookup,
1828         .calc_md5_hash  =       tcp_v6_md5_hash_skb,
1829         .md5_parse      =       tcp_v6_parse_md5_keys,
1830 };
1831 #endif
1832
1833 /*
1834  *      TCP over IPv4 via INET6 API
1835  */
1836 static const struct inet_connection_sock_af_ops ipv6_mapped = {
1837         .queue_xmit        = ip_queue_xmit,
1838         .send_check        = tcp_v4_send_check,
1839         .rebuild_header    = inet_sk_rebuild_header,
1840         .sk_rx_dst_set     = inet_sk_rx_dst_set,
1841         .conn_request      = tcp_v6_conn_request,
1842         .syn_recv_sock     = tcp_v6_syn_recv_sock,
1843         .net_header_len    = sizeof(struct iphdr),
1844         .setsockopt        = ipv6_setsockopt,
1845         .getsockopt        = ipv6_getsockopt,
1846         .addr2sockaddr     = inet6_csk_addr2sockaddr,
1847         .sockaddr_len      = sizeof(struct sockaddr_in6),
1848 #ifdef CONFIG_COMPAT
1849         .compat_setsockopt = compat_ipv6_setsockopt,
1850         .compat_getsockopt = compat_ipv6_getsockopt,
1851 #endif
1852         .mtu_reduced       = tcp_v4_mtu_reduced,
1853 };
1854
1855 #ifdef CONFIG_TCP_MD5SIG
1856 static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific = {
1857         .md5_lookup     =       tcp_v4_md5_lookup,
1858         .calc_md5_hash  =       tcp_v4_md5_hash_skb,
1859         .md5_parse      =       tcp_v6_parse_md5_keys,
1860 };
1861 #endif
1862
1863 /* NOTE: A lot of things set to zero explicitly by call to
1864  *       sk_alloc() so need not be done here.
1865  */
1866 static int tcp_v6_init_sock(struct sock *sk)
1867 {
1868         struct inet_connection_sock *icsk = inet_csk(sk);
1869
1870         tcp_init_sock(sk);
1871
1872         icsk->icsk_af_ops = &ipv6_specific;
1873
1874 #ifdef CONFIG_TCP_MD5SIG
1875         tcp_sk(sk)->af_specific = &tcp_sock_ipv6_specific;
1876 #endif
1877
1878         return 0;
1879 }
1880
1881 static void tcp_v6_destroy_sock(struct sock *sk)
1882 {
1883         tcp_v4_destroy_sock(sk);
1884         inet6_destroy_sock(sk);
1885 }
1886
1887 #ifdef CONFIG_PROC_FS
1888 /* Proc filesystem TCPv6 sock list dumping. */
1889 static void get_openreq6(struct seq_file *seq,
1890                          const struct request_sock *req, int i)
1891 {
1892         long ttd = req->rsk_timer.expires - jiffies;
1893         const struct in6_addr *src = &inet_rsk(req)->ir_v6_loc_addr;
1894         const struct in6_addr *dest = &inet_rsk(req)->ir_v6_rmt_addr;
1895
1896         if (ttd < 0)
1897                 ttd = 0;
1898
1899         seq_printf(seq,
1900                    "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
1901                    "%02X %08X:%08X %02X:%08lX %08X %5u %8d %d %d %pK\n",
1902                    i,
1903                    src->s6_addr32[0], src->s6_addr32[1],
1904                    src->s6_addr32[2], src->s6_addr32[3],
1905                    inet_rsk(req)->ir_num,
1906                    dest->s6_addr32[0], dest->s6_addr32[1],
1907                    dest->s6_addr32[2], dest->s6_addr32[3],
1908                    ntohs(inet_rsk(req)->ir_rmt_port),
1909                    TCP_SYN_RECV,
1910                    0, 0, /* could print option size, but that is af dependent. */
1911                    1,   /* timers active (only the expire timer) */
1912                    jiffies_to_clock_t(ttd),
1913                    req->num_timeout,
1914                    from_kuid_munged(seq_user_ns(seq),
1915                                     sock_i_uid(req->rsk_listener)),
1916                    0,  /* non standard timer */
1917                    0, /* open_requests have no inode */
1918                    0, req);
1919 }
1920
1921 static void get_tcp6_sock(struct seq_file *seq, struct sock *sp, int i)
1922 {
1923         const struct in6_addr *dest, *src;
1924         __u16 destp, srcp;
1925         int timer_active;
1926         unsigned long timer_expires;
1927         const struct inet_sock *inet = inet_sk(sp);
1928         const struct tcp_sock *tp = tcp_sk(sp);
1929         const struct inet_connection_sock *icsk = inet_csk(sp);
1930         const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq;
1931         int rx_queue;
1932         int state;
1933
1934         dest  = &sp->sk_v6_daddr;
1935         src   = &sp->sk_v6_rcv_saddr;
1936         destp = ntohs(inet->inet_dport);
1937         srcp  = ntohs(inet->inet_sport);
1938
1939         if (icsk->icsk_pending == ICSK_TIME_RETRANS ||
1940             icsk->icsk_pending == ICSK_TIME_REO_TIMEOUT ||
1941             icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {
1942                 timer_active    = 1;
1943                 timer_expires   = icsk->icsk_timeout;
1944         } else if (icsk->icsk_pending == ICSK_TIME_PROBE0) {
1945                 timer_active    = 4;
1946                 timer_expires   = icsk->icsk_timeout;
1947         } else if (timer_pending(&sp->sk_timer)) {
1948                 timer_active    = 2;
1949                 timer_expires   = sp->sk_timer.expires;
1950         } else {
1951                 timer_active    = 0;
1952                 timer_expires = jiffies;
1953         }
1954
1955         state = inet_sk_state_load(sp);
1956         if (state == TCP_LISTEN)
1957                 rx_queue = READ_ONCE(sp->sk_ack_backlog);
1958         else
1959                 /* Because we don't lock the socket,
1960                  * we might find a transient negative value.
1961                  */
1962                 rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) -
1963                                       READ_ONCE(tp->copied_seq), 0);
1964
1965         seq_printf(seq,
1966                    "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
1967                    "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %lu %lu %u %u %d\n",
1968                    i,
1969                    src->s6_addr32[0], src->s6_addr32[1],
1970                    src->s6_addr32[2], src->s6_addr32[3], srcp,
1971                    dest->s6_addr32[0], dest->s6_addr32[1],
1972                    dest->s6_addr32[2], dest->s6_addr32[3], destp,
1973                    state,
1974                    READ_ONCE(tp->write_seq) - tp->snd_una,
1975                    rx_queue,
1976                    timer_active,
1977                    jiffies_delta_to_clock_t(timer_expires - jiffies),
1978                    icsk->icsk_retransmits,
1979                    from_kuid_munged(seq_user_ns(seq), sock_i_uid(sp)),
1980                    icsk->icsk_probes_out,
1981                    sock_i_ino(sp),
1982                    refcount_read(&sp->sk_refcnt), sp,
1983                    jiffies_to_clock_t(icsk->icsk_rto),
1984                    jiffies_to_clock_t(icsk->icsk_ack.ato),
1985                    (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sp),
1986                    tp->snd_cwnd,
1987                    state == TCP_LISTEN ?
1988                         fastopenq->max_qlen :
1989                         (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh)
1990                    );
1991 }
1992
1993 static void get_timewait6_sock(struct seq_file *seq,
1994                                struct inet_timewait_sock *tw, int i)
1995 {
1996         long delta = tw->tw_timer.expires - jiffies;
1997         const struct in6_addr *dest, *src;
1998         __u16 destp, srcp;
1999
2000         dest = &tw->tw_v6_daddr;
2001         src  = &tw->tw_v6_rcv_saddr;
2002         destp = ntohs(tw->tw_dport);
2003         srcp  = ntohs(tw->tw_sport);
2004
2005         seq_printf(seq,
2006                    "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2007                    "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n",
2008                    i,
2009                    src->s6_addr32[0], src->s6_addr32[1],
2010                    src->s6_addr32[2], src->s6_addr32[3], srcp,
2011                    dest->s6_addr32[0], dest->s6_addr32[1],
2012                    dest->s6_addr32[2], dest->s6_addr32[3], destp,
2013                    tw->tw_substate, 0, 0,
2014                    3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
2015                    refcount_read(&tw->tw_refcnt), tw);
2016 }
2017
2018 static int tcp6_seq_show(struct seq_file *seq, void *v)
2019 {
2020         struct tcp_iter_state *st;
2021         struct sock *sk = v;
2022
2023         if (v == SEQ_START_TOKEN) {
2024                 seq_puts(seq,
2025                          "  sl  "
2026                          "local_address                         "
2027                          "remote_address                        "
2028                          "st tx_queue rx_queue tr tm->when retrnsmt"
2029                          "   uid  timeout inode\n");
2030                 goto out;
2031         }
2032         st = seq->private;
2033
2034         if (sk->sk_state == TCP_TIME_WAIT)
2035                 get_timewait6_sock(seq, v, st->num);
2036         else if (sk->sk_state == TCP_NEW_SYN_RECV)
2037                 get_openreq6(seq, v, st->num);
2038         else
2039                 get_tcp6_sock(seq, v, st->num);
2040 out:
2041         return 0;
2042 }
2043
2044 static const struct seq_operations tcp6_seq_ops = {
2045         .show           = tcp6_seq_show,
2046         .start          = tcp_seq_start,
2047         .next           = tcp_seq_next,
2048         .stop           = tcp_seq_stop,
2049 };
2050
2051 static struct tcp_seq_afinfo tcp6_seq_afinfo = {
2052         .family         = AF_INET6,
2053 };
2054
2055 int __net_init tcp6_proc_init(struct net *net)
2056 {
2057         if (!proc_create_net_data("tcp6", 0444, net->proc_net, &tcp6_seq_ops,
2058                         sizeof(struct tcp_iter_state), &tcp6_seq_afinfo))
2059                 return -ENOMEM;
2060         return 0;
2061 }
2062
2063 void tcp6_proc_exit(struct net *net)
2064 {
2065         remove_proc_entry("tcp6", net->proc_net);
2066 }
2067 #endif
2068
2069 struct proto tcpv6_prot = {
2070         .name                   = "TCPv6",
2071         .owner                  = THIS_MODULE,
2072         .close                  = tcp_close,
2073         .pre_connect            = tcp_v6_pre_connect,
2074         .connect                = tcp_v6_connect,
2075         .disconnect             = tcp_disconnect,
2076         .accept                 = inet_csk_accept,
2077         .ioctl                  = tcp_ioctl,
2078         .init                   = tcp_v6_init_sock,
2079         .destroy                = tcp_v6_destroy_sock,
2080         .shutdown               = tcp_shutdown,
2081         .setsockopt             = tcp_setsockopt,
2082         .getsockopt             = tcp_getsockopt,
2083         .keepalive              = tcp_set_keepalive,
2084         .recvmsg                = tcp_recvmsg,
2085         .sendmsg                = tcp_sendmsg,
2086         .sendpage               = tcp_sendpage,
2087         .backlog_rcv            = tcp_v6_do_rcv,
2088         .release_cb             = tcp_release_cb,
2089         .hash                   = inet6_hash,
2090         .unhash                 = inet_unhash,
2091         .get_port               = inet_csk_get_port,
2092         .enter_memory_pressure  = tcp_enter_memory_pressure,
2093         .leave_memory_pressure  = tcp_leave_memory_pressure,
2094         .stream_memory_free     = tcp_stream_memory_free,
2095         .sockets_allocated      = &tcp_sockets_allocated,
2096         .memory_allocated       = &tcp_memory_allocated,
2097         .memory_pressure        = &tcp_memory_pressure,
2098         .orphan_count           = &tcp_orphan_count,
2099         .sysctl_mem             = sysctl_tcp_mem,
2100         .sysctl_wmem_offset     = offsetof(struct net, ipv4.sysctl_tcp_wmem),
2101         .sysctl_rmem_offset     = offsetof(struct net, ipv4.sysctl_tcp_rmem),
2102         .max_header             = MAX_TCP_HEADER,
2103         .obj_size               = sizeof(struct tcp6_sock),
2104         .slab_flags             = SLAB_TYPESAFE_BY_RCU,
2105         .twsk_prot              = &tcp6_timewait_sock_ops,
2106         .rsk_prot               = &tcp6_request_sock_ops,
2107         .h.hashinfo             = &tcp_hashinfo,
2108         .no_autobind            = true,
2109 #ifdef CONFIG_COMPAT
2110         .compat_setsockopt      = compat_tcp_setsockopt,
2111         .compat_getsockopt      = compat_tcp_getsockopt,
2112 #endif
2113         .diag_destroy           = tcp_abort,
2114 };
2115
2116 /* thinking of making this const? Don't.
2117  * early_demux can change based on sysctl.
2118  */
2119 static struct inet6_protocol tcpv6_protocol = {
2120         .early_demux    =       tcp_v6_early_demux,
2121         .early_demux_handler =  tcp_v6_early_demux,
2122         .handler        =       tcp_v6_rcv,
2123         .err_handler    =       tcp_v6_err,
2124         .flags          =       INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL,
2125 };
2126
2127 static struct inet_protosw tcpv6_protosw = {
2128         .type           =       SOCK_STREAM,
2129         .protocol       =       IPPROTO_TCP,
2130         .prot           =       &tcpv6_prot,
2131         .ops            =       &inet6_stream_ops,
2132         .flags          =       INET_PROTOSW_PERMANENT |
2133                                 INET_PROTOSW_ICSK,
2134 };
2135
2136 static int __net_init tcpv6_net_init(struct net *net)
2137 {
2138         return inet_ctl_sock_create(&net->ipv6.tcp_sk, PF_INET6,
2139                                     SOCK_RAW, IPPROTO_TCP, net);
2140 }
2141
2142 static void __net_exit tcpv6_net_exit(struct net *net)
2143 {
2144         inet_ctl_sock_destroy(net->ipv6.tcp_sk);
2145 }
2146
2147 static void __net_exit tcpv6_net_exit_batch(struct list_head *net_exit_list)
2148 {
2149         inet_twsk_purge(&tcp_hashinfo, AF_INET6);
2150 }
2151
2152 static struct pernet_operations tcpv6_net_ops = {
2153         .init       = tcpv6_net_init,
2154         .exit       = tcpv6_net_exit,
2155         .exit_batch = tcpv6_net_exit_batch,
2156 };
2157
2158 int __init tcpv6_init(void)
2159 {
2160         int ret;
2161
2162         ret = inet6_add_protocol(&tcpv6_protocol, IPPROTO_TCP);
2163         if (ret)
2164                 goto out;
2165
2166         /* register inet6 protocol */
2167         ret = inet6_register_protosw(&tcpv6_protosw);
2168         if (ret)
2169                 goto out_tcpv6_protocol;
2170
2171         ret = register_pernet_subsys(&tcpv6_net_ops);
2172         if (ret)
2173                 goto out_tcpv6_protosw;
2174
2175         ret = mptcpv6_init();
2176         if (ret)
2177                 goto out_tcpv6_pernet_subsys;
2178
2179 out:
2180         return ret;
2181
2182 out_tcpv6_pernet_subsys:
2183         unregister_pernet_subsys(&tcpv6_net_ops);
2184 out_tcpv6_protosw:
2185         inet6_unregister_protosw(&tcpv6_protosw);
2186 out_tcpv6_protocol:
2187         inet6_del_protocol(&tcpv6_protocol, IPPROTO_TCP);
2188         goto out;
2189 }
2190
2191 void tcpv6_exit(void)
2192 {
2193         unregister_pernet_subsys(&tcpv6_net_ops);
2194         inet6_unregister_protosw(&tcpv6_protosw);
2195         inet6_del_protocol(&tcpv6_protocol, IPPROTO_TCP);
2196 }