net/ipv6/tcp_ipv6.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  *      TCP over IPv6
   4  *      Linux INET6 implementation
   5  *
   6  *      Authors:
   7  *      Pedro Roque             <[email protected]>
   8  *
   9  *      Based on:
  10  *      linux/net/ipv4/tcp.c
  11  *      linux/net/ipv4/tcp_input.c
  12  *      linux/net/ipv4/tcp_output.c
  13  *
  14  *      Fixes:
  15  *      Hideaki YOSHIFUJI       :       sin6_scope_id support
  16  *      YOSHIFUJI Hideaki @USAGI and:   Support IPV6_V6ONLY socket option, which
  17  *      Alexey Kuznetsov                allow both IPv4 and IPv6 sockets to bind
  18  *                                      a single port at the same time.
  19  *      YOSHIFUJI Hideaki @USAGI:       convert /proc/net/tcp6 to seq_file.
  20  */
  21
  22 #include <linux/bottom_half.h>
  23 #include <linux/module.h>
  24 #include <linux/errno.h>
  25 #include <linux/types.h>
  26 #include <linux/socket.h>
  27 #include <linux/sockios.h>
  28 #include <linux/net.h>
  29 #include <linux/jiffies.h>
  30 #include <linux/in.h>
  31 #include <linux/in6.h>
  32 #include <linux/netdevice.h>
  33 #include <linux/init.h>
  34 #include <linux/jhash.h>
  35 #include <linux/ipsec.h>
  36 #include <linux/times.h>
  37 #include <linux/slab.h>
  38 #include <linux/uaccess.h>
  39 #include <linux/ipv6.h>
  40 #include <linux/icmpv6.h>
  41 #include <linux/random.h>
  42 #include <linux/indirect_call_wrapper.h>
  43
  44 #include <net/tcp.h>
  45 #include <net/ndisc.h>
  46 #include <net/inet6_hashtables.h>
  47 #include <net/inet6_connection_sock.h>
  48 #include <net/ipv6.h>
  49 #include <net/transp_v6.h>
  50 #include <net/addrconf.h>
  51 #include <net/ip6_route.h>
  52 #include <net/ip6_checksum.h>
  53 #include <net/inet_ecn.h>
  54 #include <net/protocol.h>
  55 #include <net/xfrm.h>
  56 #include <net/snmp.h>
  57 #include <net/dsfield.h>
  58 #include <net/timewait_sock.h>
  59 #include <net/inet_common.h>
  60 #include <net/secure_seq.h>
  61 #include <net/hotdata.h>
  62 #include <net/busy_poll.h>
  63 #include <net/rstreason.h>
  64
  65 #include <linux/proc_fs.h>
  66 #include <linux/seq_file.h>
  67
  68 #include <crypto/hash.h>
  69 #include <linux/scatterlist.h>
  70
  71 #include <trace/events/tcp.h>
  72
  73 static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb,
  74                               enum sk_rst_reason reason);
  75 static void     tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
  76                                       struct request_sock *req);
  77
  78 INDIRECT_CALLABLE_SCOPE int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb);
  79
  80 static const struct inet_connection_sock_af_ops ipv6_mapped;
  81 const struct inet_connection_sock_af_ops ipv6_specific;
  82 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
  83 static const struct tcp_sock_af_ops tcp_sock_ipv6_specific;
  84 static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific;
  85 #endif
  86
  87 /* Helper returning the inet6 address from a given tcp socket.
  88  * It can be used in TCP stack instead of inet6_sk(sk).
  89  * This avoids a dereference and allow compiler optimizations.
  90  * It is a specialized version of inet6_sk_generic().
  91  */
  92 #define tcp_inet6_sk(sk) (&container_of_const(tcp_sk(sk), \
  93                                               struct tcp6_sock, tcp)->inet6)
  94
  95 static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
  96 {
  97         struct dst_entry *dst = skb_dst(skb);
  98
  99         if (dst && dst_hold_safe(dst)) {
 100                 rcu_assign_pointer(sk->sk_rx_dst, dst);
 101                 sk->sk_rx_dst_ifindex = skb->skb_iif;
 102                 sk->sk_rx_dst_cookie = rt6_get_cookie(dst_rt6_info(dst));
 103         }
 104 }
 105
 106 static u32 tcp_v6_init_seq(const struct sk_buff *skb)
 107 {
 108         return secure_tcpv6_seq(ipv6_hdr(skb)->daddr.s6_addr32,
 109                                 ipv6_hdr(skb)->saddr.s6_addr32,
 110                                 tcp_hdr(skb)->dest,
 111                                 tcp_hdr(skb)->source);
 112 }
 113
 114 static u32 tcp_v6_init_ts_off(const struct net *net, const struct sk_buff *skb)
 115 {
 116         return secure_tcpv6_ts_off(net, ipv6_hdr(skb)->daddr.s6_addr32,
 117                                    ipv6_hdr(skb)->saddr.s6_addr32);
 118 }
 119
 120 static int tcp_v6_pre_connect(struct sock *sk, struct sockaddr *uaddr,
 121                               int addr_len)
 122 {
 123         /* This check is replicated from tcp_v6_connect() and intended to
 124          * prevent BPF program called below from accessing bytes that are out
 125          * of the bound specified by user in addr_len.
 126          */
 127         if (addr_len < SIN6_LEN_RFC2133)
 128                 return -EINVAL;
 129
 130         sock_owned_by_me(sk);
 131
 132         return BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr, &addr_len);
 133 }
 134
 135 static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
 136                           int addr_len)
 137 {
 138         struct sockaddr_in6 *usin = (struct sockaddr_in6 *) uaddr;
 139         struct inet_connection_sock *icsk = inet_csk(sk);
 140         struct in6_addr *saddr = NULL, *final_p, final;
 141         struct inet_timewait_death_row *tcp_death_row;
 142         struct ipv6_pinfo *np = tcp_inet6_sk(sk);
 143         struct inet_sock *inet = inet_sk(sk);
 144         struct tcp_sock *tp = tcp_sk(sk);
 145         struct net *net = sock_net(sk);
 146         struct ipv6_txoptions *opt;
 147         struct dst_entry *dst;
 148         struct flowi6 fl6;
 149         int addr_type;
 150         int err;
 151
 152         if (addr_len < SIN6_LEN_RFC2133)
 153                 return -EINVAL;
 154
 155         if (usin->sin6_family != AF_INET6)
 156                 return -EAFNOSUPPORT;
 157
 158         memset(&fl6, 0, sizeof(fl6));
 159
 160         if (inet6_test_bit(SNDFLOW, sk)) {
 161                 fl6.flowlabel = usin->sin6_flowinfo&IPV6_FLOWINFO_MASK;
 162                 IP6_ECN_flow_init(fl6.flowlabel);
 163                 if (fl6.flowlabel&IPV6_FLOWLABEL_MASK) {
 164                         struct ip6_flowlabel *flowlabel;
 165                         flowlabel = fl6_sock_lookup(sk, fl6.flowlabel);
 166                         if (IS_ERR(flowlabel))
 167                                 return -EINVAL;
 168                         fl6_sock_release(flowlabel);
 169                 }
 170         }
 171
 172         /*
 173          *      connect() to INADDR_ANY means loopback (BSD'ism).
 174          */
 175
 176         if (ipv6_addr_any(&usin->sin6_addr)) {
 177                 if (ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr))
 178                         ipv6_addr_set_v4mapped(htonl(INADDR_LOOPBACK),
 179                                                &usin->sin6_addr);
 180                 else
 181                         usin->sin6_addr = in6addr_loopback;
 182         }
 183
 184         addr_type = ipv6_addr_type(&usin->sin6_addr);
 185
 186         if (addr_type & IPV6_ADDR_MULTICAST)
 187                 return -ENETUNREACH;
 188
 189         if (addr_type&IPV6_ADDR_LINKLOCAL) {
 190                 if (addr_len >= sizeof(struct sockaddr_in6) &&
 191                     usin->sin6_scope_id) {
 192                         /* If interface is set while binding, indices
 193                          * must coincide.
 194                          */
 195                         if (!sk_dev_equal_l3scope(sk, usin->sin6_scope_id))
 196                                 return -EINVAL;
 197
 198                         sk->sk_bound_dev_if = usin->sin6_scope_id;
 199                 }
 200
 201                 /* Connect to link-local address requires an interface */
 202                 if (!sk->sk_bound_dev_if)
 203                         return -EINVAL;
 204         }
 205
 206         if (tp->rx_opt.ts_recent_stamp &&
 207             !ipv6_addr_equal(&sk->sk_v6_daddr, &usin->sin6_addr)) {
 208                 tp->rx_opt.ts_recent = 0;
 209                 tp->rx_opt.ts_recent_stamp = 0;
 210                 WRITE_ONCE(tp->write_seq, 0);
 211         }
 212
 213         sk->sk_v6_daddr = usin->sin6_addr;
 214         np->flow_label = fl6.flowlabel;
 215
 216         /*
 217          *      TCP over IPv4
 218          */
 219
 220         if (addr_type & IPV6_ADDR_MAPPED) {
 221                 u32 exthdrlen = icsk->icsk_ext_hdr_len;
 222                 struct sockaddr_in sin;
 223
 224                 if (ipv6_only_sock(sk))
 225                         return -ENETUNREACH;
 226
 227                 sin.sin_family = AF_INET;
 228                 sin.sin_port = usin->sin6_port;
 229                 sin.sin_addr.s_addr = usin->sin6_addr.s6_addr32[3];
 230
 231                 /* Paired with READ_ONCE() in tcp_(get|set)sockopt() */
 232                 WRITE_ONCE(icsk->icsk_af_ops, &ipv6_mapped);
 233                 if (sk_is_mptcp(sk))
 234                         mptcpv6_handle_mapped(sk, true);
 235                 sk->sk_backlog_rcv = tcp_v4_do_rcv;
 236 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
 237                 tp->af_specific = &tcp_sock_ipv6_mapped_specific;
 238 #endif
 239
 240                 err = tcp_v4_connect(sk, (struct sockaddr *)&sin, sizeof(sin));
 241
 242                 if (err) {
 243                         icsk->icsk_ext_hdr_len = exthdrlen;
 244                         /* Paired with READ_ONCE() in tcp_(get|set)sockopt() */
 245                         WRITE_ONCE(icsk->icsk_af_ops, &ipv6_specific);
 246                         if (sk_is_mptcp(sk))
 247                                 mptcpv6_handle_mapped(sk, false);
 248                         sk->sk_backlog_rcv = tcp_v6_do_rcv;
 249 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
 250                         tp->af_specific = &tcp_sock_ipv6_specific;
 251 #endif
 252                         goto failure;
 253                 }
 254                 np->saddr = sk->sk_v6_rcv_saddr;
 255
 256                 return err;
 257         }
 258
 259         if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr))
 260                 saddr = &sk->sk_v6_rcv_saddr;
 261
 262         fl6.flowi6_proto = IPPROTO_TCP;
 263         fl6.daddr = sk->sk_v6_daddr;
 264         fl6.saddr = saddr ? *saddr : np->saddr;
 265         fl6.flowlabel = ip6_make_flowinfo(np->tclass, np->flow_label);
 266         fl6.flowi6_oif = sk->sk_bound_dev_if;
 267         fl6.flowi6_mark = sk->sk_mark;
 268         fl6.fl6_dport = usin->sin6_port;
 269         fl6.fl6_sport = inet->inet_sport;
 270         fl6.flowi6_uid = sk->sk_uid;
 271
 272         opt = rcu_dereference_protected(np->opt, lockdep_sock_is_held(sk));
 273         final_p = fl6_update_dst(&fl6, opt, &final);
 274
 275         security_sk_classify_flow(sk, flowi6_to_flowi_common(&fl6));
 276
 277         dst = ip6_dst_lookup_flow(net, sk, &fl6, final_p);
 278         if (IS_ERR(dst)) {
 279                 err = PTR_ERR(dst);
 280                 goto failure;
 281         }
 282
 283         tp->tcp_usec_ts = dst_tcp_usec_ts(dst);
 284         tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;
 285
 286         if (!saddr) {
 287                 saddr = &fl6.saddr;
 288
 289                 err = inet_bhash2_update_saddr(sk, saddr, AF_INET6);
 290                 if (err)
 291                         goto failure;
 292         }
 293
 294         /* set the source address */
 295         np->saddr = *saddr;
 296         inet->inet_rcv_saddr = LOOPBACK4_IPV6;
 297
 298         sk->sk_gso_type = SKB_GSO_TCPV6;
 299         ip6_dst_store(sk, dst, NULL, NULL);
 300
 301         icsk->icsk_ext_hdr_len = 0;
 302         if (opt)
 303                 icsk->icsk_ext_hdr_len = opt->opt_flen +
 304                                          opt->opt_nflen;
 305
 306         tp->rx_opt.mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) - sizeof(struct ipv6hdr);
 307
 308         inet->inet_dport = usin->sin6_port;
 309
 310         tcp_set_state(sk, TCP_SYN_SENT);
 311         err = inet6_hash_connect(tcp_death_row, sk);
 312         if (err)
 313                 goto late_failure;
 314
 315         sk_set_txhash(sk);
 316
 317         if (likely(!tp->repair)) {
 318                 if (!tp->write_seq)
 319                         WRITE_ONCE(tp->write_seq,
 320                                    secure_tcpv6_seq(np->saddr.s6_addr32,
 321                                                     sk->sk_v6_daddr.s6_addr32,
 322                                                     inet->inet_sport,
 323                                                     inet->inet_dport));
 324                 tp->tsoffset = secure_tcpv6_ts_off(net, np->saddr.s6_addr32,
 325                                                    sk->sk_v6_daddr.s6_addr32);
 326         }
 327
 328         if (tcp_fastopen_defer_connect(sk, &err))
 329                 return err;
 330         if (err)
 331                 goto late_failure;
 332
 333         err = tcp_connect(sk);
 334         if (err)
 335                 goto late_failure;
 336
 337         return 0;
 338
 339 late_failure:
 340         tcp_set_state(sk, TCP_CLOSE);
 341         inet_bhash2_reset_saddr(sk);
 342 failure:
 343         inet->inet_dport = 0;
 344         sk->sk_route_caps = 0;
 345         return err;
 346 }
 347
 348 static void tcp_v6_mtu_reduced(struct sock *sk)
 349 {
 350         struct dst_entry *dst;
 351         u32 mtu;
 352
 353         if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
 354                 return;
 355
 356         mtu = READ_ONCE(tcp_sk(sk)->mtu_info);
 357
 358         /* Drop requests trying to increase our current mss.
 359          * Check done in __ip6_rt_update_pmtu() is too late.
 360          */
 361         if (tcp_mtu_to_mss(sk, mtu) >= tcp_sk(sk)->mss_cache)
 362                 return;
 363
 364         dst = inet6_csk_update_pmtu(sk, mtu);
 365         if (!dst)
 366                 return;
 367
 368         if (inet_csk(sk)->icsk_pmtu_cookie > dst_mtu(dst)) {
 369                 tcp_sync_mss(sk, dst_mtu(dst));
 370                 tcp_simple_retransmit(sk);
 371         }
 372 }
 373
 374 static int tcp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
 375                 u8 type, u8 code, int offset, __be32 info)
 376 {
 377         const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data;
 378         const struct tcphdr *th = (struct tcphdr *)(skb->data+offset);
 379         struct net *net = dev_net(skb->dev);
 380         struct request_sock *fastopen;
 381         struct ipv6_pinfo *np;
 382         struct tcp_sock *tp;
 383         __u32 seq, snd_una;
 384         struct sock *sk;
 385         bool fatal;
 386         int err;
 387
 388         sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
 389                                         &hdr->daddr, th->dest,
 390                                         &hdr->saddr, ntohs(th->source),
 391                                         skb->dev->ifindex, inet6_sdif(skb));
 392
 393         if (!sk) {
 394                 __ICMP6_INC_STATS(net, __in6_dev_get(skb->dev),
 395                                   ICMP6_MIB_INERRORS);
 396                 return -ENOENT;
 397         }
 398
 399         if (sk->sk_state == TCP_TIME_WAIT) {
 400                 /* To increase the counter of ignored icmps for TCP-AO */
 401                 tcp_ao_ignore_icmp(sk, AF_INET6, type, code);
 402                 inet_twsk_put(inet_twsk(sk));
 403                 return 0;
 404         }
 405         seq = ntohl(th->seq);
 406         fatal = icmpv6_err_convert(type, code, &err);
 407         if (sk->sk_state == TCP_NEW_SYN_RECV) {
 408                 tcp_req_err(sk, seq, fatal);
 409                 return 0;
 410         }
 411
 412         if (tcp_ao_ignore_icmp(sk, AF_INET6, type, code)) {
 413                 sock_put(sk);
 414                 return 0;
 415         }
 416
 417         bh_lock_sock(sk);
 418         if (sock_owned_by_user(sk) && type != ICMPV6_PKT_TOOBIG)
 419                 __NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS);
 420
 421         if (sk->sk_state == TCP_CLOSE)
 422                 goto out;
 423
 424         if (static_branch_unlikely(&ip6_min_hopcount)) {
 425                 /* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */
 426                 if (ipv6_hdr(skb)->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount)) {
 427                         __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
 428                         goto out;
 429                 }
 430         }
 431
 432         tp = tcp_sk(sk);
 433         /* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
 434         fastopen = rcu_dereference(tp->fastopen_rsk);
 435         snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;
 436         if (sk->sk_state != TCP_LISTEN &&
 437             !between(seq, snd_una, tp->snd_nxt)) {
 438                 __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
 439                 goto out;
 440         }
 441
 442         np = tcp_inet6_sk(sk);
 443
 444         if (type == NDISC_REDIRECT) {
 445                 if (!sock_owned_by_user(sk)) {
 446                         struct dst_entry *dst = __sk_dst_check(sk, np->dst_cookie);
 447
 448                         if (dst)
 449                                 dst->ops->redirect(dst, sk, skb);
 450                 }
 451                 goto out;
 452         }
 453
 454         if (type == ICMPV6_PKT_TOOBIG) {
 455                 u32 mtu = ntohl(info);
 456
 457                 /* We are not interested in TCP_LISTEN and open_requests
 458                  * (SYN-ACKs send out by Linux are always <576bytes so
 459                  * they should go through unfragmented).
 460                  */
 461                 if (sk->sk_state == TCP_LISTEN)
 462                         goto out;
 463
 464                 if (!ip6_sk_accept_pmtu(sk))
 465                         goto out;
 466
 467                 if (mtu < IPV6_MIN_MTU)
 468                         goto out;
 469
 470                 WRITE_ONCE(tp->mtu_info, mtu);
 471
 472                 if (!sock_owned_by_user(sk))
 473                         tcp_v6_mtu_reduced(sk);
 474                 else if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED,
 475                                            &sk->sk_tsq_flags))
 476                         sock_hold(sk);
 477                 goto out;
 478         }
 479
 480
 481         /* Might be for an request_sock */
 482         switch (sk->sk_state) {
 483         case TCP_SYN_SENT:
 484         case TCP_SYN_RECV:
 485                 /* Only in fast or simultaneous open. If a fast open socket is
 486                  * already accepted it is treated as a connected one below.
 487                  */
 488                 if (fastopen && !fastopen->sk)
 489                         break;
 490
 491                 ipv6_icmp_error(sk, skb, err, th->dest, ntohl(info), (u8 *)th);
 492
 493                 if (!sock_owned_by_user(sk)) {
 494                         WRITE_ONCE(sk->sk_err, err);
 495                         sk_error_report(sk);            /* Wake people up to see the error (see connect in sock.c) */
 496
 497                         tcp_done(sk);
 498                 } else {
 499                         WRITE_ONCE(sk->sk_err_soft, err);
 500                 }
 501                 goto out;
 502         case TCP_LISTEN:
 503                 break;
 504         default:
 505                 /* check if this ICMP message allows revert of backoff.
 506                  * (see RFC 6069)
 507                  */
 508                 if (!fastopen && type == ICMPV6_DEST_UNREACH &&
 509                     code == ICMPV6_NOROUTE)
 510                         tcp_ld_RTO_revert(sk, seq);
 511         }
 512
 513         if (!sock_owned_by_user(sk) && inet6_test_bit(RECVERR6, sk)) {
 514                 WRITE_ONCE(sk->sk_err, err);
 515                 sk_error_report(sk);
 516         } else {
 517                 WRITE_ONCE(sk->sk_err_soft, err);
 518         }
 519 out:
 520         bh_unlock_sock(sk);
 521         sock_put(sk);
 522         return 0;
 523 }
 524
 525
 526 static int tcp_v6_send_synack(const struct sock *sk, struct dst_entry *dst,
 527                               struct flowi *fl,
 528                               struct request_sock *req,
 529                               struct tcp_fastopen_cookie *foc,
 530                               enum tcp_synack_type synack_type,
 531                               struct sk_buff *syn_skb)
 532 {
 533         struct inet_request_sock *ireq = inet_rsk(req);
 534         const struct ipv6_pinfo *np = tcp_inet6_sk(sk);
 535         struct ipv6_txoptions *opt;
 536         struct flowi6 *fl6 = &fl->u.ip6;
 537         struct sk_buff *skb;
 538         int err = -ENOMEM;
 539         u8 tclass;
 540
 541         /* First, grab a route. */
 542         if (!dst && (dst = inet6_csk_route_req(sk, fl6, req,
 543                                                IPPROTO_TCP)) == NULL)
 544                 goto done;
 545
 546         skb = tcp_make_synack(sk, dst, req, foc, synack_type, syn_skb);
 547
 548         if (skb) {
 549                 __tcp_v6_send_check(skb, &ireq->ir_v6_loc_addr,
 550                                     &ireq->ir_v6_rmt_addr);
 551
 552                 fl6->daddr = ireq->ir_v6_rmt_addr;
 553                 if (inet6_test_bit(REPFLOW, sk) && ireq->pktopts)
 554                         fl6->flowlabel = ip6_flowlabel(ipv6_hdr(ireq->pktopts));
 555
 556                 tclass = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos) ?
 557                                 (tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) |
 558                                 (np->tclass & INET_ECN_MASK) :
 559                                 np->tclass;
 560
 561                 if (!INET_ECN_is_capable(tclass) &&
 562                     tcp_bpf_ca_needs_ecn((struct sock *)req))
 563                         tclass |= INET_ECN_ECT_0;
 564
 565                 rcu_read_lock();
 566                 opt = ireq->ipv6_opt;
 567                 if (!opt)
 568                         opt = rcu_dereference(np->opt);
 569                 err = ip6_xmit(sk, skb, fl6, skb->mark ? : READ_ONCE(sk->sk_mark),
 570                                opt, tclass, READ_ONCE(sk->sk_priority));
 571                 rcu_read_unlock();
 572                 err = net_xmit_eval(err);
 573         }
 574
 575 done:
 576         return err;
 577 }
 578
 579
 580 static void tcp_v6_reqsk_destructor(struct request_sock *req)
 581 {
 582         kfree(inet_rsk(req)->ipv6_opt);
 583         consume_skb(inet_rsk(req)->pktopts);
 584 }
 585
 586 #ifdef CONFIG_TCP_MD5SIG
 587 static struct tcp_md5sig_key *tcp_v6_md5_do_lookup(const struct sock *sk,
 588                                                    const struct in6_addr *addr,
 589                                                    int l3index)
 590 {
 591         return tcp_md5_do_lookup(sk, l3index,
 592                                  (union tcp_md5_addr *)addr, AF_INET6);
 593 }
 594
 595 static struct tcp_md5sig_key *tcp_v6_md5_lookup(const struct sock *sk,
 596                                                 const struct sock *addr_sk)
 597 {
 598         int l3index;
 599
 600         l3index = l3mdev_master_ifindex_by_index(sock_net(sk),
 601                                                  addr_sk->sk_bound_dev_if);
 602         return tcp_v6_md5_do_lookup(sk, &addr_sk->sk_v6_daddr,
 603                                     l3index);
 604 }
 605
 606 static int tcp_v6_parse_md5_keys(struct sock *sk, int optname,
 607                                  sockptr_t optval, int optlen)
 608 {
 609         struct tcp_md5sig cmd;
 610         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&cmd.tcpm_addr;
 611         union tcp_ao_addr *addr;
 612         int l3index = 0;
 613         u8 prefixlen;
 614         bool l3flag;
 615         u8 flags;
 616
 617         if (optlen < sizeof(cmd))
 618                 return -EINVAL;
 619
 620         if (copy_from_sockptr(&cmd, optval, sizeof(cmd)))
 621                 return -EFAULT;
 622
 623         if (sin6->sin6_family != AF_INET6)
 624                 return -EINVAL;
 625
 626         flags = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
 627         l3flag = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
 628
 629         if (optname == TCP_MD5SIG_EXT &&
 630             cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) {
 631                 prefixlen = cmd.tcpm_prefixlen;
 632                 if (prefixlen > 128 || (ipv6_addr_v4mapped(&sin6->sin6_addr) &&
 633                                         prefixlen > 32))
 634                         return -EINVAL;
 635         } else {
 636                 prefixlen = ipv6_addr_v4mapped(&sin6->sin6_addr) ? 32 : 128;
 637         }
 638
 639         if (optname == TCP_MD5SIG_EXT && cmd.tcpm_ifindex &&
 640             cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) {
 641                 struct net_device *dev;
 642
 643                 rcu_read_lock();
 644                 dev = dev_get_by_index_rcu(sock_net(sk), cmd.tcpm_ifindex);
 645                 if (dev && netif_is_l3_master(dev))
 646                         l3index = dev->ifindex;
 647                 rcu_read_unlock();
 648
 649                 /* ok to reference set/not set outside of rcu;
 650                  * right now device MUST be an L3 master
 651                  */
 652                 if (!dev || !l3index)
 653                         return -EINVAL;
 654         }
 655
 656         if (!cmd.tcpm_keylen) {
 657                 if (ipv6_addr_v4mapped(&sin6->sin6_addr))
 658                         return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3],
 659                                               AF_INET, prefixlen,
 660                                               l3index, flags);
 661                 return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr,
 662                                       AF_INET6, prefixlen, l3index, flags);
 663         }
 664
 665         if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
 666                 return -EINVAL;
 667
 668         if (ipv6_addr_v4mapped(&sin6->sin6_addr)) {
 669                 addr = (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3];
 670
 671                 /* Don't allow keys for peers that have a matching TCP-AO key.
 672                  * See the comment in tcp_ao_add_cmd()
 673                  */
 674                 if (tcp_ao_required(sk, addr, AF_INET,
 675                                     l3flag ? l3index : -1, false))
 676                         return -EKEYREJECTED;
 677                 return tcp_md5_do_add(sk, addr,
 678                                       AF_INET, prefixlen, l3index, flags,
 679                                       cmd.tcpm_key, cmd.tcpm_keylen);
 680         }
 681
 682         addr = (union tcp_md5_addr *)&sin6->sin6_addr;
 683
 684         /* Don't allow keys for peers that have a matching TCP-AO key.
 685          * See the comment in tcp_ao_add_cmd()
 686          */
 687         if (tcp_ao_required(sk, addr, AF_INET6, l3flag ? l3index : -1, false))
 688                 return -EKEYREJECTED;
 689
 690         return tcp_md5_do_add(sk, addr, AF_INET6, prefixlen, l3index, flags,
 691                               cmd.tcpm_key, cmd.tcpm_keylen);
 692 }
 693
 694 static int tcp_v6_md5_hash_headers(struct tcp_sigpool *hp,
 695                                    const struct in6_addr *daddr,
 696                                    const struct in6_addr *saddr,
 697                                    const struct tcphdr *th, int nbytes)
 698 {
 699         struct tcp6_pseudohdr *bp;
 700         struct scatterlist sg;
 701         struct tcphdr *_th;
 702
 703         bp = hp->scratch;
 704         /* 1. TCP pseudo-header (RFC2460) */
 705         bp->saddr = *saddr;
 706         bp->daddr = *daddr;
 707         bp->protocol = cpu_to_be32(IPPROTO_TCP);
 708         bp->len = cpu_to_be32(nbytes);
 709
 710         _th = (struct tcphdr *)(bp + 1);
 711         memcpy(_th, th, sizeof(*th));
 712         _th->check = 0;
 713
 714         sg_init_one(&sg, bp, sizeof(*bp) + sizeof(*th));
 715         ahash_request_set_crypt(hp->req, &sg, NULL,
 716                                 sizeof(*bp) + sizeof(*th));
 717         return crypto_ahash_update(hp->req);
 718 }
 719
 720 static int tcp_v6_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
 721                                const struct in6_addr *daddr, struct in6_addr *saddr,
 722                                const struct tcphdr *th)
 723 {
 724         struct tcp_sigpool hp;
 725
 726         if (tcp_sigpool_start(tcp_md5_sigpool_id, &hp))
 727                 goto clear_hash_nostart;
 728
 729         if (crypto_ahash_init(hp.req))
 730                 goto clear_hash;
 731         if (tcp_v6_md5_hash_headers(&hp, daddr, saddr, th, th->doff << 2))
 732                 goto clear_hash;
 733         if (tcp_md5_hash_key(&hp, key))
 734                 goto clear_hash;
 735         ahash_request_set_crypt(hp.req, NULL, md5_hash, 0);
 736         if (crypto_ahash_final(hp.req))
 737                 goto clear_hash;
 738
 739         tcp_sigpool_end(&hp);
 740         return 0;
 741
 742 clear_hash:
 743         tcp_sigpool_end(&hp);
 744 clear_hash_nostart:
 745         memset(md5_hash, 0, 16);
 746         return 1;
 747 }
 748
 749 static int tcp_v6_md5_hash_skb(char *md5_hash,
 750                                const struct tcp_md5sig_key *key,
 751                                const struct sock *sk,
 752                                const struct sk_buff *skb)
 753 {
 754         const struct tcphdr *th = tcp_hdr(skb);
 755         const struct in6_addr *saddr, *daddr;
 756         struct tcp_sigpool hp;
 757
 758         if (sk) { /* valid for establish/request sockets */
 759                 saddr = &sk->sk_v6_rcv_saddr;
 760                 daddr = &sk->sk_v6_daddr;
 761         } else {
 762                 const struct ipv6hdr *ip6h = ipv6_hdr(skb);
 763                 saddr = &ip6h->saddr;
 764                 daddr = &ip6h->daddr;
 765         }
 766
 767         if (tcp_sigpool_start(tcp_md5_sigpool_id, &hp))
 768                 goto clear_hash_nostart;
 769
 770         if (crypto_ahash_init(hp.req))
 771                 goto clear_hash;
 772
 773         if (tcp_v6_md5_hash_headers(&hp, daddr, saddr, th, skb->len))
 774                 goto clear_hash;
 775         if (tcp_sigpool_hash_skb_data(&hp, skb, th->doff << 2))
 776                 goto clear_hash;
 777         if (tcp_md5_hash_key(&hp, key))
 778                 goto clear_hash;
 779         ahash_request_set_crypt(hp.req, NULL, md5_hash, 0);
 780         if (crypto_ahash_final(hp.req))
 781                 goto clear_hash;
 782
 783         tcp_sigpool_end(&hp);
 784         return 0;
 785
 786 clear_hash:
 787         tcp_sigpool_end(&hp);
 788 clear_hash_nostart:
 789         memset(md5_hash, 0, 16);
 790         return 1;
 791 }
 792 #endif
 793
 794 static void tcp_v6_init_req(struct request_sock *req,
 795                             const struct sock *sk_listener,
 796                             struct sk_buff *skb,
 797                             u32 tw_isn)
 798 {
 799         bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags);
 800         struct inet_request_sock *ireq = inet_rsk(req);
 801         const struct ipv6_pinfo *np = tcp_inet6_sk(sk_listener);
 802
 803         ireq->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr;
 804         ireq->ir_v6_loc_addr = ipv6_hdr(skb)->daddr;
 805
 806         /* So that link locals have meaning */
 807         if ((!sk_listener->sk_bound_dev_if || l3_slave) &&
 808             ipv6_addr_type(&ireq->ir_v6_rmt_addr) & IPV6_ADDR_LINKLOCAL)
 809                 ireq->ir_iif = tcp_v6_iif(skb);
 810
 811         if (!tw_isn &&
 812             (ipv6_opt_accepted(sk_listener, skb, &TCP_SKB_CB(skb)->header.h6) ||
 813              np->rxopt.bits.rxinfo ||
 814              np->rxopt.bits.rxoinfo || np->rxopt.bits.rxhlim ||
 815              np->rxopt.bits.rxohlim || inet6_test_bit(REPFLOW, sk_listener))) {
 816                 refcount_inc(&skb->users);
 817                 ireq->pktopts = skb;
 818         }
 819 }
 820
 821 static struct dst_entry *tcp_v6_route_req(const struct sock *sk,
 822                                           struct sk_buff *skb,
 823                                           struct flowi *fl,
 824                                           struct request_sock *req,
 825                                           u32 tw_isn)
 826 {
 827         tcp_v6_init_req(req, sk, skb, tw_isn);
 828
 829         if (security_inet_conn_request(sk, skb, req))
 830                 return NULL;
 831
 832         return inet6_csk_route_req(sk, &fl->u.ip6, req, IPPROTO_TCP);
 833 }
 834
 835 struct request_sock_ops tcp6_request_sock_ops __read_mostly = {
 836         .family         =       AF_INET6,
 837         .obj_size       =       sizeof(struct tcp6_request_sock),
 838         .rtx_syn_ack    =       tcp_rtx_synack,
 839         .send_ack       =       tcp_v6_reqsk_send_ack,
 840         .destructor     =       tcp_v6_reqsk_destructor,
 841         .send_reset     =       tcp_v6_send_reset,
 842         .syn_ack_timeout =      tcp_syn_ack_timeout,
 843 };
 844
 845 const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops = {
 846         .mss_clamp      =       IPV6_MIN_MTU - sizeof(struct tcphdr) -
 847                                 sizeof(struct ipv6hdr),
 848 #ifdef CONFIG_TCP_MD5SIG
 849         .req_md5_lookup =       tcp_v6_md5_lookup,
 850         .calc_md5_hash  =       tcp_v6_md5_hash_skb,
 851 #endif
 852 #ifdef CONFIG_TCP_AO
 853         .ao_lookup      =       tcp_v6_ao_lookup_rsk,
 854         .ao_calc_key    =       tcp_v6_ao_calc_key_rsk,
 855         .ao_synack_hash =       tcp_v6_ao_synack_hash,
 856 #endif
 857 #ifdef CONFIG_SYN_COOKIES
 858         .cookie_init_seq =      cookie_v6_init_sequence,
 859 #endif
 860         .route_req      =       tcp_v6_route_req,
 861         .init_seq       =       tcp_v6_init_seq,
 862         .init_ts_off    =       tcp_v6_init_ts_off,
 863         .send_synack    =       tcp_v6_send_synack,
 864 };
 865
 866 static void tcp_v6_send_response(const struct sock *sk, struct sk_buff *skb, u32 seq,
 867                                  u32 ack, u32 win, u32 tsval, u32 tsecr,
 868                                  int oif, int rst, u8 tclass, __be32 label,
 869                                  u32 priority, u32 txhash, struct tcp_key *key)
 870 {
 871         const struct tcphdr *th = tcp_hdr(skb);
 872         struct tcphdr *t1;
 873         struct sk_buff *buff;
 874         struct flowi6 fl6;
 875         struct net *net = sk ? sock_net(sk) : dev_net(skb_dst(skb)->dev);
 876         struct sock *ctl_sk = net->ipv6.tcp_sk;
 877         unsigned int tot_len = sizeof(struct tcphdr);
 878         __be32 mrst = 0, *topt;
 879         struct dst_entry *dst;
 880         __u32 mark = 0;
 881
 882         if (tsecr)
 883                 tot_len += TCPOLEN_TSTAMP_ALIGNED;
 884         if (tcp_key_is_md5(key))
 885                 tot_len += TCPOLEN_MD5SIG_ALIGNED;
 886         if (tcp_key_is_ao(key))
 887                 tot_len += tcp_ao_len_aligned(key->ao_key);
 888
 889 #ifdef CONFIG_MPTCP
 890         if (rst && !tcp_key_is_md5(key)) {
 891                 mrst = mptcp_reset_option(skb);
 892
 893                 if (mrst)
 894                         tot_len += sizeof(__be32);
 895         }
 896 #endif
 897
 898         buff = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC);
 899         if (!buff)
 900                 return;
 901
 902         skb_reserve(buff, MAX_TCP_HEADER);
 903
 904         t1 = skb_push(buff, tot_len);
 905         skb_reset_transport_header(buff);
 906
 907         /* Swap the send and the receive. */
 908         memset(t1, 0, sizeof(*t1));
 909         t1->dest = th->source;
 910         t1->source = th->dest;
 911         t1->doff = tot_len / 4;
 912         t1->seq = htonl(seq);
 913         t1->ack_seq = htonl(ack);
 914         t1->ack = !rst || !th->ack;
 915         t1->rst = rst;
 916         t1->window = htons(win);
 917
 918         topt = (__be32 *)(t1 + 1);
 919
 920         if (tsecr) {
 921                 *topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
 922                                 (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP);
 923                 *topt++ = htonl(tsval);
 924                 *topt++ = htonl(tsecr);
 925         }
 926
 927         if (mrst)
 928                 *topt++ = mrst;
 929
 930 #ifdef CONFIG_TCP_MD5SIG
 931         if (tcp_key_is_md5(key)) {
 932                 *topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
 933                                 (TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG);
 934                 tcp_v6_md5_hash_hdr((__u8 *)topt, key->md5_key,
 935                                     &ipv6_hdr(skb)->saddr,
 936                                     &ipv6_hdr(skb)->daddr, t1);
 937         }
 938 #endif
 939 #ifdef CONFIG_TCP_AO
 940         if (tcp_key_is_ao(key)) {
 941                 *topt++ = htonl((TCPOPT_AO << 24) |
 942                                 (tcp_ao_len(key->ao_key) << 16) |
 943                                 (key->ao_key->sndid << 8) |
 944                                 (key->rcv_next));
 945
 946                 tcp_ao_hash_hdr(AF_INET6, (char *)topt, key->ao_key,
 947                                 key->traffic_key,
 948                                 (union tcp_ao_addr *)&ipv6_hdr(skb)->saddr,
 949                                 (union tcp_ao_addr *)&ipv6_hdr(skb)->daddr,
 950                                 t1, key->sne);
 951         }
 952 #endif
 953
 954         memset(&fl6, 0, sizeof(fl6));
 955         fl6.daddr = ipv6_hdr(skb)->saddr;
 956         fl6.saddr = ipv6_hdr(skb)->daddr;
 957         fl6.flowlabel = label;
 958
 959         buff->ip_summed = CHECKSUM_PARTIAL;
 960
 961         __tcp_v6_send_check(buff, &fl6.saddr, &fl6.daddr);
 962
 963         fl6.flowi6_proto = IPPROTO_TCP;
 964         if (rt6_need_strict(&fl6.daddr) && !oif)
 965                 fl6.flowi6_oif = tcp_v6_iif(skb);
 966         else {
 967                 if (!oif && netif_index_is_l3_master(net, skb->skb_iif))
 968                         oif = skb->skb_iif;
 969
 970                 fl6.flowi6_oif = oif;
 971         }
 972
 973         if (sk) {
 974                 if (sk->sk_state == TCP_TIME_WAIT)
 975                         mark = inet_twsk(sk)->tw_mark;
 976                 else
 977                         mark = READ_ONCE(sk->sk_mark);
 978                 skb_set_delivery_time(buff, tcp_transmit_time(sk), true);
 979         }
 980         if (txhash) {
 981                 /* autoflowlabel/skb_get_hash_flowi6 rely on buff->hash */
 982                 skb_set_hash(buff, txhash, PKT_HASH_TYPE_L4);
 983         }
 984         fl6.flowi6_mark = IP6_REPLY_MARK(net, skb->mark) ?: mark;
 985         fl6.fl6_dport = t1->dest;
 986         fl6.fl6_sport = t1->source;
 987         fl6.flowi6_uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL);
 988         security_skb_classify_flow(skb, flowi6_to_flowi_common(&fl6));
 989
 990         /* Pass a socket to ip6_dst_lookup either it is for RST
 991          * Underlying function will use this to retrieve the network
 992          * namespace
 993          */
 994         if (sk && sk->sk_state != TCP_TIME_WAIT)
 995                 dst = ip6_dst_lookup_flow(net, sk, &fl6, NULL); /*sk's xfrm_policy can be referred*/
 996         else
 997                 dst = ip6_dst_lookup_flow(net, ctl_sk, &fl6, NULL);
 998         if (!IS_ERR(dst)) {
 999                 skb_dst_set(buff, dst);
1000                 ip6_xmit(ctl_sk, buff, &fl6, fl6.flowi6_mark, NULL,
1001                          tclass & ~INET_ECN_MASK, priority);
1002                 TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
1003                 if (rst)
1004                         TCP_INC_STATS(net, TCP_MIB_OUTRSTS);
1005                 return;
1006         }
1007
1008         kfree_skb(buff);
1009 }
1010
1011 static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb,
1012                               enum sk_rst_reason reason)
1013 {
1014         const struct tcphdr *th = tcp_hdr(skb);
1015         struct ipv6hdr *ipv6h = ipv6_hdr(skb);
1016         const __u8 *md5_hash_location = NULL;
1017 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1018         bool allocated_traffic_key = false;
1019 #endif
1020         const struct tcp_ao_hdr *aoh;
1021         struct tcp_key key = {};
1022         u32 seq = 0, ack_seq = 0;
1023         __be32 label = 0;
1024         u32 priority = 0;
1025         struct net *net;
1026         u32 txhash = 0;
1027         int oif = 0;
1028 #ifdef CONFIG_TCP_MD5SIG
1029         unsigned char newhash[16];
1030         int genhash;
1031         struct sock *sk1 = NULL;
1032 #endif
1033
1034         if (th->rst)
1035                 return;
1036
1037         /* If sk not NULL, it means we did a successful lookup and incoming
1038          * route had to be correct. prequeue might have dropped our dst.
1039          */
1040         if (!sk && !ipv6_unicast_destination(skb))
1041                 return;
1042
1043         net = sk ? sock_net(sk) : dev_net(skb_dst(skb)->dev);
1044         /* Invalid TCP option size or twice included auth */
1045         if (tcp_parse_auth_options(th, &md5_hash_location, &aoh))
1046                 return;
1047 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1048         rcu_read_lock();
1049 #endif
1050 #ifdef CONFIG_TCP_MD5SIG
1051         if (sk && sk_fullsock(sk)) {
1052                 int l3index;
1053
1054                 /* sdif set, means packet ingressed via a device
1055                  * in an L3 domain and inet_iif is set to it.
1056                  */
1057                 l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1058                 key.md5_key = tcp_v6_md5_do_lookup(sk, &ipv6h->saddr, l3index);
1059                 if (key.md5_key)
1060                         key.type = TCP_KEY_MD5;
1061         } else if (md5_hash_location) {
1062                 int dif = tcp_v6_iif_l3_slave(skb);
1063                 int sdif = tcp_v6_sdif(skb);
1064                 int l3index;
1065
1066                 /*
1067                  * active side is lost. Try to find listening socket through
1068                  * source port, and then find md5 key through listening socket.
1069                  * we are not loose security here:
1070                  * Incoming packet is checked with md5 hash with finding key,
1071                  * no RST generated if md5 hash doesn't match.
1072                  */
1073                 sk1 = inet6_lookup_listener(net, net->ipv4.tcp_death_row.hashinfo,
1074                                             NULL, 0, &ipv6h->saddr, th->source,
1075                                             &ipv6h->daddr, ntohs(th->source),
1076                                             dif, sdif);
1077                 if (!sk1)
1078                         goto out;
1079
1080                 /* sdif set, means packet ingressed via a device
1081                  * in an L3 domain and dif is set to it.
1082                  */
1083                 l3index = tcp_v6_sdif(skb) ? dif : 0;
1084
1085                 key.md5_key = tcp_v6_md5_do_lookup(sk1, &ipv6h->saddr, l3index);
1086                 if (!key.md5_key)
1087                         goto out;
1088                 key.type = TCP_KEY_MD5;
1089
1090                 genhash = tcp_v6_md5_hash_skb(newhash, key.md5_key, NULL, skb);
1091                 if (genhash || memcmp(md5_hash_location, newhash, 16) != 0)
1092                         goto out;
1093         }
1094 #endif
1095
1096         if (th->ack)
1097                 seq = ntohl(th->ack_seq);
1098         else
1099                 ack_seq = ntohl(th->seq) + th->syn + th->fin + skb->len -
1100                           (th->doff << 2);
1101
1102 #ifdef CONFIG_TCP_AO
1103         if (aoh) {
1104                 int l3index;
1105
1106                 l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1107                 if (tcp_ao_prepare_reset(sk, skb, aoh, l3index, seq,
1108                                          &key.ao_key, &key.traffic_key,
1109                                          &allocated_traffic_key,
1110                                          &key.rcv_next, &key.sne))
1111                         goto out;
1112                 key.type = TCP_KEY_AO;
1113         }
1114 #endif
1115
1116         if (sk) {
1117                 oif = sk->sk_bound_dev_if;
1118                 if (sk_fullsock(sk)) {
1119                         if (inet6_test_bit(REPFLOW, sk))
1120                                 label = ip6_flowlabel(ipv6h);
1121                         priority = READ_ONCE(sk->sk_priority);
1122                         txhash = sk->sk_txhash;
1123                 }
1124                 if (sk->sk_state == TCP_TIME_WAIT) {
1125                         label = cpu_to_be32(inet_twsk(sk)->tw_flowlabel);
1126                         priority = inet_twsk(sk)->tw_priority;
1127                         txhash = inet_twsk(sk)->tw_txhash;
1128                 }
1129         } else {
1130                 if (net->ipv6.sysctl.flowlabel_reflect & FLOWLABEL_REFLECT_TCP_RESET)
1131                         label = ip6_flowlabel(ipv6h);
1132         }
1133
1134         trace_tcp_send_reset(sk, skb, reason);
1135
1136         tcp_v6_send_response(sk, skb, seq, ack_seq, 0, 0, 0, oif, 1,
1137                              ipv6_get_dsfield(ipv6h), label, priority, txhash,
1138                              &key);
1139
1140 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1141 out:
1142         if (allocated_traffic_key)
1143                 kfree(key.traffic_key);
1144         rcu_read_unlock();
1145 #endif
1146 }
1147
1148 static void tcp_v6_send_ack(const struct sock *sk, struct sk_buff *skb, u32 seq,
1149                             u32 ack, u32 win, u32 tsval, u32 tsecr, int oif,
1150                             struct tcp_key *key, u8 tclass,
1151                             __be32 label, u32 priority, u32 txhash)
1152 {
1153         tcp_v6_send_response(sk, skb, seq, ack, win, tsval, tsecr, oif, 0,
1154                              tclass, label, priority, txhash, key);
1155 }
1156
1157 static void tcp_v6_timewait_ack(struct sock *sk, struct sk_buff *skb)
1158 {
1159         struct inet_timewait_sock *tw = inet_twsk(sk);
1160         struct tcp_timewait_sock *tcptw = tcp_twsk(sk);
1161         struct tcp_key key = {};
1162 #ifdef CONFIG_TCP_AO
1163         struct tcp_ao_info *ao_info;
1164
1165         if (static_branch_unlikely(&tcp_ao_needed.key)) {
1166
1167                 /* FIXME: the segment to-be-acked is not verified yet */
1168                 ao_info = rcu_dereference(tcptw->ao_info);
1169                 if (ao_info) {
1170                         const struct tcp_ao_hdr *aoh;
1171
1172                         /* Invalid TCP option size or twice included auth */
1173                         if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh))
1174                                 goto out;
1175                         if (aoh)
1176                                 key.ao_key = tcp_ao_established_key(ao_info,
1177                                                 aoh->rnext_keyid, -1);
1178                 }
1179         }
1180         if (key.ao_key) {
1181                 struct tcp_ao_key *rnext_key;
1182
1183                 key.traffic_key = snd_other_key(key.ao_key);
1184                 /* rcv_next switches to our rcv_next */
1185                 rnext_key = READ_ONCE(ao_info->rnext_key);
1186                 key.rcv_next = rnext_key->rcvid;
1187                 key.sne = READ_ONCE(ao_info->snd_sne);
1188                 key.type = TCP_KEY_AO;
1189 #else
1190         if (0) {
1191 #endif
1192 #ifdef CONFIG_TCP_MD5SIG
1193         } else if (static_branch_unlikely(&tcp_md5_needed.key)) {
1194                 key.md5_key = tcp_twsk_md5_key(tcptw);
1195                 if (key.md5_key)
1196                         key.type = TCP_KEY_MD5;
1197 #endif
1198         }
1199
1200         tcp_v6_send_ack(sk, skb, tcptw->tw_snd_nxt, tcptw->tw_rcv_nxt,
1201                         tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale,
1202                         tcp_tw_tsval(tcptw),
1203                         tcptw->tw_ts_recent, tw->tw_bound_dev_if, &key,
1204                         tw->tw_tclass, cpu_to_be32(tw->tw_flowlabel), tw->tw_priority,
1205                         tw->tw_txhash);
1206
1207 #ifdef CONFIG_TCP_AO
1208 out:
1209 #endif
1210         inet_twsk_put(tw);
1211 }
1212
1213 static void tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
1214                                   struct request_sock *req)
1215 {
1216         struct tcp_key key = {};
1217
1218 #ifdef CONFIG_TCP_AO
1219         if (static_branch_unlikely(&tcp_ao_needed.key) &&
1220             tcp_rsk_used_ao(req)) {
1221                 const struct in6_addr *addr = &ipv6_hdr(skb)->saddr;
1222                 const struct tcp_ao_hdr *aoh;
1223                 int l3index;
1224
1225                 l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1226                 /* Invalid TCP option size or twice included auth */
1227                 if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh))
1228                         return;
1229                 if (!aoh)
1230                         return;
1231                 key.ao_key = tcp_ao_do_lookup(sk, l3index,
1232                                               (union tcp_ao_addr *)addr,
1233                                               AF_INET6, aoh->rnext_keyid, -1);
1234                 if (unlikely(!key.ao_key)) {
1235                         /* Send ACK with any matching MKT for the peer */
1236                         key.ao_key = tcp_ao_do_lookup(sk, l3index,
1237                                                       (union tcp_ao_addr *)addr,
1238                                                       AF_INET6, -1, -1);
1239                         /* Matching key disappeared (user removed the key?)
1240                          * let the handshake timeout.
1241                          */
1242                         if (!key.ao_key) {
1243                                 net_info_ratelimited("TCP-AO key for (%pI6, %d)->(%pI6, %d) suddenly disappeared, won't ACK new connection\n",
1244                                                      addr,
1245                                                      ntohs(tcp_hdr(skb)->source),
1246                                                      &ipv6_hdr(skb)->daddr,
1247                                                      ntohs(tcp_hdr(skb)->dest));
1248                                 return;
1249                         }
1250                 }
1251                 key.traffic_key = kmalloc(tcp_ao_digest_size(key.ao_key), GFP_ATOMIC);
1252                 if (!key.traffic_key)
1253                         return;
1254
1255                 key.type = TCP_KEY_AO;
1256                 key.rcv_next = aoh->keyid;
1257                 tcp_v6_ao_calc_key_rsk(key.ao_key, key.traffic_key, req);
1258 #else
1259         if (0) {
1260 #endif
1261 #ifdef CONFIG_TCP_MD5SIG
1262         } else if (static_branch_unlikely(&tcp_md5_needed.key)) {
1263                 int l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1264
1265                 key.md5_key = tcp_v6_md5_do_lookup(sk, &ipv6_hdr(skb)->saddr,
1266                                                    l3index);
1267                 if (key.md5_key)
1268                         key.type = TCP_KEY_MD5;
1269 #endif
1270         }
1271
1272         /* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV
1273          * sk->sk_state == TCP_SYN_RECV -> for Fast Open.
1274          */
1275         tcp_v6_send_ack(sk, skb, (sk->sk_state == TCP_LISTEN) ?
1276                         tcp_rsk(req)->snt_isn + 1 : tcp_sk(sk)->snd_nxt,
1277                         tcp_rsk(req)->rcv_nxt,
1278                         tcp_synack_window(req) >> inet_rsk(req)->rcv_wscale,
1279                         tcp_rsk_tsval(tcp_rsk(req)),
1280                         READ_ONCE(req->ts_recent), sk->sk_bound_dev_if,
1281                         &key, ipv6_get_dsfield(ipv6_hdr(skb)), 0,
1282                         READ_ONCE(sk->sk_priority),
1283                         READ_ONCE(tcp_rsk(req)->txhash));
1284         if (tcp_key_is_ao(&key))
1285                 kfree(key.traffic_key);
1286 }
1287
1288
1289 static struct sock *tcp_v6_cookie_check(struct sock *sk, struct sk_buff *skb)
1290 {
1291 #ifdef CONFIG_SYN_COOKIES
1292         const struct tcphdr *th = tcp_hdr(skb);
1293
1294         if (!th->syn)
1295                 sk = cookie_v6_check(sk, skb);
1296 #endif
1297         return sk;
1298 }
1299
1300 u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph,
1301                          struct tcphdr *th, u32 *cookie)
1302 {
1303         u16 mss = 0;
1304 #ifdef CONFIG_SYN_COOKIES
1305         mss = tcp_get_syncookie_mss(&tcp6_request_sock_ops,
1306                                     &tcp_request_sock_ipv6_ops, sk, th);
1307         if (mss) {
1308                 *cookie = __cookie_v6_init_sequence(iph, th, &mss);
1309                 tcp_synq_overflow(sk);
1310         }
1311 #endif
1312         return mss;
1313 }
1314
1315 static int tcp_v6_conn_request(struct sock *sk, struct sk_buff *skb)
1316 {
1317         if (skb->protocol == htons(ETH_P_IP))
1318                 return tcp_v4_conn_request(sk, skb);
1319
1320         if (!ipv6_unicast_destination(skb))
1321                 goto drop;
1322
1323         if (ipv6_addr_v4mapped(&ipv6_hdr(skb)->saddr)) {
1324                 __IP6_INC_STATS(sock_net(sk), NULL, IPSTATS_MIB_INHDRERRORS);
1325                 return 0;
1326         }
1327
1328         return tcp_conn_request(&tcp6_request_sock_ops,
1329                                 &tcp_request_sock_ipv6_ops, sk, skb);
1330
1331 drop:
1332         tcp_listendrop(sk);
1333         return 0; /* don't send reset */
1334 }
1335
1336 static void tcp_v6_restore_cb(struct sk_buff *skb)
1337 {
1338         /* We need to move header back to the beginning if xfrm6_policy_check()
1339          * and tcp_v6_fill_cb() are going to be called again.
1340          * ip6_datagram_recv_specific_ctl() also expects IP6CB to be there.
1341          */
1342         memmove(IP6CB(skb), &TCP_SKB_CB(skb)->header.h6,
1343                 sizeof(struct inet6_skb_parm));
1344 }
1345
1346 static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
1347                                          struct request_sock *req,
1348                                          struct dst_entry *dst,
1349                                          struct request_sock *req_unhash,
1350                                          bool *own_req)
1351 {
1352         struct inet_request_sock *ireq;
1353         struct ipv6_pinfo *newnp;
1354         const struct ipv6_pinfo *np = tcp_inet6_sk(sk);
1355         struct ipv6_txoptions *opt;
1356         struct inet_sock *newinet;
1357         bool found_dup_sk = false;
1358         struct tcp_sock *newtp;
1359         struct sock *newsk;
1360 #ifdef CONFIG_TCP_MD5SIG
1361         struct tcp_md5sig_key *key;
1362         int l3index;
1363 #endif
1364         struct flowi6 fl6;
1365
1366         if (skb->protocol == htons(ETH_P_IP)) {
1367                 /*
1368                  *      v6 mapped
1369                  */
1370
1371                 newsk = tcp_v4_syn_recv_sock(sk, skb, req, dst,
1372                                              req_unhash, own_req);
1373
1374                 if (!newsk)
1375                         return NULL;
1376
1377                 inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk);
1378
1379                 newnp = tcp_inet6_sk(newsk);
1380                 newtp = tcp_sk(newsk);
1381
1382                 memcpy(newnp, np, sizeof(struct ipv6_pinfo));
1383
1384                 newnp->saddr = newsk->sk_v6_rcv_saddr;
1385
1386                 inet_csk(newsk)->icsk_af_ops = &ipv6_mapped;
1387                 if (sk_is_mptcp(newsk))
1388                         mptcpv6_handle_mapped(newsk, true);
1389                 newsk->sk_backlog_rcv = tcp_v4_do_rcv;
1390 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1391                 newtp->af_specific = &tcp_sock_ipv6_mapped_specific;
1392 #endif
1393
1394                 newnp->ipv6_mc_list = NULL;
1395                 newnp->ipv6_ac_list = NULL;
1396                 newnp->ipv6_fl_list = NULL;
1397                 newnp->pktoptions  = NULL;
1398                 newnp->opt         = NULL;
1399                 newnp->mcast_oif   = inet_iif(skb);
1400                 newnp->mcast_hops  = ip_hdr(skb)->ttl;
1401                 newnp->rcv_flowinfo = 0;
1402                 if (inet6_test_bit(REPFLOW, sk))
1403                         newnp->flow_label = 0;
1404
1405                 /*
1406                  * No need to charge this sock to the relevant IPv6 refcnt debug socks count
1407                  * here, tcp_create_openreq_child now does this for us, see the comment in
1408                  * that function for the gory details. -acme
1409                  */
1410
1411                 /* It is tricky place. Until this moment IPv4 tcp
1412                    worked with IPv6 icsk.icsk_af_ops.
1413                    Sync it now.
1414                  */
1415                 tcp_sync_mss(newsk, inet_csk(newsk)->icsk_pmtu_cookie);
1416
1417                 return newsk;
1418         }
1419
1420         ireq = inet_rsk(req);
1421
1422         if (sk_acceptq_is_full(sk))
1423                 goto out_overflow;
1424
1425         if (!dst) {
1426                 dst = inet6_csk_route_req(sk, &fl6, req, IPPROTO_TCP);
1427                 if (!dst)
1428                         goto out;
1429         }
1430
1431         newsk = tcp_create_openreq_child(sk, req, skb);
1432         if (!newsk)
1433                 goto out_nonewsk;
1434
1435         /*
1436          * No need to charge this sock to the relevant IPv6 refcnt debug socks
1437          * count here, tcp_create_openreq_child now does this for us, see the
1438          * comment in that function for the gory details. -acme
1439          */
1440
1441         newsk->sk_gso_type = SKB_GSO_TCPV6;
1442         inet6_sk_rx_dst_set(newsk, skb);
1443
1444         inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk);
1445
1446         newtp = tcp_sk(newsk);
1447         newinet = inet_sk(newsk);
1448         newnp = tcp_inet6_sk(newsk);
1449
1450         memcpy(newnp, np, sizeof(struct ipv6_pinfo));
1451
1452         ip6_dst_store(newsk, dst, NULL, NULL);
1453
1454         newsk->sk_v6_daddr = ireq->ir_v6_rmt_addr;
1455         newnp->saddr = ireq->ir_v6_loc_addr;
1456         newsk->sk_v6_rcv_saddr = ireq->ir_v6_loc_addr;
1457         newsk->sk_bound_dev_if = ireq->ir_iif;
1458
1459         /* Now IPv6 options...
1460
1461            First: no IPv4 options.
1462          */
1463         newinet->inet_opt = NULL;
1464         newnp->ipv6_mc_list = NULL;
1465         newnp->ipv6_ac_list = NULL;
1466         newnp->ipv6_fl_list = NULL;
1467
1468         /* Clone RX bits */
1469         newnp->rxopt.all = np->rxopt.all;
1470
1471         newnp->pktoptions = NULL;
1472         newnp->opt        = NULL;
1473         newnp->mcast_oif  = tcp_v6_iif(skb);
1474         newnp->mcast_hops = ipv6_hdr(skb)->hop_limit;
1475         newnp->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(skb));
1476         if (inet6_test_bit(REPFLOW, sk))
1477                 newnp->flow_label = ip6_flowlabel(ipv6_hdr(skb));
1478
1479         /* Set ToS of the new socket based upon the value of incoming SYN.
1480          * ECT bits are set later in tcp_init_transfer().
1481          */
1482         if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos))
1483                 newnp->tclass = tcp_rsk(req)->syn_tos & ~INET_ECN_MASK;
1484
1485         /* Clone native IPv6 options from listening socket (if any)
1486
1487            Yes, keeping reference count would be much more clever,
1488            but we make one more one thing there: reattach optmem
1489            to newsk.
1490          */
1491         opt = ireq->ipv6_opt;
1492         if (!opt)
1493                 opt = rcu_dereference(np->opt);
1494         if (opt) {
1495                 opt = ipv6_dup_options(newsk, opt);
1496                 RCU_INIT_POINTER(newnp->opt, opt);
1497         }
1498         inet_csk(newsk)->icsk_ext_hdr_len = 0;
1499         if (opt)
1500                 inet_csk(newsk)->icsk_ext_hdr_len = opt->opt_nflen +
1501                                                     opt->opt_flen;
1502
1503         tcp_ca_openreq_child(newsk, dst);
1504
1505         tcp_sync_mss(newsk, dst_mtu(dst));
1506         newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst));
1507
1508         tcp_initialize_rcv_mss(newsk);
1509
1510         newinet->inet_daddr = newinet->inet_saddr = LOOPBACK4_IPV6;
1511         newinet->inet_rcv_saddr = LOOPBACK4_IPV6;
1512
1513 #ifdef CONFIG_TCP_MD5SIG
1514         l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif);
1515
1516         if (!tcp_rsk_used_ao(req)) {
1517                 /* Copy over the MD5 key from the original socket */
1518                 key = tcp_v6_md5_do_lookup(sk, &newsk->sk_v6_daddr, l3index);
1519                 if (key) {
1520                         const union tcp_md5_addr *addr;
1521
1522                         addr = (union tcp_md5_addr *)&newsk->sk_v6_daddr;
1523                         if (tcp_md5_key_copy(newsk, addr, AF_INET6, 128, l3index, key)) {
1524                                 inet_csk_prepare_forced_close(newsk);
1525                                 tcp_done(newsk);
1526                                 goto out;
1527                         }
1528                 }
1529         }
1530 #endif
1531 #ifdef CONFIG_TCP_AO
1532         /* Copy over tcp_ao_info if any */
1533         if (tcp_ao_copy_all_matching(sk, newsk, req, skb, AF_INET6))
1534                 goto out; /* OOM */
1535 #endif
1536
1537         if (__inet_inherit_port(sk, newsk) < 0) {
1538                 inet_csk_prepare_forced_close(newsk);
1539                 tcp_done(newsk);
1540                 goto out;
1541         }
1542         *own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash),
1543                                        &found_dup_sk);
1544         if (*own_req) {
1545                 tcp_move_syn(newtp, req);
1546
1547                 /* Clone pktoptions received with SYN, if we own the req */
1548                 if (ireq->pktopts) {
1549                         newnp->pktoptions = skb_clone_and_charge_r(ireq->pktopts, newsk);
1550                         consume_skb(ireq->pktopts);
1551                         ireq->pktopts = NULL;
1552                         if (newnp->pktoptions)
1553                                 tcp_v6_restore_cb(newnp->pktoptions);
1554                 }
1555         } else {
1556                 if (!req_unhash && found_dup_sk) {
1557                         /* This code path should only be executed in the
1558                          * syncookie case only
1559                          */
1560                         bh_unlock_sock(newsk);
1561                         sock_put(newsk);
1562                         newsk = NULL;
1563                 }
1564         }
1565
1566         return newsk;
1567
1568 out_overflow:
1569         __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
1570 out_nonewsk:
1571         dst_release(dst);
1572 out:
1573         tcp_listendrop(sk);
1574         return NULL;
1575 }
1576
1577 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
1578                                                            u32));
1579 /* The socket must have it's spinlock held when we get
1580  * here, unless it is a TCP_LISTEN socket.
1581  *
1582  * We have a potential double-lock case here, so even when
1583  * doing backlog processing we use the BH locking scheme.
1584  * This is because we cannot sleep with the original spinlock
1585  * held.
1586  */
1587 INDIRECT_CALLABLE_SCOPE
1588 int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)
1589 {
1590         struct ipv6_pinfo *np = tcp_inet6_sk(sk);
1591         struct sk_buff *opt_skb = NULL;
1592         enum skb_drop_reason reason;
1593         struct tcp_sock *tp;
1594
1595         /* Imagine: socket is IPv6. IPv4 packet arrives,
1596            goes to IPv4 receive handler and backlogged.
1597            From backlog it always goes here. Kerboom...
1598            Fortunately, tcp_rcv_established and rcv_established
1599            handle them correctly, but it is not case with
1600            tcp_v6_hnd_req and tcp_v6_send_reset().   --ANK
1601          */
1602
1603         if (skb->protocol == htons(ETH_P_IP))
1604                 return tcp_v4_do_rcv(sk, skb);
1605
1606         /*
1607          *      socket locking is here for SMP purposes as backlog rcv
1608          *      is currently called with bh processing disabled.
1609          */
1610
1611         /* Do Stevens' IPV6_PKTOPTIONS.
1612
1613            Yes, guys, it is the only place in our code, where we
1614            may make it not affecting IPv4.
1615            The rest of code is protocol independent,
1616            and I do not like idea to uglify IPv4.
1617
1618            Actually, all the idea behind IPV6_PKTOPTIONS
1619            looks not very well thought. For now we latch
1620            options, received in the last packet, enqueued
1621            by tcp. Feel free to propose better solution.
1622                                                --ANK (980728)
1623          */
1624         if (np->rxopt.all)
1625                 opt_skb = skb_clone_and_charge_r(skb, sk);
1626
1627         if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
1628                 struct dst_entry *dst;
1629
1630                 dst = rcu_dereference_protected(sk->sk_rx_dst,
1631                                                 lockdep_sock_is_held(sk));
1632
1633                 sock_rps_save_rxhash(sk, skb);
1634                 sk_mark_napi_id(sk, skb);
1635                 if (dst) {
1636                         if (sk->sk_rx_dst_ifindex != skb->skb_iif ||
1637                             INDIRECT_CALL_1(dst->ops->check, ip6_dst_check,
1638                                             dst, sk->sk_rx_dst_cookie) == NULL) {
1639                                 RCU_INIT_POINTER(sk->sk_rx_dst, NULL);
1640                                 dst_release(dst);
1641                         }
1642                 }
1643
1644                 tcp_rcv_established(sk, skb);
1645                 if (opt_skb)
1646                         goto ipv6_pktoptions;
1647                 return 0;
1648         }
1649
1650         if (tcp_checksum_complete(skb))
1651                 goto csum_err;
1652
1653         if (sk->sk_state == TCP_LISTEN) {
1654                 struct sock *nsk = tcp_v6_cookie_check(sk, skb);
1655
1656                 if (nsk != sk) {
1657                         if (nsk) {
1658                                 reason = tcp_child_process(sk, nsk, skb);
1659                                 if (reason)
1660                                         goto reset;
1661                         }
1662                         if (opt_skb)
1663                                 __kfree_skb(opt_skb);
1664                         return 0;
1665                 }
1666         } else
1667                 sock_rps_save_rxhash(sk, skb);
1668
1669         reason = tcp_rcv_state_process(sk, skb);
1670         if (reason)
1671                 goto reset;
1672         if (opt_skb)
1673                 goto ipv6_pktoptions;
1674         return 0;
1675
1676 reset:
1677         tcp_v6_send_reset(sk, skb, sk_rst_convert_drop_reason(reason));
1678 discard:
1679         if (opt_skb)
1680                 __kfree_skb(opt_skb);
1681         kfree_skb_reason(skb, reason);
1682         return 0;
1683 csum_err:
1684         reason = SKB_DROP_REASON_TCP_CSUM;
1685         trace_tcp_bad_csum(skb);
1686         TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1687         TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1688         goto discard;
1689
1690
1691 ipv6_pktoptions:
1692         /* Do you ask, what is it?
1693
1694            1. skb was enqueued by tcp.
1695            2. skb is added to tail of read queue, rather than out of order.
1696            3. socket is not in passive state.
1697            4. Finally, it really contains options, which user wants to receive.
1698          */
1699         tp = tcp_sk(sk);
1700         if (TCP_SKB_CB(opt_skb)->end_seq == tp->rcv_nxt &&
1701             !((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) {
1702                 if (np->rxopt.bits.rxinfo || np->rxopt.bits.rxoinfo)
1703                         WRITE_ONCE(np->mcast_oif, tcp_v6_iif(opt_skb));
1704                 if (np->rxopt.bits.rxhlim || np->rxopt.bits.rxohlim)
1705                         WRITE_ONCE(np->mcast_hops,
1706                                    ipv6_hdr(opt_skb)->hop_limit);
1707                 if (np->rxopt.bits.rxflow || np->rxopt.bits.rxtclass)
1708                         np->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(opt_skb));
1709                 if (inet6_test_bit(REPFLOW, sk))
1710                         np->flow_label = ip6_flowlabel(ipv6_hdr(opt_skb));
1711                 if (ipv6_opt_accepted(sk, opt_skb, &TCP_SKB_CB(opt_skb)->header.h6)) {
1712                         tcp_v6_restore_cb(opt_skb);
1713                         opt_skb = xchg(&np->pktoptions, opt_skb);
1714                 } else {
1715                         __kfree_skb(opt_skb);
1716                         opt_skb = xchg(&np->pktoptions, NULL);
1717                 }
1718         }
1719
1720         consume_skb(opt_skb);
1721         return 0;
1722 }
1723
1724 static void tcp_v6_fill_cb(struct sk_buff *skb, const struct ipv6hdr *hdr,
1725                            const struct tcphdr *th)
1726 {
1727         /* This is tricky: we move IP6CB at its correct location into
1728          * TCP_SKB_CB(). It must be done after xfrm6_policy_check(), because
1729          * _decode_session6() uses IP6CB().
1730          * barrier() makes sure compiler won't play aliasing games.
1731          */
1732         memmove(&TCP_SKB_CB(skb)->header.h6, IP6CB(skb),
1733                 sizeof(struct inet6_skb_parm));
1734         barrier();
1735
1736         TCP_SKB_CB(skb)->seq = ntohl(th->seq);
1737         TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
1738                                     skb->len - th->doff*4);
1739         TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
1740         TCP_SKB_CB(skb)->tcp_flags = tcp_flag_byte(th);
1741         TCP_SKB_CB(skb)->ip_dsfield = ipv6_get_dsfield(hdr);
1742         TCP_SKB_CB(skb)->sacked = 0;
1743         TCP_SKB_CB(skb)->has_rxtstamp =
1744                         skb->tstamp || skb_hwtstamps(skb)->hwtstamp;
1745 }
1746
1747 INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb)
1748 {
1749         enum skb_drop_reason drop_reason;
1750         int sdif = inet6_sdif(skb);
1751         int dif = inet6_iif(skb);
1752         const struct tcphdr *th;
1753         const struct ipv6hdr *hdr;
1754         bool refcounted;
1755         struct sock *sk;
1756         int ret;
1757         u32 isn;
1758         struct net *net = dev_net(skb->dev);
1759
1760         drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
1761         if (skb->pkt_type != PACKET_HOST)
1762                 goto discard_it;
1763
1764         /*
1765          *      Count it even if it's bad.
1766          */
1767         __TCP_INC_STATS(net, TCP_MIB_INSEGS);
1768
1769         if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
1770                 goto discard_it;
1771
1772         th = (const struct tcphdr *)skb->data;
1773
1774         if (unlikely(th->doff < sizeof(struct tcphdr) / 4)) {
1775                 drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL;
1776                 goto bad_packet;
1777         }
1778         if (!pskb_may_pull(skb, th->doff*4))
1779                 goto discard_it;
1780
1781         if (skb_checksum_init(skb, IPPROTO_TCP, ip6_compute_pseudo))
1782                 goto csum_error;
1783
1784         th = (const struct tcphdr *)skb->data;
1785         hdr = ipv6_hdr(skb);
1786
1787 lookup:
1788         sk = __inet6_lookup_skb(net->ipv4.tcp_death_row.hashinfo, skb, __tcp_hdrlen(th),
1789                                 th->source, th->dest, inet6_iif(skb), sdif,
1790                                 &refcounted);
1791         if (!sk)
1792                 goto no_tcp_socket;
1793
1794         if (sk->sk_state == TCP_TIME_WAIT)
1795                 goto do_time_wait;
1796
1797         if (sk->sk_state == TCP_NEW_SYN_RECV) {
1798                 struct request_sock *req = inet_reqsk(sk);
1799                 bool req_stolen = false;
1800                 struct sock *nsk;
1801
1802                 sk = req->rsk_listener;
1803                 if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb))
1804                         drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1805                 else
1806                         drop_reason = tcp_inbound_hash(sk, req, skb,
1807                                                        &hdr->saddr, &hdr->daddr,
1808                                                        AF_INET6, dif, sdif);
1809                 if (drop_reason) {
1810                         sk_drops_add(sk, skb);
1811                         reqsk_put(req);
1812                         goto discard_it;
1813                 }
1814                 if (tcp_checksum_complete(skb)) {
1815                         reqsk_put(req);
1816                         goto csum_error;
1817                 }
1818                 if (unlikely(sk->sk_state != TCP_LISTEN)) {
1819                         nsk = reuseport_migrate_sock(sk, req_to_sk(req), skb);
1820                         if (!nsk) {
1821                                 inet_csk_reqsk_queue_drop_and_put(sk, req);
1822                                 goto lookup;
1823                         }
1824                         sk = nsk;
1825                         /* reuseport_migrate_sock() has already held one sk_refcnt
1826                          * before returning.
1827                          */
1828                 } else {
1829                         sock_hold(sk);
1830                 }
1831                 refcounted = true;
1832                 nsk = NULL;
1833                 if (!tcp_filter(sk, skb)) {
1834                         th = (const struct tcphdr *)skb->data;
1835                         hdr = ipv6_hdr(skb);
1836                         tcp_v6_fill_cb(skb, hdr, th);
1837                         nsk = tcp_check_req(sk, skb, req, false, &req_stolen);
1838                 } else {
1839                         drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
1840                 }
1841                 if (!nsk) {
1842                         reqsk_put(req);
1843                         if (req_stolen) {
1844                                 /* Another cpu got exclusive access to req
1845                                  * and created a full blown socket.
1846                                  * Try to feed this packet to this socket
1847                                  * instead of discarding it.
1848                                  */
1849                                 tcp_v6_restore_cb(skb);
1850                                 sock_put(sk);
1851                                 goto lookup;
1852                         }
1853                         goto discard_and_relse;
1854                 }
1855                 nf_reset_ct(skb);
1856                 if (nsk == sk) {
1857                         reqsk_put(req);
1858                         tcp_v6_restore_cb(skb);
1859                 } else {
1860                         drop_reason = tcp_child_process(sk, nsk, skb);
1861                         if (drop_reason) {
1862                                 enum sk_rst_reason rst_reason;
1863
1864                                 rst_reason = sk_rst_convert_drop_reason(drop_reason);
1865                                 tcp_v6_send_reset(nsk, skb, rst_reason);
1866                                 goto discard_and_relse;
1867                         }
1868                         sock_put(sk);
1869                         return 0;
1870                 }
1871         }
1872
1873 process:
1874         if (static_branch_unlikely(&ip6_min_hopcount)) {
1875                 /* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */
1876                 if (unlikely(hdr->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount))) {
1877                         __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
1878                         drop_reason = SKB_DROP_REASON_TCP_MINTTL;
1879                         goto discard_and_relse;
1880                 }
1881         }
1882
1883         if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb)) {
1884                 drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1885                 goto discard_and_relse;
1886         }
1887
1888         drop_reason = tcp_inbound_hash(sk, NULL, skb, &hdr->saddr, &hdr->daddr,
1889                                        AF_INET6, dif, sdif);
1890         if (drop_reason)
1891                 goto discard_and_relse;
1892
1893         nf_reset_ct(skb);
1894
1895         if (tcp_filter(sk, skb)) {
1896                 drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
1897                 goto discard_and_relse;
1898         }
1899         th = (const struct tcphdr *)skb->data;
1900         hdr = ipv6_hdr(skb);
1901         tcp_v6_fill_cb(skb, hdr, th);
1902
1903         skb->dev = NULL;
1904
1905         if (sk->sk_state == TCP_LISTEN) {
1906                 ret = tcp_v6_do_rcv(sk, skb);
1907                 goto put_and_return;
1908         }
1909
1910         sk_incoming_cpu_update(sk);
1911
1912         bh_lock_sock_nested(sk);
1913         tcp_segs_in(tcp_sk(sk), skb);
1914         ret = 0;
1915         if (!sock_owned_by_user(sk)) {
1916                 ret = tcp_v6_do_rcv(sk, skb);
1917         } else {
1918                 if (tcp_add_backlog(sk, skb, &drop_reason))
1919                         goto discard_and_relse;
1920         }
1921         bh_unlock_sock(sk);
1922 put_and_return:
1923         if (refcounted)
1924                 sock_put(sk);
1925         return ret ? -1 : 0;
1926
1927 no_tcp_socket:
1928         drop_reason = SKB_DROP_REASON_NO_SOCKET;
1929         if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb))
1930                 goto discard_it;
1931
1932         tcp_v6_fill_cb(skb, hdr, th);
1933
1934         if (tcp_checksum_complete(skb)) {
1935 csum_error:
1936                 drop_reason = SKB_DROP_REASON_TCP_CSUM;
1937                 trace_tcp_bad_csum(skb);
1938                 __TCP_INC_STATS(net, TCP_MIB_CSUMERRORS);
1939 bad_packet:
1940                 __TCP_INC_STATS(net, TCP_MIB_INERRS);
1941         } else {
1942                 tcp_v6_send_reset(NULL, skb, sk_rst_convert_drop_reason(drop_reason));
1943         }
1944
1945 discard_it:
1946         SKB_DR_OR(drop_reason, NOT_SPECIFIED);
1947         kfree_skb_reason(skb, drop_reason);
1948         return 0;
1949
1950 discard_and_relse:
1951         sk_drops_add(sk, skb);
1952         if (refcounted)
1953                 sock_put(sk);
1954         goto discard_it;
1955
1956 do_time_wait:
1957         if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb)) {
1958                 drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1959                 inet_twsk_put(inet_twsk(sk));
1960                 goto discard_it;
1961         }
1962
1963         tcp_v6_fill_cb(skb, hdr, th);
1964
1965         if (tcp_checksum_complete(skb)) {
1966                 inet_twsk_put(inet_twsk(sk));
1967                 goto csum_error;
1968         }
1969
1970         switch (tcp_timewait_state_process(inet_twsk(sk), skb, th, &isn)) {
1971         case TCP_TW_SYN:
1972         {
1973                 struct sock *sk2;
1974
1975                 sk2 = inet6_lookup_listener(net, net->ipv4.tcp_death_row.hashinfo,
1976                                             skb, __tcp_hdrlen(th),
1977                                             &ipv6_hdr(skb)->saddr, th->source,
1978                                             &ipv6_hdr(skb)->daddr,
1979                                             ntohs(th->dest),
1980                                             tcp_v6_iif_l3_slave(skb),
1981                                             sdif);
1982                 if (sk2) {
1983                         struct inet_timewait_sock *tw = inet_twsk(sk);
1984                         inet_twsk_deschedule_put(tw);
1985                         sk = sk2;
1986                         tcp_v6_restore_cb(skb);
1987                         refcounted = false;
1988                         __this_cpu_write(tcp_tw_isn, isn);
1989                         goto process;
1990                 }
1991         }
1992                 /* to ACK */
1993                 fallthrough;
1994         case TCP_TW_ACK:
1995                 tcp_v6_timewait_ack(sk, skb);
1996                 break;
1997         case TCP_TW_RST:
1998                 tcp_v6_send_reset(sk, skb, SK_RST_REASON_TCP_TIMEWAIT_SOCKET);
1999                 inet_twsk_deschedule_put(inet_twsk(sk));
2000                 goto discard_it;
2001         case TCP_TW_SUCCESS:
2002                 ;
2003         }
2004         goto discard_it;
2005 }
2006
2007 void tcp_v6_early_demux(struct sk_buff *skb)
2008 {
2009         struct net *net = dev_net(skb->dev);
2010         const struct ipv6hdr *hdr;
2011         const struct tcphdr *th;
2012         struct sock *sk;
2013
2014         if (skb->pkt_type != PACKET_HOST)
2015                 return;
2016
2017         if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct tcphdr)))
2018                 return;
2019
2020         hdr = ipv6_hdr(skb);
2021         th = tcp_hdr(skb);
2022
2023         if (th->doff < sizeof(struct tcphdr) / 4)
2024                 return;
2025
2026         /* Note : We use inet6_iif() here, not tcp_v6_iif() */
2027         sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
2028                                         &hdr->saddr, th->source,
2029                                         &hdr->daddr, ntohs(th->dest),
2030                                         inet6_iif(skb), inet6_sdif(skb));
2031         if (sk) {
2032                 skb->sk = sk;
2033                 skb->destructor = sock_edemux;
2034                 if (sk_fullsock(sk)) {
2035                         struct dst_entry *dst = rcu_dereference(sk->sk_rx_dst);
2036
2037                         if (dst)
2038                                 dst = dst_check(dst, sk->sk_rx_dst_cookie);
2039                         if (dst &&
2040                             sk->sk_rx_dst_ifindex == skb->skb_iif)
2041                                 skb_dst_set_noref(skb, dst);
2042                 }
2043         }
2044 }
2045
2046 static struct timewait_sock_ops tcp6_timewait_sock_ops = {
2047         .twsk_obj_size  = sizeof(struct tcp6_timewait_sock),
2048         .twsk_destructor = tcp_twsk_destructor,
2049 };
2050
2051 INDIRECT_CALLABLE_SCOPE void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb)
2052 {
2053         __tcp_v6_send_check(skb, &sk->sk_v6_rcv_saddr, &sk->sk_v6_daddr);
2054 }
2055
2056 const struct inet_connection_sock_af_ops ipv6_specific = {
2057         .queue_xmit        = inet6_csk_xmit,
2058         .send_check        = tcp_v6_send_check,
2059         .rebuild_header    = inet6_sk_rebuild_header,
2060         .sk_rx_dst_set     = inet6_sk_rx_dst_set,
2061         .conn_request      = tcp_v6_conn_request,
2062         .syn_recv_sock     = tcp_v6_syn_recv_sock,
2063         .net_header_len    = sizeof(struct ipv6hdr),
2064         .setsockopt        = ipv6_setsockopt,
2065         .getsockopt        = ipv6_getsockopt,
2066         .addr2sockaddr     = inet6_csk_addr2sockaddr,
2067         .sockaddr_len      = sizeof(struct sockaddr_in6),
2068         .mtu_reduced       = tcp_v6_mtu_reduced,
2069 };
2070
2071 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2072 static const struct tcp_sock_af_ops tcp_sock_ipv6_specific = {
2073 #ifdef CONFIG_TCP_MD5SIG
2074         .md5_lookup     =       tcp_v6_md5_lookup,
2075         .calc_md5_hash  =       tcp_v6_md5_hash_skb,
2076         .md5_parse      =       tcp_v6_parse_md5_keys,
2077 #endif
2078 #ifdef CONFIG_TCP_AO
2079         .ao_lookup      =       tcp_v6_ao_lookup,
2080         .calc_ao_hash   =       tcp_v6_ao_hash_skb,
2081         .ao_parse       =       tcp_v6_parse_ao,
2082         .ao_calc_key_sk =       tcp_v6_ao_calc_key_sk,
2083 #endif
2084 };
2085 #endif
2086
2087 /*
2088  *      TCP over IPv4 via INET6 API
2089  */
2090 static const struct inet_connection_sock_af_ops ipv6_mapped = {
2091         .queue_xmit        = ip_queue_xmit,
2092         .send_check        = tcp_v4_send_check,
2093         .rebuild_header    = inet_sk_rebuild_header,
2094         .sk_rx_dst_set     = inet_sk_rx_dst_set,
2095         .conn_request      = tcp_v6_conn_request,
2096         .syn_recv_sock     = tcp_v6_syn_recv_sock,
2097         .net_header_len    = sizeof(struct iphdr),
2098         .setsockopt        = ipv6_setsockopt,
2099         .getsockopt        = ipv6_getsockopt,
2100         .addr2sockaddr     = inet6_csk_addr2sockaddr,
2101         .sockaddr_len      = sizeof(struct sockaddr_in6),
2102         .mtu_reduced       = tcp_v4_mtu_reduced,
2103 };
2104
2105 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2106 static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific = {
2107 #ifdef CONFIG_TCP_MD5SIG
2108         .md5_lookup     =       tcp_v4_md5_lookup,
2109         .calc_md5_hash  =       tcp_v4_md5_hash_skb,
2110         .md5_parse      =       tcp_v6_parse_md5_keys,
2111 #endif
2112 #ifdef CONFIG_TCP_AO
2113         .ao_lookup      =       tcp_v6_ao_lookup,
2114         .calc_ao_hash   =       tcp_v4_ao_hash_skb,
2115         .ao_parse       =       tcp_v6_parse_ao,
2116         .ao_calc_key_sk =       tcp_v4_ao_calc_key_sk,
2117 #endif
2118 };
2119 #endif
2120
2121 /* NOTE: A lot of things set to zero explicitly by call to
2122  *       sk_alloc() so need not be done here.
2123  */
2124 static int tcp_v6_init_sock(struct sock *sk)
2125 {
2126         struct inet_connection_sock *icsk = inet_csk(sk);
2127
2128         tcp_init_sock(sk);
2129
2130         icsk->icsk_af_ops = &ipv6_specific;
2131
2132 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2133         tcp_sk(sk)->af_specific = &tcp_sock_ipv6_specific;
2134 #endif
2135
2136         return 0;
2137 }
2138
2139 #ifdef CONFIG_PROC_FS
2140 /* Proc filesystem TCPv6 sock list dumping. */
2141 static void get_openreq6(struct seq_file *seq,
2142                          const struct request_sock *req, int i)
2143 {
2144         long ttd = req->rsk_timer.expires - jiffies;
2145         const struct in6_addr *src = &inet_rsk(req)->ir_v6_loc_addr;
2146         const struct in6_addr *dest = &inet_rsk(req)->ir_v6_rmt_addr;
2147
2148         if (ttd < 0)
2149                 ttd = 0;
2150
2151         seq_printf(seq,
2152                    "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2153                    "%02X %08X:%08X %02X:%08lX %08X %5u %8d %d %d %pK\n",
2154                    i,
2155                    src->s6_addr32[0], src->s6_addr32[1],
2156                    src->s6_addr32[2], src->s6_addr32[3],
2157                    inet_rsk(req)->ir_num,
2158                    dest->s6_addr32[0], dest->s6_addr32[1],
2159                    dest->s6_addr32[2], dest->s6_addr32[3],
2160                    ntohs(inet_rsk(req)->ir_rmt_port),
2161                    TCP_SYN_RECV,
2162                    0, 0, /* could print option size, but that is af dependent. */
2163                    1,   /* timers active (only the expire timer) */
2164                    jiffies_to_clock_t(ttd),
2165                    req->num_timeout,
2166                    from_kuid_munged(seq_user_ns(seq),
2167                                     sock_i_uid(req->rsk_listener)),
2168                    0,  /* non standard timer */
2169                    0, /* open_requests have no inode */
2170                    0, req);
2171 }
2172
2173 static void get_tcp6_sock(struct seq_file *seq, struct sock *sp, int i)
2174 {
2175         const struct in6_addr *dest, *src;
2176         __u16 destp, srcp;
2177         int timer_active;
2178         unsigned long timer_expires;
2179         const struct inet_sock *inet = inet_sk(sp);
2180         const struct tcp_sock *tp = tcp_sk(sp);
2181         const struct inet_connection_sock *icsk = inet_csk(sp);
2182         const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq;
2183         int rx_queue;
2184         int state;
2185
2186         dest  = &sp->sk_v6_daddr;
2187         src   = &sp->sk_v6_rcv_saddr;
2188         destp = ntohs(inet->inet_dport);
2189         srcp  = ntohs(inet->inet_sport);
2190
2191         if (icsk->icsk_pending == ICSK_TIME_RETRANS ||
2192             icsk->icsk_pending == ICSK_TIME_REO_TIMEOUT ||
2193             icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {
2194                 timer_active    = 1;
2195                 timer_expires   = icsk->icsk_timeout;
2196         } else if (icsk->icsk_pending == ICSK_TIME_PROBE0) {
2197                 timer_active    = 4;
2198                 timer_expires   = icsk->icsk_timeout;
2199         } else if (timer_pending(&sp->sk_timer)) {
2200                 timer_active    = 2;
2201                 timer_expires   = sp->sk_timer.expires;
2202         } else {
2203                 timer_active    = 0;
2204                 timer_expires = jiffies;
2205         }
2206
2207         state = inet_sk_state_load(sp);
2208         if (state == TCP_LISTEN)
2209                 rx_queue = READ_ONCE(sp->sk_ack_backlog);
2210         else
2211                 /* Because we don't lock the socket,
2212                  * we might find a transient negative value.
2213                  */
2214                 rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) -
2215                                       READ_ONCE(tp->copied_seq), 0);
2216
2217         seq_printf(seq,
2218                    "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2219                    "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %lu %lu %u %u %d\n",
2220                    i,
2221                    src->s6_addr32[0], src->s6_addr32[1],
2222                    src->s6_addr32[2], src->s6_addr32[3], srcp,
2223                    dest->s6_addr32[0], dest->s6_addr32[1],
2224                    dest->s6_addr32[2], dest->s6_addr32[3], destp,
2225                    state,
2226                    READ_ONCE(tp->write_seq) - tp->snd_una,
2227                    rx_queue,
2228                    timer_active,
2229                    jiffies_delta_to_clock_t(timer_expires - jiffies),
2230                    icsk->icsk_retransmits,
2231                    from_kuid_munged(seq_user_ns(seq), sock_i_uid(sp)),
2232                    icsk->icsk_probes_out,
2233                    sock_i_ino(sp),
2234                    refcount_read(&sp->sk_refcnt), sp,
2235                    jiffies_to_clock_t(icsk->icsk_rto),
2236                    jiffies_to_clock_t(icsk->icsk_ack.ato),
2237                    (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sp),
2238                    tcp_snd_cwnd(tp),
2239                    state == TCP_LISTEN ?
2240                         fastopenq->max_qlen :
2241                         (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh)
2242                    );
2243 }
2244
2245 static void get_timewait6_sock(struct seq_file *seq,
2246                                struct inet_timewait_sock *tw, int i)
2247 {
2248         long delta = tw->tw_timer.expires - jiffies;
2249         const struct in6_addr *dest, *src;
2250         __u16 destp, srcp;
2251
2252         dest = &tw->tw_v6_daddr;
2253         src  = &tw->tw_v6_rcv_saddr;
2254         destp = ntohs(tw->tw_dport);
2255         srcp  = ntohs(tw->tw_sport);
2256
2257         seq_printf(seq,
2258                    "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2259                    "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n",
2260                    i,
2261                    src->s6_addr32[0], src->s6_addr32[1],
2262                    src->s6_addr32[2], src->s6_addr32[3], srcp,
2263                    dest->s6_addr32[0], dest->s6_addr32[1],
2264                    dest->s6_addr32[2], dest->s6_addr32[3], destp,
2265                    tw->tw_substate, 0, 0,
2266                    3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
2267                    refcount_read(&tw->tw_refcnt), tw);
2268 }
2269
2270 static int tcp6_seq_show(struct seq_file *seq, void *v)
2271 {
2272         struct tcp_iter_state *st;
2273         struct sock *sk = v;
2274
2275         if (v == SEQ_START_TOKEN) {
2276                 seq_puts(seq,
2277                          "  sl  "
2278                          "local_address                         "
2279                          "remote_address                        "
2280                          "st tx_queue rx_queue tr tm->when retrnsmt"
2281                          "   uid  timeout inode\n");
2282                 goto out;
2283         }
2284         st = seq->private;
2285
2286         if (sk->sk_state == TCP_TIME_WAIT)
2287                 get_timewait6_sock(seq, v, st->num);
2288         else if (sk->sk_state == TCP_NEW_SYN_RECV)
2289                 get_openreq6(seq, v, st->num);
2290         else
2291                 get_tcp6_sock(seq, v, st->num);
2292 out:
2293         return 0;
2294 }
2295
2296 static const struct seq_operations tcp6_seq_ops = {
2297         .show           = tcp6_seq_show,
2298         .start          = tcp_seq_start,
2299         .next           = tcp_seq_next,
2300         .stop           = tcp_seq_stop,
2301 };
2302
2303 static struct tcp_seq_afinfo tcp6_seq_afinfo = {
2304         .family         = AF_INET6,
2305 };
2306
2307 int __net_init tcp6_proc_init(struct net *net)
2308 {
2309         if (!proc_create_net_data("tcp6", 0444, net->proc_net, &tcp6_seq_ops,
2310                         sizeof(struct tcp_iter_state), &tcp6_seq_afinfo))
2311                 return -ENOMEM;
2312         return 0;
2313 }
2314
2315 void tcp6_proc_exit(struct net *net)
2316 {
2317         remove_proc_entry("tcp6", net->proc_net);
2318 }
2319 #endif
2320
2321 struct proto tcpv6_prot = {
2322         .name                   = "TCPv6",
2323         .owner                  = THIS_MODULE,
2324         .close                  = tcp_close,
2325         .pre_connect            = tcp_v6_pre_connect,
2326         .connect                = tcp_v6_connect,
2327         .disconnect             = tcp_disconnect,
2328         .accept                 = inet_csk_accept,
2329         .ioctl                  = tcp_ioctl,
2330         .init                   = tcp_v6_init_sock,
2331         .destroy                = tcp_v4_destroy_sock,
2332         .shutdown               = tcp_shutdown,
2333         .setsockopt             = tcp_setsockopt,
2334         .getsockopt             = tcp_getsockopt,
2335         .bpf_bypass_getsockopt  = tcp_bpf_bypass_getsockopt,
2336         .keepalive              = tcp_set_keepalive,
2337         .recvmsg                = tcp_recvmsg,
2338         .sendmsg                = tcp_sendmsg,
2339         .splice_eof             = tcp_splice_eof,
2340         .backlog_rcv            = tcp_v6_do_rcv,
2341         .release_cb             = tcp_release_cb,
2342         .hash                   = inet6_hash,
2343         .unhash                 = inet_unhash,
2344         .get_port               = inet_csk_get_port,
2345         .put_port               = inet_put_port,
2346 #ifdef CONFIG_BPF_SYSCALL
2347         .psock_update_sk_prot   = tcp_bpf_update_proto,
2348 #endif
2349         .enter_memory_pressure  = tcp_enter_memory_pressure,
2350         .leave_memory_pressure  = tcp_leave_memory_pressure,
2351         .stream_memory_free     = tcp_stream_memory_free,
2352         .sockets_allocated      = &tcp_sockets_allocated,
2353
2354         .memory_allocated       = &tcp_memory_allocated,
2355         .per_cpu_fw_alloc       = &tcp_memory_per_cpu_fw_alloc,
2356
2357         .memory_pressure        = &tcp_memory_pressure,
2358         .orphan_count           = &tcp_orphan_count,
2359         .sysctl_mem             = sysctl_tcp_mem,
2360         .sysctl_wmem_offset     = offsetof(struct net, ipv4.sysctl_tcp_wmem),
2361         .sysctl_rmem_offset     = offsetof(struct net, ipv4.sysctl_tcp_rmem),
2362         .max_header             = MAX_TCP_HEADER,
2363         .obj_size               = sizeof(struct tcp6_sock),
2364         .ipv6_pinfo_offset = offsetof(struct tcp6_sock, inet6),
2365         .slab_flags             = SLAB_TYPESAFE_BY_RCU,
2366         .twsk_prot              = &tcp6_timewait_sock_ops,
2367         .rsk_prot               = &tcp6_request_sock_ops,
2368         .h.hashinfo             = NULL,
2369         .no_autobind            = true,
2370         .diag_destroy           = tcp_abort,
2371 };
2372 EXPORT_SYMBOL_GPL(tcpv6_prot);
2373
2374
2375 static struct inet_protosw tcpv6_protosw = {
2376         .type           =       SOCK_STREAM,
2377         .protocol       =       IPPROTO_TCP,
2378         .prot           =       &tcpv6_prot,
2379         .ops            =       &inet6_stream_ops,
2380         .flags          =       INET_PROTOSW_PERMANENT |
2381                                 INET_PROTOSW_ICSK,
2382 };
2383
2384 static int __net_init tcpv6_net_init(struct net *net)
2385 {
2386         return inet_ctl_sock_create(&net->ipv6.tcp_sk, PF_INET6,
2387                                     SOCK_RAW, IPPROTO_TCP, net);
2388 }
2389
2390 static void __net_exit tcpv6_net_exit(struct net *net)
2391 {
2392         inet_ctl_sock_destroy(net->ipv6.tcp_sk);
2393 }
2394
2395 static struct pernet_operations tcpv6_net_ops = {
2396         .init       = tcpv6_net_init,
2397         .exit       = tcpv6_net_exit,
2398 };
2399
2400 int __init tcpv6_init(void)
2401 {
2402         int ret;
2403
2404         net_hotdata.tcpv6_protocol = (struct inet6_protocol) {
2405                 .handler     = tcp_v6_rcv,
2406                 .err_handler = tcp_v6_err,
2407                 .flags       = INET6_PROTO_NOPOLICY | INET6_PROTO_FINAL,
2408         };
2409         ret = inet6_add_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
2410         if (ret)
2411                 goto out;
2412
2413         /* register inet6 protocol */
2414         ret = inet6_register_protosw(&tcpv6_protosw);
2415         if (ret)
2416                 goto out_tcpv6_protocol;
2417
2418         ret = register_pernet_subsys(&tcpv6_net_ops);
2419         if (ret)
2420                 goto out_tcpv6_protosw;
2421
2422         ret = mptcpv6_init();
2423         if (ret)
2424                 goto out_tcpv6_pernet_subsys;
2425
2426 out:
2427         return ret;
2428
2429 out_tcpv6_pernet_subsys:
2430         unregister_pernet_subsys(&tcpv6_net_ops);
2431 out_tcpv6_protosw:
2432         inet6_unregister_protosw(&tcpv6_protosw);
2433 out_tcpv6_protocol:
2434         inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
2435         goto out;
2436 }
2437
2438 void tcpv6_exit(void)
2439 {
2440         unregister_pernet_subsys(&tcpv6_net_ops);
2441         inet6_unregister_protosw(&tcpv6_protosw);
2442         inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
2443 }