net/ipv6/tcp_ipv6.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  *      TCP over IPv6
   4  *      Linux INET6 implementation
   5  *
   6  *      Authors:
   7  *      Pedro Roque             <[email protected]>
   8  *
   9  *      Based on:
  10  *      linux/net/ipv4/tcp.c
  11  *      linux/net/ipv4/tcp_input.c
  12  *      linux/net/ipv4/tcp_output.c
  13  *
  14  *      Fixes:
  15  *      Hideaki YOSHIFUJI       :       sin6_scope_id support
  16  *      YOSHIFUJI Hideaki @USAGI and:   Support IPV6_V6ONLY socket option, which
  17  *      Alexey Kuznetsov                allow both IPv4 and IPv6 sockets to bind
  18  *                                      a single port at the same time.
  19  *      YOSHIFUJI Hideaki @USAGI:       convert /proc/net/tcp6 to seq_file.
  20  */
  21
  22 #include <linux/bottom_half.h>
  23 #include <linux/module.h>
  24 #include <linux/errno.h>
  25 #include <linux/types.h>
  26 #include <linux/socket.h>
  27 #include <linux/sockios.h>
  28 #include <linux/net.h>
  29 #include <linux/jiffies.h>
  30 #include <linux/in.h>
  31 #include <linux/in6.h>
  32 #include <linux/netdevice.h>
  33 #include <linux/init.h>
  34 #include <linux/jhash.h>
  35 #include <linux/ipsec.h>
  36 #include <linux/times.h>
  37 #include <linux/slab.h>
  38 #include <linux/uaccess.h>
  39 #include <linux/ipv6.h>
  40 #include <linux/icmpv6.h>
  41 #include <linux/random.h>
  42 #include <linux/indirect_call_wrapper.h>
  43
  44 #include <net/tcp.h>
  45 #include <net/ndisc.h>
  46 #include <net/inet6_hashtables.h>
  47 #include <net/inet6_connection_sock.h>
  48 #include <net/ipv6.h>
  49 #include <net/transp_v6.h>
  50 #include <net/addrconf.h>
  51 #include <net/ip6_route.h>
  52 #include <net/ip6_checksum.h>
  53 #include <net/inet_ecn.h>
  54 #include <net/protocol.h>
  55 #include <net/xfrm.h>
  56 #include <net/snmp.h>
  57 #include <net/dsfield.h>
  58 #include <net/timewait_sock.h>
  59 #include <net/inet_common.h>
  60 #include <net/secure_seq.h>
  61 #include <net/hotdata.h>
  62 #include <net/busy_poll.h>
  63 #include <net/rstreason.h>
  64
  65 #include <linux/proc_fs.h>
  66 #include <linux/seq_file.h>
  67
  68 #include <crypto/hash.h>
  69 #include <linux/scatterlist.h>
  70
  71 #include <trace/events/tcp.h>
  72
  73 static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb,
  74                               enum sk_rst_reason reason);
  75 static void     tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
  76                                       struct request_sock *req);
  77
  78 INDIRECT_CALLABLE_SCOPE int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb);
  79
  80 static const struct inet_connection_sock_af_ops ipv6_mapped;
  81 const struct inet_connection_sock_af_ops ipv6_specific;
  82 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
  83 static const struct tcp_sock_af_ops tcp_sock_ipv6_specific;
  84 static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific;
  85 #endif
  86
  87 /* Helper returning the inet6 address from a given tcp socket.
  88  * It can be used in TCP stack instead of inet6_sk(sk).
  89  * This avoids a dereference and allow compiler optimizations.
  90  * It is a specialized version of inet6_sk_generic().
  91  */
  92 #define tcp_inet6_sk(sk) (&container_of_const(tcp_sk(sk), \
  93                                               struct tcp6_sock, tcp)->inet6)
  94
  95 static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
  96 {
  97         struct dst_entry *dst = skb_dst(skb);
  98
  99         if (dst && dst_hold_safe(dst)) {
 100                 rcu_assign_pointer(sk->sk_rx_dst, dst);
 101                 sk->sk_rx_dst_ifindex = skb->skb_iif;
 102                 sk->sk_rx_dst_cookie = rt6_get_cookie(dst_rt6_info(dst));
 103         }
 104 }
 105
 106 static u32 tcp_v6_init_seq(const struct sk_buff *skb)
 107 {
 108         return secure_tcpv6_seq(ipv6_hdr(skb)->daddr.s6_addr32,
 109                                 ipv6_hdr(skb)->saddr.s6_addr32,
 110                                 tcp_hdr(skb)->dest,
 111                                 tcp_hdr(skb)->source);
 112 }
 113
 114 static u32 tcp_v6_init_ts_off(const struct net *net, const struct sk_buff *skb)
 115 {
 116         return secure_tcpv6_ts_off(net, ipv6_hdr(skb)->daddr.s6_addr32,
 117                                    ipv6_hdr(skb)->saddr.s6_addr32);
 118 }
 119
 120 static int tcp_v6_pre_connect(struct sock *sk, struct sockaddr *uaddr,
 121                               int addr_len)
 122 {
 123         /* This check is replicated from tcp_v6_connect() and intended to
 124          * prevent BPF program called below from accessing bytes that are out
 125          * of the bound specified by user in addr_len.
 126          */
 127         if (addr_len < SIN6_LEN_RFC2133)
 128                 return -EINVAL;
 129
 130         sock_owned_by_me(sk);
 131
 132         return BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr, &addr_len);
 133 }
 134
 135 static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
 136                           int addr_len)
 137 {
 138         struct sockaddr_in6 *usin = (struct sockaddr_in6 *) uaddr;
 139         struct inet_connection_sock *icsk = inet_csk(sk);
 140         struct in6_addr *saddr = NULL, *final_p, final;
 141         struct inet_timewait_death_row *tcp_death_row;
 142         struct ipv6_pinfo *np = tcp_inet6_sk(sk);
 143         struct inet_sock *inet = inet_sk(sk);
 144         struct tcp_sock *tp = tcp_sk(sk);
 145         struct net *net = sock_net(sk);
 146         struct ipv6_txoptions *opt;
 147         struct dst_entry *dst;
 148         struct flowi6 fl6;
 149         int addr_type;
 150         int err;
 151
 152         if (addr_len < SIN6_LEN_RFC2133)
 153                 return -EINVAL;
 154
 155         if (usin->sin6_family != AF_INET6)
 156                 return -EAFNOSUPPORT;
 157
 158         memset(&fl6, 0, sizeof(fl6));
 159
 160         if (inet6_test_bit(SNDFLOW, sk)) {
 161                 fl6.flowlabel = usin->sin6_flowinfo&IPV6_FLOWINFO_MASK;
 162                 IP6_ECN_flow_init(fl6.flowlabel);
 163                 if (fl6.flowlabel&IPV6_FLOWLABEL_MASK) {
 164                         struct ip6_flowlabel *flowlabel;
 165                         flowlabel = fl6_sock_lookup(sk, fl6.flowlabel);
 166                         if (IS_ERR(flowlabel))
 167                                 return -EINVAL;
 168                         fl6_sock_release(flowlabel);
 169                 }
 170         }
 171
 172         /*
 173          *      connect() to INADDR_ANY means loopback (BSD'ism).
 174          */
 175
 176         if (ipv6_addr_any(&usin->sin6_addr)) {
 177                 if (ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr))
 178                         ipv6_addr_set_v4mapped(htonl(INADDR_LOOPBACK),
 179                                                &usin->sin6_addr);
 180                 else
 181                         usin->sin6_addr = in6addr_loopback;
 182         }
 183
 184         addr_type = ipv6_addr_type(&usin->sin6_addr);
 185
 186         if (addr_type & IPV6_ADDR_MULTICAST)
 187                 return -ENETUNREACH;
 188
 189         if (addr_type&IPV6_ADDR_LINKLOCAL) {
 190                 if (addr_len >= sizeof(struct sockaddr_in6) &&
 191                     usin->sin6_scope_id) {
 192                         /* If interface is set while binding, indices
 193                          * must coincide.
 194                          */
 195                         if (!sk_dev_equal_l3scope(sk, usin->sin6_scope_id))
 196                                 return -EINVAL;
 197
 198                         sk->sk_bound_dev_if = usin->sin6_scope_id;
 199                 }
 200
 201                 /* Connect to link-local address requires an interface */
 202                 if (!sk->sk_bound_dev_if)
 203                         return -EINVAL;
 204         }
 205
 206         if (tp->rx_opt.ts_recent_stamp &&
 207             !ipv6_addr_equal(&sk->sk_v6_daddr, &usin->sin6_addr)) {
 208                 tp->rx_opt.ts_recent = 0;
 209                 tp->rx_opt.ts_recent_stamp = 0;
 210                 WRITE_ONCE(tp->write_seq, 0);
 211         }
 212
 213         sk->sk_v6_daddr = usin->sin6_addr;
 214         np->flow_label = fl6.flowlabel;
 215
 216         /*
 217          *      TCP over IPv4
 218          */
 219
 220         if (addr_type & IPV6_ADDR_MAPPED) {
 221                 u32 exthdrlen = icsk->icsk_ext_hdr_len;
 222                 struct sockaddr_in sin;
 223
 224                 if (ipv6_only_sock(sk))
 225                         return -ENETUNREACH;
 226
 227                 sin.sin_family = AF_INET;
 228                 sin.sin_port = usin->sin6_port;
 229                 sin.sin_addr.s_addr = usin->sin6_addr.s6_addr32[3];
 230
 231                 /* Paired with READ_ONCE() in tcp_(get|set)sockopt() */
 232                 WRITE_ONCE(icsk->icsk_af_ops, &ipv6_mapped);
 233                 if (sk_is_mptcp(sk))
 234                         mptcpv6_handle_mapped(sk, true);
 235                 sk->sk_backlog_rcv = tcp_v4_do_rcv;
 236 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
 237                 tp->af_specific = &tcp_sock_ipv6_mapped_specific;
 238 #endif
 239
 240                 err = tcp_v4_connect(sk, (struct sockaddr *)&sin, sizeof(sin));
 241
 242                 if (err) {
 243                         icsk->icsk_ext_hdr_len = exthdrlen;
 244                         /* Paired with READ_ONCE() in tcp_(get|set)sockopt() */
 245                         WRITE_ONCE(icsk->icsk_af_ops, &ipv6_specific);
 246                         if (sk_is_mptcp(sk))
 247                                 mptcpv6_handle_mapped(sk, false);
 248                         sk->sk_backlog_rcv = tcp_v6_do_rcv;
 249 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
 250                         tp->af_specific = &tcp_sock_ipv6_specific;
 251 #endif
 252                         goto failure;
 253                 }
 254                 np->saddr = sk->sk_v6_rcv_saddr;
 255
 256                 return err;
 257         }
 258
 259         if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr))
 260                 saddr = &sk->sk_v6_rcv_saddr;
 261
 262         fl6.flowi6_proto = IPPROTO_TCP;
 263         fl6.daddr = sk->sk_v6_daddr;
 264         fl6.saddr = saddr ? *saddr : np->saddr;
 265         fl6.flowlabel = ip6_make_flowinfo(np->tclass, np->flow_label);
 266         fl6.flowi6_oif = sk->sk_bound_dev_if;
 267         fl6.flowi6_mark = sk->sk_mark;
 268         fl6.fl6_dport = usin->sin6_port;
 269         fl6.fl6_sport = inet->inet_sport;
 270         fl6.flowi6_uid = sk->sk_uid;
 271
 272         opt = rcu_dereference_protected(np->opt, lockdep_sock_is_held(sk));
 273         final_p = fl6_update_dst(&fl6, opt, &final);
 274
 275         security_sk_classify_flow(sk, flowi6_to_flowi_common(&fl6));
 276
 277         dst = ip6_dst_lookup_flow(net, sk, &fl6, final_p);
 278         if (IS_ERR(dst)) {
 279                 err = PTR_ERR(dst);
 280                 goto failure;
 281         }
 282
 283         tp->tcp_usec_ts = dst_tcp_usec_ts(dst);
 284         tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;
 285
 286         if (!saddr) {
 287                 saddr = &fl6.saddr;
 288
 289                 err = inet_bhash2_update_saddr(sk, saddr, AF_INET6);
 290                 if (err)
 291                         goto failure;
 292         }
 293
 294         /* set the source address */
 295         np->saddr = *saddr;
 296         inet->inet_rcv_saddr = LOOPBACK4_IPV6;
 297
 298         sk->sk_gso_type = SKB_GSO_TCPV6;
 299         ip6_dst_store(sk, dst, NULL, NULL);
 300
 301         icsk->icsk_ext_hdr_len = 0;
 302         if (opt)
 303                 icsk->icsk_ext_hdr_len = opt->opt_flen +
 304                                          opt->opt_nflen;
 305
 306         tp->rx_opt.mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) - sizeof(struct ipv6hdr);
 307
 308         inet->inet_dport = usin->sin6_port;
 309
 310         tcp_set_state(sk, TCP_SYN_SENT);
 311         err = inet6_hash_connect(tcp_death_row, sk);
 312         if (err)
 313                 goto late_failure;
 314
 315         sk_set_txhash(sk);
 316
 317         if (likely(!tp->repair)) {
 318                 if (!tp->write_seq)
 319                         WRITE_ONCE(tp->write_seq,
 320                                    secure_tcpv6_seq(np->saddr.s6_addr32,
 321                                                     sk->sk_v6_daddr.s6_addr32,
 322                                                     inet->inet_sport,
 323                                                     inet->inet_dport));
 324                 tp->tsoffset = secure_tcpv6_ts_off(net, np->saddr.s6_addr32,
 325                                                    sk->sk_v6_daddr.s6_addr32);
 326         }
 327
 328         if (tcp_fastopen_defer_connect(sk, &err))
 329                 return err;
 330         if (err)
 331                 goto late_failure;
 332
 333         err = tcp_connect(sk);
 334         if (err)
 335                 goto late_failure;
 336
 337         return 0;
 338
 339 late_failure:
 340         tcp_set_state(sk, TCP_CLOSE);
 341         inet_bhash2_reset_saddr(sk);
 342 failure:
 343         inet->inet_dport = 0;
 344         sk->sk_route_caps = 0;
 345         return err;
 346 }
 347
 348 static void tcp_v6_mtu_reduced(struct sock *sk)
 349 {
 350         struct dst_entry *dst;
 351         u32 mtu;
 352
 353         if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
 354                 return;
 355
 356         mtu = READ_ONCE(tcp_sk(sk)->mtu_info);
 357
 358         /* Drop requests trying to increase our current mss.
 359          * Check done in __ip6_rt_update_pmtu() is too late.
 360          */
 361         if (tcp_mtu_to_mss(sk, mtu) >= tcp_sk(sk)->mss_cache)
 362                 return;
 363
 364         dst = inet6_csk_update_pmtu(sk, mtu);
 365         if (!dst)
 366                 return;
 367
 368         if (inet_csk(sk)->icsk_pmtu_cookie > dst_mtu(dst)) {
 369                 tcp_sync_mss(sk, dst_mtu(dst));
 370                 tcp_simple_retransmit(sk);
 371         }
 372 }
 373
 374 static int tcp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
 375                 u8 type, u8 code, int offset, __be32 info)
 376 {
 377         const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data;
 378         const struct tcphdr *th = (struct tcphdr *)(skb->data+offset);
 379         struct net *net = dev_net(skb->dev);
 380         struct request_sock *fastopen;
 381         struct ipv6_pinfo *np;
 382         struct tcp_sock *tp;
 383         __u32 seq, snd_una;
 384         struct sock *sk;
 385         bool fatal;
 386         int err;
 387
 388         sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
 389                                         &hdr->daddr, th->dest,
 390                                         &hdr->saddr, ntohs(th->source),
 391                                         skb->dev->ifindex, inet6_sdif(skb));
 392
 393         if (!sk) {
 394                 __ICMP6_INC_STATS(net, __in6_dev_get(skb->dev),
 395                                   ICMP6_MIB_INERRORS);
 396                 return -ENOENT;
 397         }
 398
 399         if (sk->sk_state == TCP_TIME_WAIT) {
 400                 /* To increase the counter of ignored icmps for TCP-AO */
 401                 tcp_ao_ignore_icmp(sk, AF_INET6, type, code);
 402                 inet_twsk_put(inet_twsk(sk));
 403                 return 0;
 404         }
 405         seq = ntohl(th->seq);
 406         fatal = icmpv6_err_convert(type, code, &err);
 407         if (sk->sk_state == TCP_NEW_SYN_RECV) {
 408                 tcp_req_err(sk, seq, fatal);
 409                 return 0;
 410         }
 411
 412         if (tcp_ao_ignore_icmp(sk, AF_INET6, type, code)) {
 413                 sock_put(sk);
 414                 return 0;
 415         }
 416
 417         bh_lock_sock(sk);
 418         if (sock_owned_by_user(sk) && type != ICMPV6_PKT_TOOBIG)
 419                 __NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS);
 420
 421         if (sk->sk_state == TCP_CLOSE)
 422                 goto out;
 423
 424         if (static_branch_unlikely(&ip6_min_hopcount)) {
 425                 /* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */
 426                 if (ipv6_hdr(skb)->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount)) {
 427                         __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
 428                         goto out;
 429                 }
 430         }
 431
 432         tp = tcp_sk(sk);
 433         /* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
 434         fastopen = rcu_dereference(tp->fastopen_rsk);
 435         snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;
 436         if (sk->sk_state != TCP_LISTEN &&
 437             !between(seq, snd_una, tp->snd_nxt)) {
 438                 __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
 439                 goto out;
 440         }
 441
 442         np = tcp_inet6_sk(sk);
 443
 444         if (type == NDISC_REDIRECT) {
 445                 if (!sock_owned_by_user(sk)) {
 446                         struct dst_entry *dst = __sk_dst_check(sk, np->dst_cookie);
 447
 448                         if (dst)
 449                                 dst->ops->redirect(dst, sk, skb);
 450                 }
 451                 goto out;
 452         }
 453
 454         if (type == ICMPV6_PKT_TOOBIG) {
 455                 u32 mtu = ntohl(info);
 456
 457                 /* We are not interested in TCP_LISTEN and open_requests
 458                  * (SYN-ACKs send out by Linux are always <576bytes so
 459                  * they should go through unfragmented).
 460                  */
 461                 if (sk->sk_state == TCP_LISTEN)
 462                         goto out;
 463
 464                 if (!ip6_sk_accept_pmtu(sk))
 465                         goto out;
 466
 467                 if (mtu < IPV6_MIN_MTU)
 468                         goto out;
 469
 470                 WRITE_ONCE(tp->mtu_info, mtu);
 471
 472                 if (!sock_owned_by_user(sk))
 473                         tcp_v6_mtu_reduced(sk);
 474                 else if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED,
 475                                            &sk->sk_tsq_flags))
 476                         sock_hold(sk);
 477                 goto out;
 478         }
 479
 480
 481         /* Might be for an request_sock */
 482         switch (sk->sk_state) {
 483         case TCP_SYN_SENT:
 484         case TCP_SYN_RECV:
 485                 /* Only in fast or simultaneous open. If a fast open socket is
 486                  * already accepted it is treated as a connected one below.
 487                  */
 488                 if (fastopen && !fastopen->sk)
 489                         break;
 490
 491                 ipv6_icmp_error(sk, skb, err, th->dest, ntohl(info), (u8 *)th);
 492
 493                 if (!sock_owned_by_user(sk))
 494                         tcp_done_with_error(sk, err);
 495                 else
 496                         WRITE_ONCE(sk->sk_err_soft, err);
 497                 goto out;
 498         case TCP_LISTEN:
 499                 break;
 500         default:
 501                 /* check if this ICMP message allows revert of backoff.
 502                  * (see RFC 6069)
 503                  */
 504                 if (!fastopen && type == ICMPV6_DEST_UNREACH &&
 505                     code == ICMPV6_NOROUTE)
 506                         tcp_ld_RTO_revert(sk, seq);
 507         }
 508
 509         if (!sock_owned_by_user(sk) && inet6_test_bit(RECVERR6, sk)) {
 510                 WRITE_ONCE(sk->sk_err, err);
 511                 sk_error_report(sk);
 512         } else {
 513                 WRITE_ONCE(sk->sk_err_soft, err);
 514         }
 515 out:
 516         bh_unlock_sock(sk);
 517         sock_put(sk);
 518         return 0;
 519 }
 520
 521
 522 static int tcp_v6_send_synack(const struct sock *sk, struct dst_entry *dst,
 523                               struct flowi *fl,
 524                               struct request_sock *req,
 525                               struct tcp_fastopen_cookie *foc,
 526                               enum tcp_synack_type synack_type,
 527                               struct sk_buff *syn_skb)
 528 {
 529         struct inet_request_sock *ireq = inet_rsk(req);
 530         const struct ipv6_pinfo *np = tcp_inet6_sk(sk);
 531         struct ipv6_txoptions *opt;
 532         struct flowi6 *fl6 = &fl->u.ip6;
 533         struct sk_buff *skb;
 534         int err = -ENOMEM;
 535         u8 tclass;
 536
 537         /* First, grab a route. */
 538         if (!dst && (dst = inet6_csk_route_req(sk, fl6, req,
 539                                                IPPROTO_TCP)) == NULL)
 540                 goto done;
 541
 542         skb = tcp_make_synack(sk, dst, req, foc, synack_type, syn_skb);
 543
 544         if (skb) {
 545                 __tcp_v6_send_check(skb, &ireq->ir_v6_loc_addr,
 546                                     &ireq->ir_v6_rmt_addr);
 547
 548                 fl6->daddr = ireq->ir_v6_rmt_addr;
 549                 if (inet6_test_bit(REPFLOW, sk) && ireq->pktopts)
 550                         fl6->flowlabel = ip6_flowlabel(ipv6_hdr(ireq->pktopts));
 551
 552                 tclass = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos) ?
 553                                 (tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) |
 554                                 (np->tclass & INET_ECN_MASK) :
 555                                 np->tclass;
 556
 557                 if (!INET_ECN_is_capable(tclass) &&
 558                     tcp_bpf_ca_needs_ecn((struct sock *)req))
 559                         tclass |= INET_ECN_ECT_0;
 560
 561                 rcu_read_lock();
 562                 opt = ireq->ipv6_opt;
 563                 if (!opt)
 564                         opt = rcu_dereference(np->opt);
 565                 err = ip6_xmit(sk, skb, fl6, skb->mark ? : READ_ONCE(sk->sk_mark),
 566                                opt, tclass, READ_ONCE(sk->sk_priority));
 567                 rcu_read_unlock();
 568                 err = net_xmit_eval(err);
 569         }
 570
 571 done:
 572         return err;
 573 }
 574
 575
 576 static void tcp_v6_reqsk_destructor(struct request_sock *req)
 577 {
 578         kfree(inet_rsk(req)->ipv6_opt);
 579         consume_skb(inet_rsk(req)->pktopts);
 580 }
 581
 582 #ifdef CONFIG_TCP_MD5SIG
 583 static struct tcp_md5sig_key *tcp_v6_md5_do_lookup(const struct sock *sk,
 584                                                    const struct in6_addr *addr,
 585                                                    int l3index)
 586 {
 587         return tcp_md5_do_lookup(sk, l3index,
 588                                  (union tcp_md5_addr *)addr, AF_INET6);
 589 }
 590
 591 static struct tcp_md5sig_key *tcp_v6_md5_lookup(const struct sock *sk,
 592                                                 const struct sock *addr_sk)
 593 {
 594         int l3index;
 595
 596         l3index = l3mdev_master_ifindex_by_index(sock_net(sk),
 597                                                  addr_sk->sk_bound_dev_if);
 598         return tcp_v6_md5_do_lookup(sk, &addr_sk->sk_v6_daddr,
 599                                     l3index);
 600 }
 601
 602 static int tcp_v6_parse_md5_keys(struct sock *sk, int optname,
 603                                  sockptr_t optval, int optlen)
 604 {
 605         struct tcp_md5sig cmd;
 606         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&cmd.tcpm_addr;
 607         union tcp_ao_addr *addr;
 608         int l3index = 0;
 609         u8 prefixlen;
 610         bool l3flag;
 611         u8 flags;
 612
 613         if (optlen < sizeof(cmd))
 614                 return -EINVAL;
 615
 616         if (copy_from_sockptr(&cmd, optval, sizeof(cmd)))
 617                 return -EFAULT;
 618
 619         if (sin6->sin6_family != AF_INET6)
 620                 return -EINVAL;
 621
 622         flags = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
 623         l3flag = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
 624
 625         if (optname == TCP_MD5SIG_EXT &&
 626             cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) {
 627                 prefixlen = cmd.tcpm_prefixlen;
 628                 if (prefixlen > 128 || (ipv6_addr_v4mapped(&sin6->sin6_addr) &&
 629                                         prefixlen > 32))
 630                         return -EINVAL;
 631         } else {
 632                 prefixlen = ipv6_addr_v4mapped(&sin6->sin6_addr) ? 32 : 128;
 633         }
 634
 635         if (optname == TCP_MD5SIG_EXT && cmd.tcpm_ifindex &&
 636             cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) {
 637                 struct net_device *dev;
 638
 639                 rcu_read_lock();
 640                 dev = dev_get_by_index_rcu(sock_net(sk), cmd.tcpm_ifindex);
 641                 if (dev && netif_is_l3_master(dev))
 642                         l3index = dev->ifindex;
 643                 rcu_read_unlock();
 644
 645                 /* ok to reference set/not set outside of rcu;
 646                  * right now device MUST be an L3 master
 647                  */
 648                 if (!dev || !l3index)
 649                         return -EINVAL;
 650         }
 651
 652         if (!cmd.tcpm_keylen) {
 653                 if (ipv6_addr_v4mapped(&sin6->sin6_addr))
 654                         return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3],
 655                                               AF_INET, prefixlen,
 656                                               l3index, flags);
 657                 return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr,
 658                                       AF_INET6, prefixlen, l3index, flags);
 659         }
 660
 661         if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
 662                 return -EINVAL;
 663
 664         if (ipv6_addr_v4mapped(&sin6->sin6_addr)) {
 665                 addr = (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3];
 666
 667                 /* Don't allow keys for peers that have a matching TCP-AO key.
 668                  * See the comment in tcp_ao_add_cmd()
 669                  */
 670                 if (tcp_ao_required(sk, addr, AF_INET,
 671                                     l3flag ? l3index : -1, false))
 672                         return -EKEYREJECTED;
 673                 return tcp_md5_do_add(sk, addr,
 674                                       AF_INET, prefixlen, l3index, flags,
 675                                       cmd.tcpm_key, cmd.tcpm_keylen);
 676         }
 677
 678         addr = (union tcp_md5_addr *)&sin6->sin6_addr;
 679
 680         /* Don't allow keys for peers that have a matching TCP-AO key.
 681          * See the comment in tcp_ao_add_cmd()
 682          */
 683         if (tcp_ao_required(sk, addr, AF_INET6, l3flag ? l3index : -1, false))
 684                 return -EKEYREJECTED;
 685
 686         return tcp_md5_do_add(sk, addr, AF_INET6, prefixlen, l3index, flags,
 687                               cmd.tcpm_key, cmd.tcpm_keylen);
 688 }
 689
 690 static int tcp_v6_md5_hash_headers(struct tcp_sigpool *hp,
 691                                    const struct in6_addr *daddr,
 692                                    const struct in6_addr *saddr,
 693                                    const struct tcphdr *th, int nbytes)
 694 {
 695         struct tcp6_pseudohdr *bp;
 696         struct scatterlist sg;
 697         struct tcphdr *_th;
 698
 699         bp = hp->scratch;
 700         /* 1. TCP pseudo-header (RFC2460) */
 701         bp->saddr = *saddr;
 702         bp->daddr = *daddr;
 703         bp->protocol = cpu_to_be32(IPPROTO_TCP);
 704         bp->len = cpu_to_be32(nbytes);
 705
 706         _th = (struct tcphdr *)(bp + 1);
 707         memcpy(_th, th, sizeof(*th));
 708         _th->check = 0;
 709
 710         sg_init_one(&sg, bp, sizeof(*bp) + sizeof(*th));
 711         ahash_request_set_crypt(hp->req, &sg, NULL,
 712                                 sizeof(*bp) + sizeof(*th));
 713         return crypto_ahash_update(hp->req);
 714 }
 715
 716 static int tcp_v6_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
 717                                const struct in6_addr *daddr, struct in6_addr *saddr,
 718                                const struct tcphdr *th)
 719 {
 720         struct tcp_sigpool hp;
 721
 722         if (tcp_sigpool_start(tcp_md5_sigpool_id, &hp))
 723                 goto clear_hash_nostart;
 724
 725         if (crypto_ahash_init(hp.req))
 726                 goto clear_hash;
 727         if (tcp_v6_md5_hash_headers(&hp, daddr, saddr, th, th->doff << 2))
 728                 goto clear_hash;
 729         if (tcp_md5_hash_key(&hp, key))
 730                 goto clear_hash;
 731         ahash_request_set_crypt(hp.req, NULL, md5_hash, 0);
 732         if (crypto_ahash_final(hp.req))
 733                 goto clear_hash;
 734
 735         tcp_sigpool_end(&hp);
 736         return 0;
 737
 738 clear_hash:
 739         tcp_sigpool_end(&hp);
 740 clear_hash_nostart:
 741         memset(md5_hash, 0, 16);
 742         return 1;
 743 }
 744
 745 static int tcp_v6_md5_hash_skb(char *md5_hash,
 746                                const struct tcp_md5sig_key *key,
 747                                const struct sock *sk,
 748                                const struct sk_buff *skb)
 749 {
 750         const struct tcphdr *th = tcp_hdr(skb);
 751         const struct in6_addr *saddr, *daddr;
 752         struct tcp_sigpool hp;
 753
 754         if (sk) { /* valid for establish/request sockets */
 755                 saddr = &sk->sk_v6_rcv_saddr;
 756                 daddr = &sk->sk_v6_daddr;
 757         } else {
 758                 const struct ipv6hdr *ip6h = ipv6_hdr(skb);
 759                 saddr = &ip6h->saddr;
 760                 daddr = &ip6h->daddr;
 761         }
 762
 763         if (tcp_sigpool_start(tcp_md5_sigpool_id, &hp))
 764                 goto clear_hash_nostart;
 765
 766         if (crypto_ahash_init(hp.req))
 767                 goto clear_hash;
 768
 769         if (tcp_v6_md5_hash_headers(&hp, daddr, saddr, th, skb->len))
 770                 goto clear_hash;
 771         if (tcp_sigpool_hash_skb_data(&hp, skb, th->doff << 2))
 772                 goto clear_hash;
 773         if (tcp_md5_hash_key(&hp, key))
 774                 goto clear_hash;
 775         ahash_request_set_crypt(hp.req, NULL, md5_hash, 0);
 776         if (crypto_ahash_final(hp.req))
 777                 goto clear_hash;
 778
 779         tcp_sigpool_end(&hp);
 780         return 0;
 781
 782 clear_hash:
 783         tcp_sigpool_end(&hp);
 784 clear_hash_nostart:
 785         memset(md5_hash, 0, 16);
 786         return 1;
 787 }
 788 #endif
 789
 790 static void tcp_v6_init_req(struct request_sock *req,
 791                             const struct sock *sk_listener,
 792                             struct sk_buff *skb,
 793                             u32 tw_isn)
 794 {
 795         bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags);
 796         struct inet_request_sock *ireq = inet_rsk(req);
 797         const struct ipv6_pinfo *np = tcp_inet6_sk(sk_listener);
 798
 799         ireq->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr;
 800         ireq->ir_v6_loc_addr = ipv6_hdr(skb)->daddr;
 801
 802         /* So that link locals have meaning */
 803         if ((!sk_listener->sk_bound_dev_if || l3_slave) &&
 804             ipv6_addr_type(&ireq->ir_v6_rmt_addr) & IPV6_ADDR_LINKLOCAL)
 805                 ireq->ir_iif = tcp_v6_iif(skb);
 806
 807         if (!tw_isn &&
 808             (ipv6_opt_accepted(sk_listener, skb, &TCP_SKB_CB(skb)->header.h6) ||
 809              np->rxopt.bits.rxinfo ||
 810              np->rxopt.bits.rxoinfo || np->rxopt.bits.rxhlim ||
 811              np->rxopt.bits.rxohlim || inet6_test_bit(REPFLOW, sk_listener))) {
 812                 refcount_inc(&skb->users);
 813                 ireq->pktopts = skb;
 814         }
 815 }
 816
 817 static struct dst_entry *tcp_v6_route_req(const struct sock *sk,
 818                                           struct sk_buff *skb,
 819                                           struct flowi *fl,
 820                                           struct request_sock *req,
 821                                           u32 tw_isn)
 822 {
 823         tcp_v6_init_req(req, sk, skb, tw_isn);
 824
 825         if (security_inet_conn_request(sk, skb, req))
 826                 return NULL;
 827
 828         return inet6_csk_route_req(sk, &fl->u.ip6, req, IPPROTO_TCP);
 829 }
 830
 831 struct request_sock_ops tcp6_request_sock_ops __read_mostly = {
 832         .family         =       AF_INET6,
 833         .obj_size       =       sizeof(struct tcp6_request_sock),
 834         .rtx_syn_ack    =       tcp_rtx_synack,
 835         .send_ack       =       tcp_v6_reqsk_send_ack,
 836         .destructor     =       tcp_v6_reqsk_destructor,
 837         .send_reset     =       tcp_v6_send_reset,
 838         .syn_ack_timeout =      tcp_syn_ack_timeout,
 839 };
 840
 841 const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops = {
 842         .mss_clamp      =       IPV6_MIN_MTU - sizeof(struct tcphdr) -
 843                                 sizeof(struct ipv6hdr),
 844 #ifdef CONFIG_TCP_MD5SIG
 845         .req_md5_lookup =       tcp_v6_md5_lookup,
 846         .calc_md5_hash  =       tcp_v6_md5_hash_skb,
 847 #endif
 848 #ifdef CONFIG_TCP_AO
 849         .ao_lookup      =       tcp_v6_ao_lookup_rsk,
 850         .ao_calc_key    =       tcp_v6_ao_calc_key_rsk,
 851         .ao_synack_hash =       tcp_v6_ao_synack_hash,
 852 #endif
 853 #ifdef CONFIG_SYN_COOKIES
 854         .cookie_init_seq =      cookie_v6_init_sequence,
 855 #endif
 856         .route_req      =       tcp_v6_route_req,
 857         .init_seq       =       tcp_v6_init_seq,
 858         .init_ts_off    =       tcp_v6_init_ts_off,
 859         .send_synack    =       tcp_v6_send_synack,
 860 };
 861
 862 static void tcp_v6_send_response(const struct sock *sk, struct sk_buff *skb, u32 seq,
 863                                  u32 ack, u32 win, u32 tsval, u32 tsecr,
 864                                  int oif, int rst, u8 tclass, __be32 label,
 865                                  u32 priority, u32 txhash, struct tcp_key *key)
 866 {
 867         const struct tcphdr *th = tcp_hdr(skb);
 868         struct tcphdr *t1;
 869         struct sk_buff *buff;
 870         struct flowi6 fl6;
 871         struct net *net = sk ? sock_net(sk) : dev_net(skb_dst(skb)->dev);
 872         struct sock *ctl_sk = net->ipv6.tcp_sk;
 873         unsigned int tot_len = sizeof(struct tcphdr);
 874         __be32 mrst = 0, *topt;
 875         struct dst_entry *dst;
 876         __u32 mark = 0;
 877
 878         if (tsecr)
 879                 tot_len += TCPOLEN_TSTAMP_ALIGNED;
 880         if (tcp_key_is_md5(key))
 881                 tot_len += TCPOLEN_MD5SIG_ALIGNED;
 882         if (tcp_key_is_ao(key))
 883                 tot_len += tcp_ao_len_aligned(key->ao_key);
 884
 885 #ifdef CONFIG_MPTCP
 886         if (rst && !tcp_key_is_md5(key)) {
 887                 mrst = mptcp_reset_option(skb);
 888
 889                 if (mrst)
 890                         tot_len += sizeof(__be32);
 891         }
 892 #endif
 893
 894         buff = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC);
 895         if (!buff)
 896                 return;
 897
 898         skb_reserve(buff, MAX_TCP_HEADER);
 899
 900         t1 = skb_push(buff, tot_len);
 901         skb_reset_transport_header(buff);
 902
 903         /* Swap the send and the receive. */
 904         memset(t1, 0, sizeof(*t1));
 905         t1->dest = th->source;
 906         t1->source = th->dest;
 907         t1->doff = tot_len / 4;
 908         t1->seq = htonl(seq);
 909         t1->ack_seq = htonl(ack);
 910         t1->ack = !rst || !th->ack;
 911         t1->rst = rst;
 912         t1->window = htons(win);
 913
 914         topt = (__be32 *)(t1 + 1);
 915
 916         if (tsecr) {
 917                 *topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
 918                                 (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP);
 919                 *topt++ = htonl(tsval);
 920                 *topt++ = htonl(tsecr);
 921         }
 922
 923         if (mrst)
 924                 *topt++ = mrst;
 925
 926 #ifdef CONFIG_TCP_MD5SIG
 927         if (tcp_key_is_md5(key)) {
 928                 *topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
 929                                 (TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG);
 930                 tcp_v6_md5_hash_hdr((__u8 *)topt, key->md5_key,
 931                                     &ipv6_hdr(skb)->saddr,
 932                                     &ipv6_hdr(skb)->daddr, t1);
 933         }
 934 #endif
 935 #ifdef CONFIG_TCP_AO
 936         if (tcp_key_is_ao(key)) {
 937                 *topt++ = htonl((TCPOPT_AO << 24) |
 938                                 (tcp_ao_len(key->ao_key) << 16) |
 939                                 (key->ao_key->sndid << 8) |
 940                                 (key->rcv_next));
 941
 942                 tcp_ao_hash_hdr(AF_INET6, (char *)topt, key->ao_key,
 943                                 key->traffic_key,
 944                                 (union tcp_ao_addr *)&ipv6_hdr(skb)->saddr,
 945                                 (union tcp_ao_addr *)&ipv6_hdr(skb)->daddr,
 946                                 t1, key->sne);
 947         }
 948 #endif
 949
 950         memset(&fl6, 0, sizeof(fl6));
 951         fl6.daddr = ipv6_hdr(skb)->saddr;
 952         fl6.saddr = ipv6_hdr(skb)->daddr;
 953         fl6.flowlabel = label;
 954
 955         buff->ip_summed = CHECKSUM_PARTIAL;
 956
 957         __tcp_v6_send_check(buff, &fl6.saddr, &fl6.daddr);
 958
 959         fl6.flowi6_proto = IPPROTO_TCP;
 960         if (rt6_need_strict(&fl6.daddr) && !oif)
 961                 fl6.flowi6_oif = tcp_v6_iif(skb);
 962         else {
 963                 if (!oif && netif_index_is_l3_master(net, skb->skb_iif))
 964                         oif = skb->skb_iif;
 965
 966                 fl6.flowi6_oif = oif;
 967         }
 968
 969         if (sk) {
 970                 /* unconstify the socket only to attach it to buff with care. */
 971                 skb_set_owner_edemux(buff, (struct sock *)sk);
 972
 973                 if (sk->sk_state == TCP_TIME_WAIT)
 974                         mark = inet_twsk(sk)->tw_mark;
 975                 else
 976                         mark = READ_ONCE(sk->sk_mark);
 977                 skb_set_delivery_time(buff, tcp_transmit_time(sk), SKB_CLOCK_MONOTONIC);
 978         }
 979         if (txhash) {
 980                 /* autoflowlabel/skb_get_hash_flowi6 rely on buff->hash */
 981                 skb_set_hash(buff, txhash, PKT_HASH_TYPE_L4);
 982         }
 983         fl6.flowi6_mark = IP6_REPLY_MARK(net, skb->mark) ?: mark;
 984         fl6.fl6_dport = t1->dest;
 985         fl6.fl6_sport = t1->source;
 986         fl6.flowi6_uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL);
 987         security_skb_classify_flow(skb, flowi6_to_flowi_common(&fl6));
 988
 989         /* Pass a socket to ip6_dst_lookup either it is for RST
 990          * Underlying function will use this to retrieve the network
 991          * namespace
 992          */
 993         if (sk && sk->sk_state != TCP_TIME_WAIT)
 994                 dst = ip6_dst_lookup_flow(net, sk, &fl6, NULL); /*sk's xfrm_policy can be referred*/
 995         else
 996                 dst = ip6_dst_lookup_flow(net, ctl_sk, &fl6, NULL);
 997         if (!IS_ERR(dst)) {
 998                 skb_dst_set(buff, dst);
 999                 ip6_xmit(ctl_sk, buff, &fl6, fl6.flowi6_mark, NULL,
1000                          tclass & ~INET_ECN_MASK, priority);
1001                 TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
1002                 if (rst)
1003                         TCP_INC_STATS(net, TCP_MIB_OUTRSTS);
1004                 return;
1005         }
1006
1007         kfree_skb(buff);
1008 }
1009
1010 static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb,
1011                               enum sk_rst_reason reason)
1012 {
1013         const struct tcphdr *th = tcp_hdr(skb);
1014         struct ipv6hdr *ipv6h = ipv6_hdr(skb);
1015         const __u8 *md5_hash_location = NULL;
1016 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1017         bool allocated_traffic_key = false;
1018 #endif
1019         const struct tcp_ao_hdr *aoh;
1020         struct tcp_key key = {};
1021         u32 seq = 0, ack_seq = 0;
1022         __be32 label = 0;
1023         u32 priority = 0;
1024         struct net *net;
1025         u32 txhash = 0;
1026         int oif = 0;
1027 #ifdef CONFIG_TCP_MD5SIG
1028         unsigned char newhash[16];
1029         int genhash;
1030         struct sock *sk1 = NULL;
1031 #endif
1032
1033         if (th->rst)
1034                 return;
1035
1036         /* If sk not NULL, it means we did a successful lookup and incoming
1037          * route had to be correct. prequeue might have dropped our dst.
1038          */
1039         if (!sk && !ipv6_unicast_destination(skb))
1040                 return;
1041
1042         net = sk ? sock_net(sk) : dev_net(skb_dst(skb)->dev);
1043         /* Invalid TCP option size or twice included auth */
1044         if (tcp_parse_auth_options(th, &md5_hash_location, &aoh))
1045                 return;
1046 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1047         rcu_read_lock();
1048 #endif
1049 #ifdef CONFIG_TCP_MD5SIG
1050         if (sk && sk_fullsock(sk)) {
1051                 int l3index;
1052
1053                 /* sdif set, means packet ingressed via a device
1054                  * in an L3 domain and inet_iif is set to it.
1055                  */
1056                 l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1057                 key.md5_key = tcp_v6_md5_do_lookup(sk, &ipv6h->saddr, l3index);
1058                 if (key.md5_key)
1059                         key.type = TCP_KEY_MD5;
1060         } else if (md5_hash_location) {
1061                 int dif = tcp_v6_iif_l3_slave(skb);
1062                 int sdif = tcp_v6_sdif(skb);
1063                 int l3index;
1064
1065                 /*
1066                  * active side is lost. Try to find listening socket through
1067                  * source port, and then find md5 key through listening socket.
1068                  * we are not loose security here:
1069                  * Incoming packet is checked with md5 hash with finding key,
1070                  * no RST generated if md5 hash doesn't match.
1071                  */
1072                 sk1 = inet6_lookup_listener(net, net->ipv4.tcp_death_row.hashinfo,
1073                                             NULL, 0, &ipv6h->saddr, th->source,
1074                                             &ipv6h->daddr, ntohs(th->source),
1075                                             dif, sdif);
1076                 if (!sk1)
1077                         goto out;
1078
1079                 /* sdif set, means packet ingressed via a device
1080                  * in an L3 domain and dif is set to it.
1081                  */
1082                 l3index = tcp_v6_sdif(skb) ? dif : 0;
1083
1084                 key.md5_key = tcp_v6_md5_do_lookup(sk1, &ipv6h->saddr, l3index);
1085                 if (!key.md5_key)
1086                         goto out;
1087                 key.type = TCP_KEY_MD5;
1088
1089                 genhash = tcp_v6_md5_hash_skb(newhash, key.md5_key, NULL, skb);
1090                 if (genhash || memcmp(md5_hash_location, newhash, 16) != 0)
1091                         goto out;
1092         }
1093 #endif
1094
1095         if (th->ack)
1096                 seq = ntohl(th->ack_seq);
1097         else
1098                 ack_seq = ntohl(th->seq) + th->syn + th->fin + skb->len -
1099                           (th->doff << 2);
1100
1101 #ifdef CONFIG_TCP_AO
1102         if (aoh) {
1103                 int l3index;
1104
1105                 l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1106                 if (tcp_ao_prepare_reset(sk, skb, aoh, l3index, seq,
1107                                          &key.ao_key, &key.traffic_key,
1108                                          &allocated_traffic_key,
1109                                          &key.rcv_next, &key.sne))
1110                         goto out;
1111                 key.type = TCP_KEY_AO;
1112         }
1113 #endif
1114
1115         if (sk) {
1116                 oif = sk->sk_bound_dev_if;
1117                 if (sk_fullsock(sk)) {
1118                         if (inet6_test_bit(REPFLOW, sk))
1119                                 label = ip6_flowlabel(ipv6h);
1120                         priority = READ_ONCE(sk->sk_priority);
1121                         txhash = sk->sk_txhash;
1122                 }
1123                 if (sk->sk_state == TCP_TIME_WAIT) {
1124                         label = cpu_to_be32(inet_twsk(sk)->tw_flowlabel);
1125                         priority = inet_twsk(sk)->tw_priority;
1126                         txhash = inet_twsk(sk)->tw_txhash;
1127                 }
1128         } else {
1129                 if (net->ipv6.sysctl.flowlabel_reflect & FLOWLABEL_REFLECT_TCP_RESET)
1130                         label = ip6_flowlabel(ipv6h);
1131         }
1132
1133         trace_tcp_send_reset(sk, skb, reason);
1134
1135         tcp_v6_send_response(sk, skb, seq, ack_seq, 0, 0, 0, oif, 1,
1136                              ipv6_get_dsfield(ipv6h), label, priority, txhash,
1137                              &key);
1138
1139 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1140 out:
1141         if (allocated_traffic_key)
1142                 kfree(key.traffic_key);
1143         rcu_read_unlock();
1144 #endif
1145 }
1146
1147 static void tcp_v6_send_ack(const struct sock *sk, struct sk_buff *skb, u32 seq,
1148                             u32 ack, u32 win, u32 tsval, u32 tsecr, int oif,
1149                             struct tcp_key *key, u8 tclass,
1150                             __be32 label, u32 priority, u32 txhash)
1151 {
1152         tcp_v6_send_response(sk, skb, seq, ack, win, tsval, tsecr, oif, 0,
1153                              tclass, label, priority, txhash, key);
1154 }
1155
1156 static void tcp_v6_timewait_ack(struct sock *sk, struct sk_buff *skb)
1157 {
1158         struct inet_timewait_sock *tw = inet_twsk(sk);
1159         struct tcp_timewait_sock *tcptw = tcp_twsk(sk);
1160         struct tcp_key key = {};
1161 #ifdef CONFIG_TCP_AO
1162         struct tcp_ao_info *ao_info;
1163
1164         if (static_branch_unlikely(&tcp_ao_needed.key)) {
1165
1166                 /* FIXME: the segment to-be-acked is not verified yet */
1167                 ao_info = rcu_dereference(tcptw->ao_info);
1168                 if (ao_info) {
1169                         const struct tcp_ao_hdr *aoh;
1170
1171                         /* Invalid TCP option size or twice included auth */
1172                         if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh))
1173                                 goto out;
1174                         if (aoh)
1175                                 key.ao_key = tcp_ao_established_key(sk, ao_info,
1176                                                                     aoh->rnext_keyid, -1);
1177                 }
1178         }
1179         if (key.ao_key) {
1180                 struct tcp_ao_key *rnext_key;
1181
1182                 key.traffic_key = snd_other_key(key.ao_key);
1183                 /* rcv_next switches to our rcv_next */
1184                 rnext_key = READ_ONCE(ao_info->rnext_key);
1185                 key.rcv_next = rnext_key->rcvid;
1186                 key.sne = READ_ONCE(ao_info->snd_sne);
1187                 key.type = TCP_KEY_AO;
1188 #else
1189         if (0) {
1190 #endif
1191 #ifdef CONFIG_TCP_MD5SIG
1192         } else if (static_branch_unlikely(&tcp_md5_needed.key)) {
1193                 key.md5_key = tcp_twsk_md5_key(tcptw);
1194                 if (key.md5_key)
1195                         key.type = TCP_KEY_MD5;
1196 #endif
1197         }
1198
1199         tcp_v6_send_ack(sk, skb, tcptw->tw_snd_nxt,
1200                         READ_ONCE(tcptw->tw_rcv_nxt),
1201                         tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale,
1202                         tcp_tw_tsval(tcptw),
1203                         READ_ONCE(tcptw->tw_ts_recent), tw->tw_bound_dev_if,
1204                         &key, tw->tw_tclass, cpu_to_be32(tw->tw_flowlabel),
1205                         tw->tw_priority, tw->tw_txhash);
1206
1207 #ifdef CONFIG_TCP_AO
1208 out:
1209 #endif
1210         inet_twsk_put(tw);
1211 }
1212
1213 static void tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
1214                                   struct request_sock *req)
1215 {
1216         struct tcp_key key = {};
1217
1218 #ifdef CONFIG_TCP_AO
1219         if (static_branch_unlikely(&tcp_ao_needed.key) &&
1220             tcp_rsk_used_ao(req)) {
1221                 const struct in6_addr *addr = &ipv6_hdr(skb)->saddr;
1222                 const struct tcp_ao_hdr *aoh;
1223                 int l3index;
1224
1225                 l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1226                 /* Invalid TCP option size or twice included auth */
1227                 if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh))
1228                         return;
1229                 if (!aoh)
1230                         return;
1231                 key.ao_key = tcp_ao_do_lookup(sk, l3index,
1232                                               (union tcp_ao_addr *)addr,
1233                                               AF_INET6, aoh->rnext_keyid, -1);
1234                 if (unlikely(!key.ao_key)) {
1235                         /* Send ACK with any matching MKT for the peer */
1236                         key.ao_key = tcp_ao_do_lookup(sk, l3index,
1237                                                       (union tcp_ao_addr *)addr,
1238                                                       AF_INET6, -1, -1);
1239                         /* Matching key disappeared (user removed the key?)
1240                          * let the handshake timeout.
1241                          */
1242                         if (!key.ao_key) {
1243                                 net_info_ratelimited("TCP-AO key for (%pI6, %d)->(%pI6, %d) suddenly disappeared, won't ACK new connection\n",
1244                                                      addr,
1245                                                      ntohs(tcp_hdr(skb)->source),
1246                                                      &ipv6_hdr(skb)->daddr,
1247                                                      ntohs(tcp_hdr(skb)->dest));
1248                                 return;
1249                         }
1250                 }
1251                 key.traffic_key = kmalloc(tcp_ao_digest_size(key.ao_key), GFP_ATOMIC);
1252                 if (!key.traffic_key)
1253                         return;
1254
1255                 key.type = TCP_KEY_AO;
1256                 key.rcv_next = aoh->keyid;
1257                 tcp_v6_ao_calc_key_rsk(key.ao_key, key.traffic_key, req);
1258 #else
1259         if (0) {
1260 #endif
1261 #ifdef CONFIG_TCP_MD5SIG
1262         } else if (static_branch_unlikely(&tcp_md5_needed.key)) {
1263                 int l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1264
1265                 key.md5_key = tcp_v6_md5_do_lookup(sk, &ipv6_hdr(skb)->saddr,
1266                                                    l3index);
1267                 if (key.md5_key)
1268                         key.type = TCP_KEY_MD5;
1269 #endif
1270         }
1271
1272         /* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV
1273          * sk->sk_state == TCP_SYN_RECV -> for Fast Open.
1274          */
1275         tcp_v6_send_ack(sk, skb, (sk->sk_state == TCP_LISTEN) ?
1276                         tcp_rsk(req)->snt_isn + 1 : tcp_sk(sk)->snd_nxt,
1277                         tcp_rsk(req)->rcv_nxt,
1278                         tcp_synack_window(req) >> inet_rsk(req)->rcv_wscale,
1279                         tcp_rsk_tsval(tcp_rsk(req)),
1280                         READ_ONCE(req->ts_recent), sk->sk_bound_dev_if,
1281                         &key, ipv6_get_dsfield(ipv6_hdr(skb)), 0,
1282                         READ_ONCE(sk->sk_priority),
1283                         READ_ONCE(tcp_rsk(req)->txhash));
1284         if (tcp_key_is_ao(&key))
1285                 kfree(key.traffic_key);
1286 }
1287
1288
1289 static struct sock *tcp_v6_cookie_check(struct sock *sk, struct sk_buff *skb)
1290 {
1291 #ifdef CONFIG_SYN_COOKIES
1292         const struct tcphdr *th = tcp_hdr(skb);
1293
1294         if (!th->syn)
1295                 sk = cookie_v6_check(sk, skb);
1296 #endif
1297         return sk;
1298 }
1299
1300 u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph,
1301                          struct tcphdr *th, u32 *cookie)
1302 {
1303         u16 mss = 0;
1304 #ifdef CONFIG_SYN_COOKIES
1305         mss = tcp_get_syncookie_mss(&tcp6_request_sock_ops,
1306                                     &tcp_request_sock_ipv6_ops, sk, th);
1307         if (mss) {
1308                 *cookie = __cookie_v6_init_sequence(iph, th, &mss);
1309                 tcp_synq_overflow(sk);
1310         }
1311 #endif
1312         return mss;
1313 }
1314
1315 static int tcp_v6_conn_request(struct sock *sk, struct sk_buff *skb)
1316 {
1317         if (skb->protocol == htons(ETH_P_IP))
1318                 return tcp_v4_conn_request(sk, skb);
1319
1320         if (!ipv6_unicast_destination(skb))
1321                 goto drop;
1322
1323         if (ipv6_addr_v4mapped(&ipv6_hdr(skb)->saddr)) {
1324                 __IP6_INC_STATS(sock_net(sk), NULL, IPSTATS_MIB_INHDRERRORS);
1325                 return 0;
1326         }
1327
1328         return tcp_conn_request(&tcp6_request_sock_ops,
1329                                 &tcp_request_sock_ipv6_ops, sk, skb);
1330
1331 drop:
1332         tcp_listendrop(sk);
1333         return 0; /* don't send reset */
1334 }
1335
1336 static void tcp_v6_restore_cb(struct sk_buff *skb)
1337 {
1338         /* We need to move header back to the beginning if xfrm6_policy_check()
1339          * and tcp_v6_fill_cb() are going to be called again.
1340          * ip6_datagram_recv_specific_ctl() also expects IP6CB to be there.
1341          */
1342         memmove(IP6CB(skb), &TCP_SKB_CB(skb)->header.h6,
1343                 sizeof(struct inet6_skb_parm));
1344 }
1345
1346 static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
1347                                          struct request_sock *req,
1348                                          struct dst_entry *dst,
1349                                          struct request_sock *req_unhash,
1350                                          bool *own_req)
1351 {
1352         struct inet_request_sock *ireq;
1353         struct ipv6_pinfo *newnp;
1354         const struct ipv6_pinfo *np = tcp_inet6_sk(sk);
1355         struct ipv6_txoptions *opt;
1356         struct inet_sock *newinet;
1357         bool found_dup_sk = false;
1358         struct tcp_sock *newtp;
1359         struct sock *newsk;
1360 #ifdef CONFIG_TCP_MD5SIG
1361         struct tcp_md5sig_key *key;
1362         int l3index;
1363 #endif
1364         struct flowi6 fl6;
1365
1366         if (skb->protocol == htons(ETH_P_IP)) {
1367                 /*
1368                  *      v6 mapped
1369                  */
1370
1371                 newsk = tcp_v4_syn_recv_sock(sk, skb, req, dst,
1372                                              req_unhash, own_req);
1373
1374                 if (!newsk)
1375                         return NULL;
1376
1377                 inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk);
1378
1379                 newnp = tcp_inet6_sk(newsk);
1380                 newtp = tcp_sk(newsk);
1381
1382                 memcpy(newnp, np, sizeof(struct ipv6_pinfo));
1383
1384                 newnp->saddr = newsk->sk_v6_rcv_saddr;
1385
1386                 inet_csk(newsk)->icsk_af_ops = &ipv6_mapped;
1387                 if (sk_is_mptcp(newsk))
1388                         mptcpv6_handle_mapped(newsk, true);
1389                 newsk->sk_backlog_rcv = tcp_v4_do_rcv;
1390 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1391                 newtp->af_specific = &tcp_sock_ipv6_mapped_specific;
1392 #endif
1393
1394                 newnp->ipv6_mc_list = NULL;
1395                 newnp->ipv6_ac_list = NULL;
1396                 newnp->ipv6_fl_list = NULL;
1397                 newnp->pktoptions  = NULL;
1398                 newnp->opt         = NULL;
1399                 newnp->mcast_oif   = inet_iif(skb);
1400                 newnp->mcast_hops  = ip_hdr(skb)->ttl;
1401                 newnp->rcv_flowinfo = 0;
1402                 if (inet6_test_bit(REPFLOW, sk))
1403                         newnp->flow_label = 0;
1404
1405                 /*
1406                  * No need to charge this sock to the relevant IPv6 refcnt debug socks count
1407                  * here, tcp_create_openreq_child now does this for us, see the comment in
1408                  * that function for the gory details. -acme
1409                  */
1410
1411                 /* It is tricky place. Until this moment IPv4 tcp
1412                    worked with IPv6 icsk.icsk_af_ops.
1413                    Sync it now.
1414                  */
1415                 tcp_sync_mss(newsk, inet_csk(newsk)->icsk_pmtu_cookie);
1416
1417                 return newsk;
1418         }
1419
1420         ireq = inet_rsk(req);
1421
1422         if (sk_acceptq_is_full(sk))
1423                 goto out_overflow;
1424
1425         if (!dst) {
1426                 dst = inet6_csk_route_req(sk, &fl6, req, IPPROTO_TCP);
1427                 if (!dst)
1428                         goto out;
1429         }
1430
1431         newsk = tcp_create_openreq_child(sk, req, skb);
1432         if (!newsk)
1433                 goto out_nonewsk;
1434
1435         /*
1436          * No need to charge this sock to the relevant IPv6 refcnt debug socks
1437          * count here, tcp_create_openreq_child now does this for us, see the
1438          * comment in that function for the gory details. -acme
1439          */
1440
1441         newsk->sk_gso_type = SKB_GSO_TCPV6;
1442         inet6_sk_rx_dst_set(newsk, skb);
1443
1444         inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk);
1445
1446         newtp = tcp_sk(newsk);
1447         newinet = inet_sk(newsk);
1448         newnp = tcp_inet6_sk(newsk);
1449
1450         memcpy(newnp, np, sizeof(struct ipv6_pinfo));
1451
1452         ip6_dst_store(newsk, dst, NULL, NULL);
1453
1454         newsk->sk_v6_daddr = ireq->ir_v6_rmt_addr;
1455         newnp->saddr = ireq->ir_v6_loc_addr;
1456         newsk->sk_v6_rcv_saddr = ireq->ir_v6_loc_addr;
1457         newsk->sk_bound_dev_if = ireq->ir_iif;
1458
1459         /* Now IPv6 options...
1460
1461            First: no IPv4 options.
1462          */
1463         newinet->inet_opt = NULL;
1464         newnp->ipv6_mc_list = NULL;
1465         newnp->ipv6_ac_list = NULL;
1466         newnp->ipv6_fl_list = NULL;
1467
1468         /* Clone RX bits */
1469         newnp->rxopt.all = np->rxopt.all;
1470
1471         newnp->pktoptions = NULL;
1472         newnp->opt        = NULL;
1473         newnp->mcast_oif  = tcp_v6_iif(skb);
1474         newnp->mcast_hops = ipv6_hdr(skb)->hop_limit;
1475         newnp->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(skb));
1476         if (inet6_test_bit(REPFLOW, sk))
1477                 newnp->flow_label = ip6_flowlabel(ipv6_hdr(skb));
1478
1479         /* Set ToS of the new socket based upon the value of incoming SYN.
1480          * ECT bits are set later in tcp_init_transfer().
1481          */
1482         if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos))
1483                 newnp->tclass = tcp_rsk(req)->syn_tos & ~INET_ECN_MASK;
1484
1485         /* Clone native IPv6 options from listening socket (if any)
1486
1487            Yes, keeping reference count would be much more clever,
1488            but we make one more one thing there: reattach optmem
1489            to newsk.
1490          */
1491         opt = ireq->ipv6_opt;
1492         if (!opt)
1493                 opt = rcu_dereference(np->opt);
1494         if (opt) {
1495                 opt = ipv6_dup_options(newsk, opt);
1496                 RCU_INIT_POINTER(newnp->opt, opt);
1497         }
1498         inet_csk(newsk)->icsk_ext_hdr_len = 0;
1499         if (opt)
1500                 inet_csk(newsk)->icsk_ext_hdr_len = opt->opt_nflen +
1501                                                     opt->opt_flen;
1502
1503         tcp_ca_openreq_child(newsk, dst);
1504
1505         tcp_sync_mss(newsk, dst_mtu(dst));
1506         newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst));
1507
1508         tcp_initialize_rcv_mss(newsk);
1509
1510         newinet->inet_daddr = newinet->inet_saddr = LOOPBACK4_IPV6;
1511         newinet->inet_rcv_saddr = LOOPBACK4_IPV6;
1512
1513 #ifdef CONFIG_TCP_MD5SIG
1514         l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif);
1515
1516         if (!tcp_rsk_used_ao(req)) {
1517                 /* Copy over the MD5 key from the original socket */
1518                 key = tcp_v6_md5_do_lookup(sk, &newsk->sk_v6_daddr, l3index);
1519                 if (key) {
1520                         const union tcp_md5_addr *addr;
1521
1522                         addr = (union tcp_md5_addr *)&newsk->sk_v6_daddr;
1523                         if (tcp_md5_key_copy(newsk, addr, AF_INET6, 128, l3index, key)) {
1524                                 inet_csk_prepare_forced_close(newsk);
1525                                 tcp_done(newsk);
1526                                 goto out;
1527                         }
1528                 }
1529         }
1530 #endif
1531 #ifdef CONFIG_TCP_AO
1532         /* Copy over tcp_ao_info if any */
1533         if (tcp_ao_copy_all_matching(sk, newsk, req, skb, AF_INET6))
1534                 goto out; /* OOM */
1535 #endif
1536
1537         if (__inet_inherit_port(sk, newsk) < 0) {
1538                 inet_csk_prepare_forced_close(newsk);
1539                 tcp_done(newsk);
1540                 goto out;
1541         }
1542         *own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash),
1543                                        &found_dup_sk);
1544         if (*own_req) {
1545                 tcp_move_syn(newtp, req);
1546
1547                 /* Clone pktoptions received with SYN, if we own the req */
1548                 if (ireq->pktopts) {
1549                         newnp->pktoptions = skb_clone_and_charge_r(ireq->pktopts, newsk);
1550                         consume_skb(ireq->pktopts);
1551                         ireq->pktopts = NULL;
1552                         if (newnp->pktoptions)
1553                                 tcp_v6_restore_cb(newnp->pktoptions);
1554                 }
1555         } else {
1556                 if (!req_unhash && found_dup_sk) {
1557                         /* This code path should only be executed in the
1558                          * syncookie case only
1559                          */
1560                         bh_unlock_sock(newsk);
1561                         sock_put(newsk);
1562                         newsk = NULL;
1563                 }
1564         }
1565
1566         return newsk;
1567
1568 out_overflow:
1569         __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
1570 out_nonewsk:
1571         dst_release(dst);
1572 out:
1573         tcp_listendrop(sk);
1574         return NULL;
1575 }
1576
1577 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
1578                                                            u32));
1579 /* The socket must have it's spinlock held when we get
1580  * here, unless it is a TCP_LISTEN socket.
1581  *
1582  * We have a potential double-lock case here, so even when
1583  * doing backlog processing we use the BH locking scheme.
1584  * This is because we cannot sleep with the original spinlock
1585  * held.
1586  */
1587 INDIRECT_CALLABLE_SCOPE
1588 int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)
1589 {
1590         struct ipv6_pinfo *np = tcp_inet6_sk(sk);
1591         struct sk_buff *opt_skb = NULL;
1592         enum skb_drop_reason reason;
1593         struct tcp_sock *tp;
1594
1595         /* Imagine: socket is IPv6. IPv4 packet arrives,
1596            goes to IPv4 receive handler and backlogged.
1597            From backlog it always goes here. Kerboom...
1598            Fortunately, tcp_rcv_established and rcv_established
1599            handle them correctly, but it is not case with
1600            tcp_v6_hnd_req and tcp_v6_send_reset().   --ANK
1601          */
1602
1603         if (skb->protocol == htons(ETH_P_IP))
1604                 return tcp_v4_do_rcv(sk, skb);
1605
1606         /*
1607          *      socket locking is here for SMP purposes as backlog rcv
1608          *      is currently called with bh processing disabled.
1609          */
1610
1611         /* Do Stevens' IPV6_PKTOPTIONS.
1612
1613            Yes, guys, it is the only place in our code, where we
1614            may make it not affecting IPv4.
1615            The rest of code is protocol independent,
1616            and I do not like idea to uglify IPv4.
1617
1618            Actually, all the idea behind IPV6_PKTOPTIONS
1619            looks not very well thought. For now we latch
1620            options, received in the last packet, enqueued
1621            by tcp. Feel free to propose better solution.
1622                                                --ANK (980728)
1623          */
1624         if (np->rxopt.all && sk->sk_state != TCP_LISTEN)
1625                 opt_skb = skb_clone_and_charge_r(skb, sk);
1626
1627         if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
1628                 struct dst_entry *dst;
1629
1630                 dst = rcu_dereference_protected(sk->sk_rx_dst,
1631                                                 lockdep_sock_is_held(sk));
1632
1633                 sock_rps_save_rxhash(sk, skb);
1634                 sk_mark_napi_id(sk, skb);
1635                 if (dst) {
1636                         if (sk->sk_rx_dst_ifindex != skb->skb_iif ||
1637                             INDIRECT_CALL_1(dst->ops->check, ip6_dst_check,
1638                                             dst, sk->sk_rx_dst_cookie) == NULL) {
1639                                 RCU_INIT_POINTER(sk->sk_rx_dst, NULL);
1640                                 dst_release(dst);
1641                         }
1642                 }
1643
1644                 tcp_rcv_established(sk, skb);
1645                 if (opt_skb)
1646                         goto ipv6_pktoptions;
1647                 return 0;
1648         }
1649
1650         if (tcp_checksum_complete(skb))
1651                 goto csum_err;
1652
1653         if (sk->sk_state == TCP_LISTEN) {
1654                 struct sock *nsk = tcp_v6_cookie_check(sk, skb);
1655
1656                 if (nsk != sk) {
1657                         if (nsk) {
1658                                 reason = tcp_child_process(sk, nsk, skb);
1659                                 if (reason)
1660                                         goto reset;
1661                         }
1662                         return 0;
1663                 }
1664         } else
1665                 sock_rps_save_rxhash(sk, skb);
1666
1667         reason = tcp_rcv_state_process(sk, skb);
1668         if (reason)
1669                 goto reset;
1670         if (opt_skb)
1671                 goto ipv6_pktoptions;
1672         return 0;
1673
1674 reset:
1675         tcp_v6_send_reset(sk, skb, sk_rst_convert_drop_reason(reason));
1676 discard:
1677         if (opt_skb)
1678                 __kfree_skb(opt_skb);
1679         sk_skb_reason_drop(sk, skb, reason);
1680         return 0;
1681 csum_err:
1682         reason = SKB_DROP_REASON_TCP_CSUM;
1683         trace_tcp_bad_csum(skb);
1684         TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1685         TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1686         goto discard;
1687
1688
1689 ipv6_pktoptions:
1690         /* Do you ask, what is it?
1691
1692            1. skb was enqueued by tcp.
1693            2. skb is added to tail of read queue, rather than out of order.
1694            3. socket is not in passive state.
1695            4. Finally, it really contains options, which user wants to receive.
1696          */
1697         tp = tcp_sk(sk);
1698         if (TCP_SKB_CB(opt_skb)->end_seq == tp->rcv_nxt &&
1699             !((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) {
1700                 if (np->rxopt.bits.rxinfo || np->rxopt.bits.rxoinfo)
1701                         WRITE_ONCE(np->mcast_oif, tcp_v6_iif(opt_skb));
1702                 if (np->rxopt.bits.rxhlim || np->rxopt.bits.rxohlim)
1703                         WRITE_ONCE(np->mcast_hops,
1704                                    ipv6_hdr(opt_skb)->hop_limit);
1705                 if (np->rxopt.bits.rxflow || np->rxopt.bits.rxtclass)
1706                         np->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(opt_skb));
1707                 if (inet6_test_bit(REPFLOW, sk))
1708                         np->flow_label = ip6_flowlabel(ipv6_hdr(opt_skb));
1709                 if (ipv6_opt_accepted(sk, opt_skb, &TCP_SKB_CB(opt_skb)->header.h6)) {
1710                         tcp_v6_restore_cb(opt_skb);
1711                         opt_skb = xchg(&np->pktoptions, opt_skb);
1712                 } else {
1713                         __kfree_skb(opt_skb);
1714                         opt_skb = xchg(&np->pktoptions, NULL);
1715                 }
1716         }
1717
1718         consume_skb(opt_skb);
1719         return 0;
1720 }
1721
1722 static void tcp_v6_fill_cb(struct sk_buff *skb, const struct ipv6hdr *hdr,
1723                            const struct tcphdr *th)
1724 {
1725         /* This is tricky: we move IP6CB at its correct location into
1726          * TCP_SKB_CB(). It must be done after xfrm6_policy_check(), because
1727          * _decode_session6() uses IP6CB().
1728          * barrier() makes sure compiler won't play aliasing games.
1729          */
1730         memmove(&TCP_SKB_CB(skb)->header.h6, IP6CB(skb),
1731                 sizeof(struct inet6_skb_parm));
1732         barrier();
1733
1734         TCP_SKB_CB(skb)->seq = ntohl(th->seq);
1735         TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
1736                                     skb->len - th->doff*4);
1737         TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
1738         TCP_SKB_CB(skb)->tcp_flags = tcp_flag_byte(th);
1739         TCP_SKB_CB(skb)->ip_dsfield = ipv6_get_dsfield(hdr);
1740         TCP_SKB_CB(skb)->sacked = 0;
1741         TCP_SKB_CB(skb)->has_rxtstamp =
1742                         skb->tstamp || skb_hwtstamps(skb)->hwtstamp;
1743 }
1744
1745 INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb)
1746 {
1747         enum skb_drop_reason drop_reason;
1748         int sdif = inet6_sdif(skb);
1749         int dif = inet6_iif(skb);
1750         const struct tcphdr *th;
1751         const struct ipv6hdr *hdr;
1752         struct sock *sk = NULL;
1753         bool refcounted;
1754         int ret;
1755         u32 isn;
1756         struct net *net = dev_net(skb->dev);
1757
1758         drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
1759         if (skb->pkt_type != PACKET_HOST)
1760                 goto discard_it;
1761
1762         /*
1763          *      Count it even if it's bad.
1764          */
1765         __TCP_INC_STATS(net, TCP_MIB_INSEGS);
1766
1767         if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
1768                 goto discard_it;
1769
1770         th = (const struct tcphdr *)skb->data;
1771
1772         if (unlikely(th->doff < sizeof(struct tcphdr) / 4)) {
1773                 drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL;
1774                 goto bad_packet;
1775         }
1776         if (!pskb_may_pull(skb, th->doff*4))
1777                 goto discard_it;
1778
1779         if (skb_checksum_init(skb, IPPROTO_TCP, ip6_compute_pseudo))
1780                 goto csum_error;
1781
1782         th = (const struct tcphdr *)skb->data;
1783         hdr = ipv6_hdr(skb);
1784
1785 lookup:
1786         sk = __inet6_lookup_skb(net->ipv4.tcp_death_row.hashinfo, skb, __tcp_hdrlen(th),
1787                                 th->source, th->dest, inet6_iif(skb), sdif,
1788                                 &refcounted);
1789         if (!sk)
1790                 goto no_tcp_socket;
1791
1792         if (sk->sk_state == TCP_TIME_WAIT)
1793                 goto do_time_wait;
1794
1795         if (sk->sk_state == TCP_NEW_SYN_RECV) {
1796                 struct request_sock *req = inet_reqsk(sk);
1797                 bool req_stolen = false;
1798                 struct sock *nsk;
1799
1800                 sk = req->rsk_listener;
1801                 if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb))
1802                         drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1803                 else
1804                         drop_reason = tcp_inbound_hash(sk, req, skb,
1805                                                        &hdr->saddr, &hdr->daddr,
1806                                                        AF_INET6, dif, sdif);
1807                 if (drop_reason) {
1808                         sk_drops_add(sk, skb);
1809                         reqsk_put(req);
1810                         goto discard_it;
1811                 }
1812                 if (tcp_checksum_complete(skb)) {
1813                         reqsk_put(req);
1814                         goto csum_error;
1815                 }
1816                 if (unlikely(sk->sk_state != TCP_LISTEN)) {
1817                         nsk = reuseport_migrate_sock(sk, req_to_sk(req), skb);
1818                         if (!nsk) {
1819                                 inet_csk_reqsk_queue_drop_and_put(sk, req);
1820                                 goto lookup;
1821                         }
1822                         sk = nsk;
1823                         /* reuseport_migrate_sock() has already held one sk_refcnt
1824                          * before returning.
1825                          */
1826                 } else {
1827                         sock_hold(sk);
1828                 }
1829                 refcounted = true;
1830                 nsk = NULL;
1831                 if (!tcp_filter(sk, skb)) {
1832                         th = (const struct tcphdr *)skb->data;
1833                         hdr = ipv6_hdr(skb);
1834                         tcp_v6_fill_cb(skb, hdr, th);
1835                         nsk = tcp_check_req(sk, skb, req, false, &req_stolen);
1836                 } else {
1837                         drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
1838                 }
1839                 if (!nsk) {
1840                         reqsk_put(req);
1841                         if (req_stolen) {
1842                                 /* Another cpu got exclusive access to req
1843                                  * and created a full blown socket.
1844                                  * Try to feed this packet to this socket
1845                                  * instead of discarding it.
1846                                  */
1847                                 tcp_v6_restore_cb(skb);
1848                                 sock_put(sk);
1849                                 goto lookup;
1850                         }
1851                         goto discard_and_relse;
1852                 }
1853                 nf_reset_ct(skb);
1854                 if (nsk == sk) {
1855                         reqsk_put(req);
1856                         tcp_v6_restore_cb(skb);
1857                 } else {
1858                         drop_reason = tcp_child_process(sk, nsk, skb);
1859                         if (drop_reason) {
1860                                 enum sk_rst_reason rst_reason;
1861
1862                                 rst_reason = sk_rst_convert_drop_reason(drop_reason);
1863                                 tcp_v6_send_reset(nsk, skb, rst_reason);
1864                                 goto discard_and_relse;
1865                         }
1866                         sock_put(sk);
1867                         return 0;
1868                 }
1869         }
1870
1871 process:
1872         if (static_branch_unlikely(&ip6_min_hopcount)) {
1873                 /* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */
1874                 if (unlikely(hdr->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount))) {
1875                         __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
1876                         drop_reason = SKB_DROP_REASON_TCP_MINTTL;
1877                         goto discard_and_relse;
1878                 }
1879         }
1880
1881         if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb)) {
1882                 drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1883                 goto discard_and_relse;
1884         }
1885
1886         drop_reason = tcp_inbound_hash(sk, NULL, skb, &hdr->saddr, &hdr->daddr,
1887                                        AF_INET6, dif, sdif);
1888         if (drop_reason)
1889                 goto discard_and_relse;
1890
1891         nf_reset_ct(skb);
1892
1893         if (tcp_filter(sk, skb)) {
1894                 drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
1895                 goto discard_and_relse;
1896         }
1897         th = (const struct tcphdr *)skb->data;
1898         hdr = ipv6_hdr(skb);
1899         tcp_v6_fill_cb(skb, hdr, th);
1900
1901         skb->dev = NULL;
1902
1903         if (sk->sk_state == TCP_LISTEN) {
1904                 ret = tcp_v6_do_rcv(sk, skb);
1905                 goto put_and_return;
1906         }
1907
1908         sk_incoming_cpu_update(sk);
1909
1910         bh_lock_sock_nested(sk);
1911         tcp_segs_in(tcp_sk(sk), skb);
1912         ret = 0;
1913         if (!sock_owned_by_user(sk)) {
1914                 ret = tcp_v6_do_rcv(sk, skb);
1915         } else {
1916                 if (tcp_add_backlog(sk, skb, &drop_reason))
1917                         goto discard_and_relse;
1918         }
1919         bh_unlock_sock(sk);
1920 put_and_return:
1921         if (refcounted)
1922                 sock_put(sk);
1923         return ret ? -1 : 0;
1924
1925 no_tcp_socket:
1926         drop_reason = SKB_DROP_REASON_NO_SOCKET;
1927         if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb))
1928                 goto discard_it;
1929
1930         tcp_v6_fill_cb(skb, hdr, th);
1931
1932         if (tcp_checksum_complete(skb)) {
1933 csum_error:
1934                 drop_reason = SKB_DROP_REASON_TCP_CSUM;
1935                 trace_tcp_bad_csum(skb);
1936                 __TCP_INC_STATS(net, TCP_MIB_CSUMERRORS);
1937 bad_packet:
1938                 __TCP_INC_STATS(net, TCP_MIB_INERRS);
1939         } else {
1940                 tcp_v6_send_reset(NULL, skb, sk_rst_convert_drop_reason(drop_reason));
1941         }
1942
1943 discard_it:
1944         SKB_DR_OR(drop_reason, NOT_SPECIFIED);
1945         sk_skb_reason_drop(sk, skb, drop_reason);
1946         return 0;
1947
1948 discard_and_relse:
1949         sk_drops_add(sk, skb);
1950         if (refcounted)
1951                 sock_put(sk);
1952         goto discard_it;
1953
1954 do_time_wait:
1955         if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb)) {
1956                 drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1957                 inet_twsk_put(inet_twsk(sk));
1958                 goto discard_it;
1959         }
1960
1961         tcp_v6_fill_cb(skb, hdr, th);
1962
1963         if (tcp_checksum_complete(skb)) {
1964                 inet_twsk_put(inet_twsk(sk));
1965                 goto csum_error;
1966         }
1967
1968         switch (tcp_timewait_state_process(inet_twsk(sk), skb, th, &isn)) {
1969         case TCP_TW_SYN:
1970         {
1971                 struct sock *sk2;
1972
1973                 sk2 = inet6_lookup_listener(net, net->ipv4.tcp_death_row.hashinfo,
1974                                             skb, __tcp_hdrlen(th),
1975                                             &ipv6_hdr(skb)->saddr, th->source,
1976                                             &ipv6_hdr(skb)->daddr,
1977                                             ntohs(th->dest),
1978                                             tcp_v6_iif_l3_slave(skb),
1979                                             sdif);
1980                 if (sk2) {
1981                         struct inet_timewait_sock *tw = inet_twsk(sk);
1982                         inet_twsk_deschedule_put(tw);
1983                         sk = sk2;
1984                         tcp_v6_restore_cb(skb);
1985                         refcounted = false;
1986                         __this_cpu_write(tcp_tw_isn, isn);
1987                         goto process;
1988                 }
1989         }
1990                 /* to ACK */
1991                 fallthrough;
1992         case TCP_TW_ACK:
1993                 tcp_v6_timewait_ack(sk, skb);
1994                 break;
1995         case TCP_TW_RST:
1996                 tcp_v6_send_reset(sk, skb, SK_RST_REASON_TCP_TIMEWAIT_SOCKET);
1997                 inet_twsk_deschedule_put(inet_twsk(sk));
1998                 goto discard_it;
1999         case TCP_TW_SUCCESS:
2000                 ;
2001         }
2002         goto discard_it;
2003 }
2004
2005 void tcp_v6_early_demux(struct sk_buff *skb)
2006 {
2007         struct net *net = dev_net(skb->dev);
2008         const struct ipv6hdr *hdr;
2009         const struct tcphdr *th;
2010         struct sock *sk;
2011
2012         if (skb->pkt_type != PACKET_HOST)
2013                 return;
2014
2015         if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct tcphdr)))
2016                 return;
2017
2018         hdr = ipv6_hdr(skb);
2019         th = tcp_hdr(skb);
2020
2021         if (th->doff < sizeof(struct tcphdr) / 4)
2022                 return;
2023
2024         /* Note : We use inet6_iif() here, not tcp_v6_iif() */
2025         sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
2026                                         &hdr->saddr, th->source,
2027                                         &hdr->daddr, ntohs(th->dest),
2028                                         inet6_iif(skb), inet6_sdif(skb));
2029         if (sk) {
2030                 skb->sk = sk;
2031                 skb->destructor = sock_edemux;
2032                 if (sk_fullsock(sk)) {
2033                         struct dst_entry *dst = rcu_dereference(sk->sk_rx_dst);
2034
2035                         if (dst)
2036                                 dst = dst_check(dst, sk->sk_rx_dst_cookie);
2037                         if (dst &&
2038                             sk->sk_rx_dst_ifindex == skb->skb_iif)
2039                                 skb_dst_set_noref(skb, dst);
2040                 }
2041         }
2042 }
2043
2044 static struct timewait_sock_ops tcp6_timewait_sock_ops = {
2045         .twsk_obj_size  = sizeof(struct tcp6_timewait_sock),
2046         .twsk_destructor = tcp_twsk_destructor,
2047 };
2048
2049 INDIRECT_CALLABLE_SCOPE void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb)
2050 {
2051         __tcp_v6_send_check(skb, &sk->sk_v6_rcv_saddr, &sk->sk_v6_daddr);
2052 }
2053
2054 const struct inet_connection_sock_af_ops ipv6_specific = {
2055         .queue_xmit        = inet6_csk_xmit,
2056         .send_check        = tcp_v6_send_check,
2057         .rebuild_header    = inet6_sk_rebuild_header,
2058         .sk_rx_dst_set     = inet6_sk_rx_dst_set,
2059         .conn_request      = tcp_v6_conn_request,
2060         .syn_recv_sock     = tcp_v6_syn_recv_sock,
2061         .net_header_len    = sizeof(struct ipv6hdr),
2062         .setsockopt        = ipv6_setsockopt,
2063         .getsockopt        = ipv6_getsockopt,
2064         .addr2sockaddr     = inet6_csk_addr2sockaddr,
2065         .sockaddr_len      = sizeof(struct sockaddr_in6),
2066         .mtu_reduced       = tcp_v6_mtu_reduced,
2067 };
2068
2069 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2070 static const struct tcp_sock_af_ops tcp_sock_ipv6_specific = {
2071 #ifdef CONFIG_TCP_MD5SIG
2072         .md5_lookup     =       tcp_v6_md5_lookup,
2073         .calc_md5_hash  =       tcp_v6_md5_hash_skb,
2074         .md5_parse      =       tcp_v6_parse_md5_keys,
2075 #endif
2076 #ifdef CONFIG_TCP_AO
2077         .ao_lookup      =       tcp_v6_ao_lookup,
2078         .calc_ao_hash   =       tcp_v6_ao_hash_skb,
2079         .ao_parse       =       tcp_v6_parse_ao,
2080         .ao_calc_key_sk =       tcp_v6_ao_calc_key_sk,
2081 #endif
2082 };
2083 #endif
2084
2085 /*
2086  *      TCP over IPv4 via INET6 API
2087  */
2088 static const struct inet_connection_sock_af_ops ipv6_mapped = {
2089         .queue_xmit        = ip_queue_xmit,
2090         .send_check        = tcp_v4_send_check,
2091         .rebuild_header    = inet_sk_rebuild_header,
2092         .sk_rx_dst_set     = inet_sk_rx_dst_set,
2093         .conn_request      = tcp_v6_conn_request,
2094         .syn_recv_sock     = tcp_v6_syn_recv_sock,
2095         .net_header_len    = sizeof(struct iphdr),
2096         .setsockopt        = ipv6_setsockopt,
2097         .getsockopt        = ipv6_getsockopt,
2098         .addr2sockaddr     = inet6_csk_addr2sockaddr,
2099         .sockaddr_len      = sizeof(struct sockaddr_in6),
2100         .mtu_reduced       = tcp_v4_mtu_reduced,
2101 };
2102
2103 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2104 static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific = {
2105 #ifdef CONFIG_TCP_MD5SIG
2106         .md5_lookup     =       tcp_v4_md5_lookup,
2107         .calc_md5_hash  =       tcp_v4_md5_hash_skb,
2108         .md5_parse      =       tcp_v6_parse_md5_keys,
2109 #endif
2110 #ifdef CONFIG_TCP_AO
2111         .ao_lookup      =       tcp_v6_ao_lookup,
2112         .calc_ao_hash   =       tcp_v4_ao_hash_skb,
2113         .ao_parse       =       tcp_v6_parse_ao,
2114         .ao_calc_key_sk =       tcp_v4_ao_calc_key_sk,
2115 #endif
2116 };
2117 #endif
2118
2119 /* NOTE: A lot of things set to zero explicitly by call to
2120  *       sk_alloc() so need not be done here.
2121  */
2122 static int tcp_v6_init_sock(struct sock *sk)
2123 {
2124         struct inet_connection_sock *icsk = inet_csk(sk);
2125
2126         tcp_init_sock(sk);
2127
2128         icsk->icsk_af_ops = &ipv6_specific;
2129
2130 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2131         tcp_sk(sk)->af_specific = &tcp_sock_ipv6_specific;
2132 #endif
2133
2134         return 0;
2135 }
2136
2137 #ifdef CONFIG_PROC_FS
2138 /* Proc filesystem TCPv6 sock list dumping. */
2139 static void get_openreq6(struct seq_file *seq,
2140                          const struct request_sock *req, int i)
2141 {
2142         long ttd = req->rsk_timer.expires - jiffies;
2143         const struct in6_addr *src = &inet_rsk(req)->ir_v6_loc_addr;
2144         const struct in6_addr *dest = &inet_rsk(req)->ir_v6_rmt_addr;
2145
2146         if (ttd < 0)
2147                 ttd = 0;
2148
2149         seq_printf(seq,
2150                    "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2151                    "%02X %08X:%08X %02X:%08lX %08X %5u %8d %d %d %pK\n",
2152                    i,
2153                    src->s6_addr32[0], src->s6_addr32[1],
2154                    src->s6_addr32[2], src->s6_addr32[3],
2155                    inet_rsk(req)->ir_num,
2156                    dest->s6_addr32[0], dest->s6_addr32[1],
2157                    dest->s6_addr32[2], dest->s6_addr32[3],
2158                    ntohs(inet_rsk(req)->ir_rmt_port),
2159                    TCP_SYN_RECV,
2160                    0, 0, /* could print option size, but that is af dependent. */
2161                    1,   /* timers active (only the expire timer) */
2162                    jiffies_to_clock_t(ttd),
2163                    req->num_timeout,
2164                    from_kuid_munged(seq_user_ns(seq),
2165                                     sock_i_uid(req->rsk_listener)),
2166                    0,  /* non standard timer */
2167                    0, /* open_requests have no inode */
2168                    0, req);
2169 }
2170
2171 static void get_tcp6_sock(struct seq_file *seq, struct sock *sp, int i)
2172 {
2173         const struct in6_addr *dest, *src;
2174         __u16 destp, srcp;
2175         int timer_active;
2176         unsigned long timer_expires;
2177         const struct inet_sock *inet = inet_sk(sp);
2178         const struct tcp_sock *tp = tcp_sk(sp);
2179         const struct inet_connection_sock *icsk = inet_csk(sp);
2180         const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq;
2181         u8 icsk_pending;
2182         int rx_queue;
2183         int state;
2184
2185         dest  = &sp->sk_v6_daddr;
2186         src   = &sp->sk_v6_rcv_saddr;
2187         destp = ntohs(inet->inet_dport);
2188         srcp  = ntohs(inet->inet_sport);
2189
2190         icsk_pending = smp_load_acquire(&icsk->icsk_pending);
2191         if (icsk_pending == ICSK_TIME_RETRANS ||
2192             icsk_pending == ICSK_TIME_REO_TIMEOUT ||
2193             icsk_pending == ICSK_TIME_LOSS_PROBE) {
2194                 timer_active    = 1;
2195                 timer_expires   = icsk->icsk_timeout;
2196         } else if (icsk_pending == ICSK_TIME_PROBE0) {
2197                 timer_active    = 4;
2198                 timer_expires   = icsk->icsk_timeout;
2199         } else if (timer_pending(&sp->sk_timer)) {
2200                 timer_active    = 2;
2201                 timer_expires   = sp->sk_timer.expires;
2202         } else {
2203                 timer_active    = 0;
2204                 timer_expires = jiffies;
2205         }
2206
2207         state = inet_sk_state_load(sp);
2208         if (state == TCP_LISTEN)
2209                 rx_queue = READ_ONCE(sp->sk_ack_backlog);
2210         else
2211                 /* Because we don't lock the socket,
2212                  * we might find a transient negative value.
2213                  */
2214                 rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) -
2215                                       READ_ONCE(tp->copied_seq), 0);
2216
2217         seq_printf(seq,
2218                    "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2219                    "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %lu %lu %u %u %d\n",
2220                    i,
2221                    src->s6_addr32[0], src->s6_addr32[1],
2222                    src->s6_addr32[2], src->s6_addr32[3], srcp,
2223                    dest->s6_addr32[0], dest->s6_addr32[1],
2224                    dest->s6_addr32[2], dest->s6_addr32[3], destp,
2225                    state,
2226                    READ_ONCE(tp->write_seq) - tp->snd_una,
2227                    rx_queue,
2228                    timer_active,
2229                    jiffies_delta_to_clock_t(timer_expires - jiffies),
2230                    icsk->icsk_retransmits,
2231                    from_kuid_munged(seq_user_ns(seq), sock_i_uid(sp)),
2232                    icsk->icsk_probes_out,
2233                    sock_i_ino(sp),
2234                    refcount_read(&sp->sk_refcnt), sp,
2235                    jiffies_to_clock_t(icsk->icsk_rto),
2236                    jiffies_to_clock_t(icsk->icsk_ack.ato),
2237                    (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sp),
2238                    tcp_snd_cwnd(tp),
2239                    state == TCP_LISTEN ?
2240                         fastopenq->max_qlen :
2241                         (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh)
2242                    );
2243 }
2244
2245 static void get_timewait6_sock(struct seq_file *seq,
2246                                struct inet_timewait_sock *tw, int i)
2247 {
2248         long delta = tw->tw_timer.expires - jiffies;
2249         const struct in6_addr *dest, *src;
2250         __u16 destp, srcp;
2251
2252         dest = &tw->tw_v6_daddr;
2253         src  = &tw->tw_v6_rcv_saddr;
2254         destp = ntohs(tw->tw_dport);
2255         srcp  = ntohs(tw->tw_sport);
2256
2257         seq_printf(seq,
2258                    "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2259                    "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n",
2260                    i,
2261                    src->s6_addr32[0], src->s6_addr32[1],
2262                    src->s6_addr32[2], src->s6_addr32[3], srcp,
2263                    dest->s6_addr32[0], dest->s6_addr32[1],
2264                    dest->s6_addr32[2], dest->s6_addr32[3], destp,
2265                    READ_ONCE(tw->tw_substate), 0, 0,
2266                    3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
2267                    refcount_read(&tw->tw_refcnt), tw);
2268 }
2269
2270 static int tcp6_seq_show(struct seq_file *seq, void *v)
2271 {
2272         struct tcp_iter_state *st;
2273         struct sock *sk = v;
2274
2275         if (v == SEQ_START_TOKEN) {
2276                 seq_puts(seq,
2277                          "  sl  "
2278                          "local_address                         "
2279                          "remote_address                        "
2280                          "st tx_queue rx_queue tr tm->when retrnsmt"
2281                          "   uid  timeout inode\n");
2282                 goto out;
2283         }
2284         st = seq->private;
2285
2286         if (sk->sk_state == TCP_TIME_WAIT)
2287                 get_timewait6_sock(seq, v, st->num);
2288         else if (sk->sk_state == TCP_NEW_SYN_RECV)
2289                 get_openreq6(seq, v, st->num);
2290         else
2291                 get_tcp6_sock(seq, v, st->num);
2292 out:
2293         return 0;
2294 }
2295
2296 static const struct seq_operations tcp6_seq_ops = {
2297         .show           = tcp6_seq_show,
2298         .start          = tcp_seq_start,
2299         .next           = tcp_seq_next,
2300         .stop           = tcp_seq_stop,
2301 };
2302
2303 static struct tcp_seq_afinfo tcp6_seq_afinfo = {
2304         .family         = AF_INET6,
2305 };
2306
2307 int __net_init tcp6_proc_init(struct net *net)
2308 {
2309         if (!proc_create_net_data("tcp6", 0444, net->proc_net, &tcp6_seq_ops,
2310                         sizeof(struct tcp_iter_state), &tcp6_seq_afinfo))
2311                 return -ENOMEM;
2312         return 0;
2313 }
2314
2315 void tcp6_proc_exit(struct net *net)
2316 {
2317         remove_proc_entry("tcp6", net->proc_net);
2318 }
2319 #endif
2320
2321 struct proto tcpv6_prot = {
2322         .name                   = "TCPv6",
2323         .owner                  = THIS_MODULE,
2324         .close                  = tcp_close,
2325         .pre_connect            = tcp_v6_pre_connect,
2326         .connect                = tcp_v6_connect,
2327         .disconnect             = tcp_disconnect,
2328         .accept                 = inet_csk_accept,
2329         .ioctl                  = tcp_ioctl,
2330         .init                   = tcp_v6_init_sock,
2331         .destroy                = tcp_v4_destroy_sock,
2332         .shutdown               = tcp_shutdown,
2333         .setsockopt             = tcp_setsockopt,
2334         .getsockopt             = tcp_getsockopt,
2335         .bpf_bypass_getsockopt  = tcp_bpf_bypass_getsockopt,
2336         .keepalive              = tcp_set_keepalive,
2337         .recvmsg                = tcp_recvmsg,
2338         .sendmsg                = tcp_sendmsg,
2339         .splice_eof             = tcp_splice_eof,
2340         .backlog_rcv            = tcp_v6_do_rcv,
2341         .release_cb             = tcp_release_cb,
2342         .hash                   = inet6_hash,
2343         .unhash                 = inet_unhash,
2344         .get_port               = inet_csk_get_port,
2345         .put_port               = inet_put_port,
2346 #ifdef CONFIG_BPF_SYSCALL
2347         .psock_update_sk_prot   = tcp_bpf_update_proto,
2348 #endif
2349         .enter_memory_pressure  = tcp_enter_memory_pressure,
2350         .leave_memory_pressure  = tcp_leave_memory_pressure,
2351         .stream_memory_free     = tcp_stream_memory_free,
2352         .sockets_allocated      = &tcp_sockets_allocated,
2353
2354         .memory_allocated       = &tcp_memory_allocated,
2355         .per_cpu_fw_alloc       = &tcp_memory_per_cpu_fw_alloc,
2356
2357         .memory_pressure        = &tcp_memory_pressure,
2358         .orphan_count           = &tcp_orphan_count,
2359         .sysctl_mem             = sysctl_tcp_mem,
2360         .sysctl_wmem_offset     = offsetof(struct net, ipv4.sysctl_tcp_wmem),
2361         .sysctl_rmem_offset     = offsetof(struct net, ipv4.sysctl_tcp_rmem),
2362         .max_header             = MAX_TCP_HEADER,
2363         .obj_size               = sizeof(struct tcp6_sock),
2364         .ipv6_pinfo_offset = offsetof(struct tcp6_sock, inet6),
2365         .slab_flags             = SLAB_TYPESAFE_BY_RCU,
2366         .twsk_prot              = &tcp6_timewait_sock_ops,
2367         .rsk_prot               = &tcp6_request_sock_ops,
2368         .h.hashinfo             = NULL,
2369         .no_autobind            = true,
2370         .diag_destroy           = tcp_abort,
2371 };
2372 EXPORT_SYMBOL_GPL(tcpv6_prot);
2373
2374
2375 static struct inet_protosw tcpv6_protosw = {
2376         .type           =       SOCK_STREAM,
2377         .protocol       =       IPPROTO_TCP,
2378         .prot           =       &tcpv6_prot,
2379         .ops            =       &inet6_stream_ops,
2380         .flags          =       INET_PROTOSW_PERMANENT |
2381                                 INET_PROTOSW_ICSK,
2382 };
2383
2384 static int __net_init tcpv6_net_init(struct net *net)
2385 {
2386         int res;
2387
2388         res = inet_ctl_sock_create(&net->ipv6.tcp_sk, PF_INET6,
2389                                    SOCK_RAW, IPPROTO_TCP, net);
2390         if (!res)
2391                 net->ipv6.tcp_sk->sk_clockid = CLOCK_MONOTONIC;
2392
2393         return res;
2394 }
2395
2396 static void __net_exit tcpv6_net_exit(struct net *net)
2397 {
2398         inet_ctl_sock_destroy(net->ipv6.tcp_sk);
2399 }
2400
2401 static struct pernet_operations tcpv6_net_ops = {
2402         .init       = tcpv6_net_init,
2403         .exit       = tcpv6_net_exit,
2404 };
2405
2406 int __init tcpv6_init(void)
2407 {
2408         int ret;
2409
2410         net_hotdata.tcpv6_protocol = (struct inet6_protocol) {
2411                 .handler     = tcp_v6_rcv,
2412                 .err_handler = tcp_v6_err,
2413                 .flags       = INET6_PROTO_NOPOLICY | INET6_PROTO_FINAL,
2414         };
2415         ret = inet6_add_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
2416         if (ret)
2417                 goto out;
2418
2419         /* register inet6 protocol */
2420         ret = inet6_register_protosw(&tcpv6_protosw);
2421         if (ret)
2422                 goto out_tcpv6_protocol;
2423
2424         ret = register_pernet_subsys(&tcpv6_net_ops);
2425         if (ret)
2426                 goto out_tcpv6_protosw;
2427
2428         ret = mptcpv6_init();
2429         if (ret)
2430                 goto out_tcpv6_pernet_subsys;
2431
2432 out:
2433         return ret;
2434
2435 out_tcpv6_pernet_subsys:
2436         unregister_pernet_subsys(&tcpv6_net_ops);
2437 out_tcpv6_protosw:
2438         inet6_unregister_protosw(&tcpv6_protosw);
2439 out_tcpv6_protocol:
2440         inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
2441         goto out;
2442 }
2443
2444 void tcpv6_exit(void)
2445 {
2446         unregister_pernet_subsys(&tcpv6_net_ops);
2447         inet6_unregister_protosw(&tcpv6_protosw);
2448         inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
2449 }