net/ipv4/tcp_ao.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * INET         An implementation of the TCP Authentication Option (TCP-AO).
   4  *              See RFC5925.
   5  *
   6  * Authors:     Dmitry Safonov <[email protected]>
   7  *              Francesco Ruggeri <[email protected]>
   8  *              Salam Noureddine <[email protected]>
   9  */
  10 #define pr_fmt(fmt) "TCP: " fmt
  11
  12 #include <crypto/hash.h>
  13 #include <linux/inetdevice.h>
  14 #include <linux/tcp.h>
  15
  16 #include <net/tcp.h>
  17 #include <net/ipv6.h>
  18 #include <net/icmp.h>
  19 #include <trace/events/tcp.h>
  20
  21 DEFINE_STATIC_KEY_DEFERRED_FALSE(tcp_ao_needed, HZ);
  22
  23 int tcp_ao_calc_traffic_key(struct tcp_ao_key *mkt, u8 *key, void *ctx,
  24                             unsigned int len, struct tcp_sigpool *hp)
  25 {
  26         struct scatterlist sg;
  27         int ret;
  28
  29         if (crypto_ahash_setkey(crypto_ahash_reqtfm(hp->req),
  30                                 mkt->key, mkt->keylen))
  31                 goto clear_hash;
  32
  33         ret = crypto_ahash_init(hp->req);
  34         if (ret)
  35                 goto clear_hash;
  36
  37         sg_init_one(&sg, ctx, len);
  38         ahash_request_set_crypt(hp->req, &sg, key, len);
  39         crypto_ahash_update(hp->req);
  40
  41         ret = crypto_ahash_final(hp->req);
  42         if (ret)
  43                 goto clear_hash;
  44
  45         return 0;
  46 clear_hash:
  47         memset(key, 0, tcp_ao_digest_size(mkt));
  48         return 1;
  49 }
  50
  51 bool tcp_ao_ignore_icmp(const struct sock *sk, int family, int type, int code)
  52 {
  53         bool ignore_icmp = false;
  54         struct tcp_ao_info *ao;
  55
  56         if (!static_branch_unlikely(&tcp_ao_needed.key))
  57                 return false;
  58
  59         /* RFC5925, 7.8:
  60          * >> A TCP-AO implementation MUST default to ignore incoming ICMPv4
  61          * messages of Type 3 (destination unreachable), Codes 2-4 (protocol
  62          * unreachable, port unreachable, and fragmentation needed -- ’hard
  63          * errors’), and ICMPv6 Type 1 (destination unreachable), Code 1
  64          * (administratively prohibited) and Code 4 (port unreachable) intended
  65          * for connections in synchronized states (ESTABLISHED, FIN-WAIT-1, FIN-
  66          * WAIT-2, CLOSE-WAIT, CLOSING, LAST-ACK, TIME-WAIT) that match MKTs.
  67          */
  68         if (family == AF_INET) {
  69                 if (type != ICMP_DEST_UNREACH)
  70                         return false;
  71                 if (code < ICMP_PROT_UNREACH || code > ICMP_FRAG_NEEDED)
  72                         return false;
  73         } else {
  74                 if (type != ICMPV6_DEST_UNREACH)
  75                         return false;
  76                 if (code != ICMPV6_ADM_PROHIBITED && code != ICMPV6_PORT_UNREACH)
  77                         return false;
  78         }
  79
  80         rcu_read_lock();
  81         switch (sk->sk_state) {
  82         case TCP_TIME_WAIT:
  83                 ao = rcu_dereference(tcp_twsk(sk)->ao_info);
  84                 break;
  85         case TCP_SYN_SENT:
  86         case TCP_SYN_RECV:
  87         case TCP_LISTEN:
  88         case TCP_NEW_SYN_RECV:
  89                 /* RFC5925 specifies to ignore ICMPs *only* on connections
  90                  * in synchronized states.
  91                  */
  92                 rcu_read_unlock();
  93                 return false;
  94         default:
  95                 ao = rcu_dereference(tcp_sk(sk)->ao_info);
  96         }
  97
  98         if (ao && !ao->accept_icmps) {
  99                 ignore_icmp = true;
 100                 __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAODROPPEDICMPS);
 101                 atomic64_inc(&ao->counters.dropped_icmp);
 102         }
 103         rcu_read_unlock();
 104
 105         return ignore_icmp;
 106 }
 107
 108 /* Optimized version of tcp_ao_do_lookup(): only for sockets for which
 109  * it's known that the keys in ao_info are matching peer's
 110  * family/address/VRF/etc.
 111  */
 112 struct tcp_ao_key *tcp_ao_established_key(struct tcp_ao_info *ao,
 113                                           int sndid, int rcvid)
 114 {
 115         struct tcp_ao_key *key;
 116
 117         hlist_for_each_entry_rcu(key, &ao->head, node) {
 118                 if ((sndid >= 0 && key->sndid != sndid) ||
 119                     (rcvid >= 0 && key->rcvid != rcvid))
 120                         continue;
 121                 return key;
 122         }
 123
 124         return NULL;
 125 }
 126
 127 static int ipv4_prefix_cmp(const struct in_addr *addr1,
 128                            const struct in_addr *addr2,
 129                            unsigned int prefixlen)
 130 {
 131         __be32 mask = inet_make_mask(prefixlen);
 132         __be32 a1 = addr1->s_addr & mask;
 133         __be32 a2 = addr2->s_addr & mask;
 134
 135         if (a1 == a2)
 136                 return 0;
 137         return memcmp(&a1, &a2, sizeof(a1));
 138 }
 139
 140 static int __tcp_ao_key_cmp(const struct tcp_ao_key *key, int l3index,
 141                             const union tcp_ao_addr *addr, u8 prefixlen,
 142                             int family, int sndid, int rcvid)
 143 {
 144         if (sndid >= 0 && key->sndid != sndid)
 145                 return (key->sndid > sndid) ? 1 : -1;
 146         if (rcvid >= 0 && key->rcvid != rcvid)
 147                 return (key->rcvid > rcvid) ? 1 : -1;
 148         if (l3index >= 0 && (key->keyflags & TCP_AO_KEYF_IFINDEX)) {
 149                 if (key->l3index != l3index)
 150                         return (key->l3index > l3index) ? 1 : -1;
 151         }
 152
 153         if (family == AF_UNSPEC)
 154                 return 0;
 155         if (key->family != family)
 156                 return (key->family > family) ? 1 : -1;
 157
 158         if (family == AF_INET) {
 159                 if (ntohl(key->addr.a4.s_addr) == INADDR_ANY)
 160                         return 0;
 161                 if (ntohl(addr->a4.s_addr) == INADDR_ANY)
 162                         return 0;
 163                 return ipv4_prefix_cmp(&key->addr.a4, &addr->a4, prefixlen);
 164 #if IS_ENABLED(CONFIG_IPV6)
 165         } else {
 166                 if (ipv6_addr_any(&key->addr.a6) || ipv6_addr_any(&addr->a6))
 167                         return 0;
 168                 if (ipv6_prefix_equal(&key->addr.a6, &addr->a6, prefixlen))
 169                         return 0;
 170                 return memcmp(&key->addr.a6, &addr->a6, sizeof(addr->a6));
 171 #endif
 172         }
 173         return -1;
 174 }
 175
 176 static int tcp_ao_key_cmp(const struct tcp_ao_key *key, int l3index,
 177                           const union tcp_ao_addr *addr, u8 prefixlen,
 178                           int family, int sndid, int rcvid)
 179 {
 180 #if IS_ENABLED(CONFIG_IPV6)
 181         if (family == AF_INET6 && ipv6_addr_v4mapped(&addr->a6)) {
 182                 __be32 addr4 = addr->a6.s6_addr32[3];
 183
 184                 return __tcp_ao_key_cmp(key, l3index,
 185                                         (union tcp_ao_addr *)&addr4,
 186                                         prefixlen, AF_INET, sndid, rcvid);
 187         }
 188 #endif
 189         return __tcp_ao_key_cmp(key, l3index, addr,
 190                                 prefixlen, family, sndid, rcvid);
 191 }
 192
 193 static struct tcp_ao_key *__tcp_ao_do_lookup(const struct sock *sk, int l3index,
 194                 const union tcp_ao_addr *addr, int family, u8 prefix,
 195                 int sndid, int rcvid)
 196 {
 197         struct tcp_ao_key *key;
 198         struct tcp_ao_info *ao;
 199
 200         if (!static_branch_unlikely(&tcp_ao_needed.key))
 201                 return NULL;
 202
 203         ao = rcu_dereference_check(tcp_sk(sk)->ao_info,
 204                                    lockdep_sock_is_held(sk));
 205         if (!ao)
 206                 return NULL;
 207
 208         hlist_for_each_entry_rcu(key, &ao->head, node) {
 209                 u8 prefixlen = min(prefix, key->prefixlen);
 210
 211                 if (!tcp_ao_key_cmp(key, l3index, addr, prefixlen,
 212                                     family, sndid, rcvid))
 213                         return key;
 214         }
 215         return NULL;
 216 }
 217
 218 struct tcp_ao_key *tcp_ao_do_lookup(const struct sock *sk, int l3index,
 219                                     const union tcp_ao_addr *addr,
 220                                     int family, int sndid, int rcvid)
 221 {
 222         return __tcp_ao_do_lookup(sk, l3index, addr, family, U8_MAX, sndid, rcvid);
 223 }
 224
 225 static struct tcp_ao_info *tcp_ao_alloc_info(gfp_t flags)
 226 {
 227         struct tcp_ao_info *ao;
 228
 229         ao = kzalloc(sizeof(*ao), flags);
 230         if (!ao)
 231                 return NULL;
 232         INIT_HLIST_HEAD(&ao->head);
 233         refcount_set(&ao->refcnt, 1);
 234
 235         return ao;
 236 }
 237
 238 static void tcp_ao_link_mkt(struct tcp_ao_info *ao, struct tcp_ao_key *mkt)
 239 {
 240         hlist_add_head_rcu(&mkt->node, &ao->head);
 241 }
 242
 243 static struct tcp_ao_key *tcp_ao_copy_key(struct sock *sk,
 244                                           struct tcp_ao_key *key)
 245 {
 246         struct tcp_ao_key *new_key;
 247
 248         new_key = sock_kmalloc(sk, tcp_ao_sizeof_key(key),
 249                                GFP_ATOMIC);
 250         if (!new_key)
 251                 return NULL;
 252
 253         *new_key = *key;
 254         INIT_HLIST_NODE(&new_key->node);
 255         tcp_sigpool_get(new_key->tcp_sigpool_id);
 256         atomic64_set(&new_key->pkt_good, 0);
 257         atomic64_set(&new_key->pkt_bad, 0);
 258
 259         return new_key;
 260 }
 261
 262 static void tcp_ao_key_free_rcu(struct rcu_head *head)
 263 {
 264         struct tcp_ao_key *key = container_of(head, struct tcp_ao_key, rcu);
 265
 266         tcp_sigpool_release(key->tcp_sigpool_id);
 267         kfree_sensitive(key);
 268 }
 269
 270 static void tcp_ao_info_free_rcu(struct rcu_head *head)
 271 {
 272         struct tcp_ao_info *ao = container_of(head, struct tcp_ao_info, rcu);
 273         struct tcp_ao_key *key;
 274         struct hlist_node *n;
 275
 276         hlist_for_each_entry_safe(key, n, &ao->head, node) {
 277                 hlist_del(&key->node);
 278                 tcp_sigpool_release(key->tcp_sigpool_id);
 279                 kfree_sensitive(key);
 280         }
 281         kfree(ao);
 282         static_branch_slow_dec_deferred(&tcp_ao_needed);
 283 }
 284
 285 static void tcp_ao_sk_omem_free(struct sock *sk, struct tcp_ao_info *ao)
 286 {
 287         size_t total_ao_sk_mem = 0;
 288         struct tcp_ao_key *key;
 289
 290         hlist_for_each_entry(key,  &ao->head, node)
 291                 total_ao_sk_mem += tcp_ao_sizeof_key(key);
 292         atomic_sub(total_ao_sk_mem, &sk->sk_omem_alloc);
 293 }
 294
 295 void tcp_ao_destroy_sock(struct sock *sk, bool twsk)
 296 {
 297         struct tcp_ao_info *ao;
 298
 299         if (twsk) {
 300                 ao = rcu_dereference_protected(tcp_twsk(sk)->ao_info, 1);
 301                 rcu_assign_pointer(tcp_twsk(sk)->ao_info, NULL);
 302         } else {
 303                 ao = rcu_dereference_protected(tcp_sk(sk)->ao_info, 1);
 304                 rcu_assign_pointer(tcp_sk(sk)->ao_info, NULL);
 305         }
 306
 307         if (!ao || !refcount_dec_and_test(&ao->refcnt))
 308                 return;
 309
 310         if (!twsk)
 311                 tcp_ao_sk_omem_free(sk, ao);
 312         call_rcu(&ao->rcu, tcp_ao_info_free_rcu);
 313 }
 314
 315 void tcp_ao_time_wait(struct tcp_timewait_sock *tcptw, struct tcp_sock *tp)
 316 {
 317         struct tcp_ao_info *ao_info = rcu_dereference_protected(tp->ao_info, 1);
 318
 319         if (ao_info) {
 320                 struct tcp_ao_key *key;
 321                 struct hlist_node *n;
 322                 int omem = 0;
 323
 324                 hlist_for_each_entry_safe(key, n, &ao_info->head, node) {
 325                         omem += tcp_ao_sizeof_key(key);
 326                 }
 327
 328                 refcount_inc(&ao_info->refcnt);
 329                 atomic_sub(omem, &(((struct sock *)tp)->sk_omem_alloc));
 330                 rcu_assign_pointer(tcptw->ao_info, ao_info);
 331         } else {
 332                 tcptw->ao_info = NULL;
 333         }
 334 }
 335
 336 /* 4 tuple and ISNs are expected in NBO */
 337 static int tcp_v4_ao_calc_key(struct tcp_ao_key *mkt, u8 *key,
 338                               __be32 saddr, __be32 daddr,
 339                               __be16 sport, __be16 dport,
 340                               __be32 sisn,  __be32 disn)
 341 {
 342         /* See RFC5926 3.1.1 */
 343         struct kdf_input_block {
 344                 u8                      counter;
 345                 u8                      label[6];
 346                 struct tcp4_ao_context  ctx;
 347                 __be16                  outlen;
 348         } __packed * tmp;
 349         struct tcp_sigpool hp;
 350         int err;
 351
 352         err = tcp_sigpool_start(mkt->tcp_sigpool_id, &hp);
 353         if (err)
 354                 return err;
 355
 356         tmp = hp.scratch;
 357         tmp->counter    = 1;
 358         memcpy(tmp->label, "TCP-AO", 6);
 359         tmp->ctx.saddr  = saddr;
 360         tmp->ctx.daddr  = daddr;
 361         tmp->ctx.sport  = sport;
 362         tmp->ctx.dport  = dport;
 363         tmp->ctx.sisn   = sisn;
 364         tmp->ctx.disn   = disn;
 365         tmp->outlen     = htons(tcp_ao_digest_size(mkt) * 8); /* in bits */
 366
 367         err = tcp_ao_calc_traffic_key(mkt, key, tmp, sizeof(*tmp), &hp);
 368         tcp_sigpool_end(&hp);
 369
 370         return err;
 371 }
 372
 373 int tcp_v4_ao_calc_key_sk(struct tcp_ao_key *mkt, u8 *key,
 374                           const struct sock *sk,
 375                           __be32 sisn, __be32 disn, bool send)
 376 {
 377         if (send)
 378                 return tcp_v4_ao_calc_key(mkt, key, sk->sk_rcv_saddr,
 379                                           sk->sk_daddr, htons(sk->sk_num),
 380                                           sk->sk_dport, sisn, disn);
 381         else
 382                 return tcp_v4_ao_calc_key(mkt, key, sk->sk_daddr,
 383                                           sk->sk_rcv_saddr, sk->sk_dport,
 384                                           htons(sk->sk_num), disn, sisn);
 385 }
 386
 387 static int tcp_ao_calc_key_sk(struct tcp_ao_key *mkt, u8 *key,
 388                               const struct sock *sk,
 389                               __be32 sisn, __be32 disn, bool send)
 390 {
 391         if (mkt->family == AF_INET)
 392                 return tcp_v4_ao_calc_key_sk(mkt, key, sk, sisn, disn, send);
 393 #if IS_ENABLED(CONFIG_IPV6)
 394         else if (mkt->family == AF_INET6)
 395                 return tcp_v6_ao_calc_key_sk(mkt, key, sk, sisn, disn, send);
 396 #endif
 397         else
 398                 return -EOPNOTSUPP;
 399 }
 400
 401 int tcp_v4_ao_calc_key_rsk(struct tcp_ao_key *mkt, u8 *key,
 402                            struct request_sock *req)
 403 {
 404         struct inet_request_sock *ireq = inet_rsk(req);
 405
 406         return tcp_v4_ao_calc_key(mkt, key,
 407                                   ireq->ir_loc_addr, ireq->ir_rmt_addr,
 408                                   htons(ireq->ir_num), ireq->ir_rmt_port,
 409                                   htonl(tcp_rsk(req)->snt_isn),
 410                                   htonl(tcp_rsk(req)->rcv_isn));
 411 }
 412
 413 static int tcp_v4_ao_calc_key_skb(struct tcp_ao_key *mkt, u8 *key,
 414                                   const struct sk_buff *skb,
 415                                   __be32 sisn, __be32 disn)
 416 {
 417         const struct iphdr *iph = ip_hdr(skb);
 418         const struct tcphdr *th = tcp_hdr(skb);
 419
 420         return tcp_v4_ao_calc_key(mkt, key, iph->saddr, iph->daddr,
 421                                   th->source, th->dest, sisn, disn);
 422 }
 423
 424 static int tcp_ao_calc_key_skb(struct tcp_ao_key *mkt, u8 *key,
 425                                const struct sk_buff *skb,
 426                                __be32 sisn, __be32 disn, int family)
 427 {
 428         if (family == AF_INET)
 429                 return tcp_v4_ao_calc_key_skb(mkt, key, skb, sisn, disn);
 430 #if IS_ENABLED(CONFIG_IPV6)
 431         else if (family == AF_INET6)
 432                 return tcp_v6_ao_calc_key_skb(mkt, key, skb, sisn, disn);
 433 #endif
 434         return -EAFNOSUPPORT;
 435 }
 436
 437 static int tcp_v4_ao_hash_pseudoheader(struct tcp_sigpool *hp,
 438                                        __be32 daddr, __be32 saddr,
 439                                        int nbytes)
 440 {
 441         struct tcp4_pseudohdr *bp;
 442         struct scatterlist sg;
 443
 444         bp = hp->scratch;
 445         bp->saddr = saddr;
 446         bp->daddr = daddr;
 447         bp->pad = 0;
 448         bp->protocol = IPPROTO_TCP;
 449         bp->len = cpu_to_be16(nbytes);
 450
 451         sg_init_one(&sg, bp, sizeof(*bp));
 452         ahash_request_set_crypt(hp->req, &sg, NULL, sizeof(*bp));
 453         return crypto_ahash_update(hp->req);
 454 }
 455
 456 static int tcp_ao_hash_pseudoheader(unsigned short int family,
 457                                     const struct sock *sk,
 458                                     const struct sk_buff *skb,
 459                                     struct tcp_sigpool *hp, int nbytes)
 460 {
 461         const struct tcphdr *th = tcp_hdr(skb);
 462
 463         /* TODO: Can we rely on checksum being zero to mean outbound pkt? */
 464         if (!th->check) {
 465                 if (family == AF_INET)
 466                         return tcp_v4_ao_hash_pseudoheader(hp, sk->sk_daddr,
 467                                         sk->sk_rcv_saddr, skb->len);
 468 #if IS_ENABLED(CONFIG_IPV6)
 469                 else if (family == AF_INET6)
 470                         return tcp_v6_ao_hash_pseudoheader(hp, &sk->sk_v6_daddr,
 471                                         &sk->sk_v6_rcv_saddr, skb->len);
 472 #endif
 473                 else
 474                         return -EAFNOSUPPORT;
 475         }
 476
 477         if (family == AF_INET) {
 478                 const struct iphdr *iph = ip_hdr(skb);
 479
 480                 return tcp_v4_ao_hash_pseudoheader(hp, iph->daddr,
 481                                 iph->saddr, skb->len);
 482 #if IS_ENABLED(CONFIG_IPV6)
 483         } else if (family == AF_INET6) {
 484                 const struct ipv6hdr *iph = ipv6_hdr(skb);
 485
 486                 return tcp_v6_ao_hash_pseudoheader(hp, &iph->daddr,
 487                                 &iph->saddr, skb->len);
 488 #endif
 489         }
 490         return -EAFNOSUPPORT;
 491 }
 492
 493 u32 tcp_ao_compute_sne(u32 next_sne, u32 next_seq, u32 seq)
 494 {
 495         u32 sne = next_sne;
 496
 497         if (before(seq, next_seq)) {
 498                 if (seq > next_seq)
 499                         sne--;
 500         } else {
 501                 if (seq < next_seq)
 502                         sne++;
 503         }
 504
 505         return sne;
 506 }
 507
 508 /* tcp_ao_hash_sne(struct tcp_sigpool *hp)
 509  * @hp  - used for hashing
 510  * @sne - sne value
 511  */
 512 static int tcp_ao_hash_sne(struct tcp_sigpool *hp, u32 sne)
 513 {
 514         struct scatterlist sg;
 515         __be32 *bp;
 516
 517         bp = (__be32 *)hp->scratch;
 518         *bp = htonl(sne);
 519
 520         sg_init_one(&sg, bp, sizeof(*bp));
 521         ahash_request_set_crypt(hp->req, &sg, NULL, sizeof(*bp));
 522         return crypto_ahash_update(hp->req);
 523 }
 524
 525 static int tcp_ao_hash_header(struct tcp_sigpool *hp,
 526                               const struct tcphdr *th,
 527                               bool exclude_options, u8 *hash,
 528                               int hash_offset, int hash_len)
 529 {
 530         struct scatterlist sg;
 531         u8 *hdr = hp->scratch;
 532         int err, len;
 533
 534         /* We are not allowed to change tcphdr, make a local copy */
 535         if (exclude_options) {
 536                 len = sizeof(*th) + sizeof(struct tcp_ao_hdr) + hash_len;
 537                 memcpy(hdr, th, sizeof(*th));
 538                 memcpy(hdr + sizeof(*th),
 539                        (u8 *)th + hash_offset - sizeof(struct tcp_ao_hdr),
 540                        sizeof(struct tcp_ao_hdr));
 541                 memset(hdr + sizeof(*th) + sizeof(struct tcp_ao_hdr),
 542                        0, hash_len);
 543                 ((struct tcphdr *)hdr)->check = 0;
 544         } else {
 545                 len = th->doff << 2;
 546                 memcpy(hdr, th, len);
 547                 /* zero out tcp-ao hash */
 548                 ((struct tcphdr *)hdr)->check = 0;
 549                 memset(hdr + hash_offset, 0, hash_len);
 550         }
 551
 552         sg_init_one(&sg, hdr, len);
 553         ahash_request_set_crypt(hp->req, &sg, NULL, len);
 554         err = crypto_ahash_update(hp->req);
 555         WARN_ON_ONCE(err != 0);
 556         return err;
 557 }
 558
 559 int tcp_ao_hash_hdr(unsigned short int family, char *ao_hash,
 560                     struct tcp_ao_key *key, const u8 *tkey,
 561                     const union tcp_ao_addr *daddr,
 562                     const union tcp_ao_addr *saddr,
 563                     const struct tcphdr *th, u32 sne)
 564 {
 565         int tkey_len = tcp_ao_digest_size(key);
 566         int hash_offset = ao_hash - (char *)th;
 567         struct tcp_sigpool hp;
 568         void *hash_buf = NULL;
 569
 570         hash_buf = kmalloc(tkey_len, GFP_ATOMIC);
 571         if (!hash_buf)
 572                 goto clear_hash_noput;
 573
 574         if (tcp_sigpool_start(key->tcp_sigpool_id, &hp))
 575                 goto clear_hash_noput;
 576
 577         if (crypto_ahash_setkey(crypto_ahash_reqtfm(hp.req), tkey, tkey_len))
 578                 goto clear_hash;
 579
 580         if (crypto_ahash_init(hp.req))
 581                 goto clear_hash;
 582
 583         if (tcp_ao_hash_sne(&hp, sne))
 584                 goto clear_hash;
 585         if (family == AF_INET) {
 586                 if (tcp_v4_ao_hash_pseudoheader(&hp, daddr->a4.s_addr,
 587                                                 saddr->a4.s_addr, th->doff * 4))
 588                         goto clear_hash;
 589 #if IS_ENABLED(CONFIG_IPV6)
 590         } else if (family == AF_INET6) {
 591                 if (tcp_v6_ao_hash_pseudoheader(&hp, &daddr->a6,
 592                                                 &saddr->a6, th->doff * 4))
 593                         goto clear_hash;
 594 #endif
 595         } else {
 596                 WARN_ON_ONCE(1);
 597                 goto clear_hash;
 598         }
 599         if (tcp_ao_hash_header(&hp, th,
 600                                !!(key->keyflags & TCP_AO_KEYF_EXCLUDE_OPT),
 601                                ao_hash, hash_offset, tcp_ao_maclen(key)))
 602                 goto clear_hash;
 603         ahash_request_set_crypt(hp.req, NULL, hash_buf, 0);
 604         if (crypto_ahash_final(hp.req))
 605                 goto clear_hash;
 606
 607         memcpy(ao_hash, hash_buf, tcp_ao_maclen(key));
 608         tcp_sigpool_end(&hp);
 609         kfree(hash_buf);
 610         return 0;
 611
 612 clear_hash:
 613         tcp_sigpool_end(&hp);
 614 clear_hash_noput:
 615         memset(ao_hash, 0, tcp_ao_maclen(key));
 616         kfree(hash_buf);
 617         return 1;
 618 }
 619
 620 int tcp_ao_hash_skb(unsigned short int family,
 621                     char *ao_hash, struct tcp_ao_key *key,
 622                     const struct sock *sk, const struct sk_buff *skb,
 623                     const u8 *tkey, int hash_offset, u32 sne)
 624 {
 625         const struct tcphdr *th = tcp_hdr(skb);
 626         int tkey_len = tcp_ao_digest_size(key);
 627         struct tcp_sigpool hp;
 628         void *hash_buf = NULL;
 629
 630         hash_buf = kmalloc(tkey_len, GFP_ATOMIC);
 631         if (!hash_buf)
 632                 goto clear_hash_noput;
 633
 634         if (tcp_sigpool_start(key->tcp_sigpool_id, &hp))
 635                 goto clear_hash_noput;
 636
 637         if (crypto_ahash_setkey(crypto_ahash_reqtfm(hp.req), tkey, tkey_len))
 638                 goto clear_hash;
 639
 640         /* For now use sha1 by default. Depends on alg in tcp_ao_key */
 641         if (crypto_ahash_init(hp.req))
 642                 goto clear_hash;
 643
 644         if (tcp_ao_hash_sne(&hp, sne))
 645                 goto clear_hash;
 646         if (tcp_ao_hash_pseudoheader(family, sk, skb, &hp, skb->len))
 647                 goto clear_hash;
 648         if (tcp_ao_hash_header(&hp, th,
 649                                !!(key->keyflags & TCP_AO_KEYF_EXCLUDE_OPT),
 650                                ao_hash, hash_offset, tcp_ao_maclen(key)))
 651                 goto clear_hash;
 652         if (tcp_sigpool_hash_skb_data(&hp, skb, th->doff << 2))
 653                 goto clear_hash;
 654         ahash_request_set_crypt(hp.req, NULL, hash_buf, 0);
 655         if (crypto_ahash_final(hp.req))
 656                 goto clear_hash;
 657
 658         memcpy(ao_hash, hash_buf, tcp_ao_maclen(key));
 659         tcp_sigpool_end(&hp);
 660         kfree(hash_buf);
 661         return 0;
 662
 663 clear_hash:
 664         tcp_sigpool_end(&hp);
 665 clear_hash_noput:
 666         memset(ao_hash, 0, tcp_ao_maclen(key));
 667         kfree(hash_buf);
 668         return 1;
 669 }
 670
 671 int tcp_v4_ao_hash_skb(char *ao_hash, struct tcp_ao_key *key,
 672                        const struct sock *sk, const struct sk_buff *skb,
 673                        const u8 *tkey, int hash_offset, u32 sne)
 674 {
 675         return tcp_ao_hash_skb(AF_INET, ao_hash, key, sk, skb,
 676                                tkey, hash_offset, sne);
 677 }
 678
 679 int tcp_v4_ao_synack_hash(char *ao_hash, struct tcp_ao_key *ao_key,
 680                           struct request_sock *req, const struct sk_buff *skb,
 681                           int hash_offset, u32 sne)
 682 {
 683         void *hash_buf = NULL;
 684         int err;
 685
 686         hash_buf = kmalloc(tcp_ao_digest_size(ao_key), GFP_ATOMIC);
 687         if (!hash_buf)
 688                 return -ENOMEM;
 689
 690         err = tcp_v4_ao_calc_key_rsk(ao_key, hash_buf, req);
 691         if (err)
 692                 goto out;
 693
 694         err = tcp_ao_hash_skb(AF_INET, ao_hash, ao_key, req_to_sk(req), skb,
 695                               hash_buf, hash_offset, sne);
 696 out:
 697         kfree(hash_buf);
 698         return err;
 699 }
 700
 701 struct tcp_ao_key *tcp_v4_ao_lookup_rsk(const struct sock *sk,
 702                                         struct request_sock *req,
 703                                         int sndid, int rcvid)
 704 {
 705         struct inet_request_sock *ireq = inet_rsk(req);
 706         union tcp_ao_addr *addr = (union tcp_ao_addr *)&ireq->ir_rmt_addr;
 707         int l3index;
 708
 709         l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif);
 710         return tcp_ao_do_lookup(sk, l3index, addr, AF_INET, sndid, rcvid);
 711 }
 712
 713 struct tcp_ao_key *tcp_v4_ao_lookup(const struct sock *sk, struct sock *addr_sk,
 714                                     int sndid, int rcvid)
 715 {
 716         int l3index = l3mdev_master_ifindex_by_index(sock_net(sk),
 717                                                      addr_sk->sk_bound_dev_if);
 718         union tcp_ao_addr *addr = (union tcp_ao_addr *)&addr_sk->sk_daddr;
 719
 720         return tcp_ao_do_lookup(sk, l3index, addr, AF_INET, sndid, rcvid);
 721 }
 722
 723 int tcp_ao_prepare_reset(const struct sock *sk, struct sk_buff *skb,
 724                          const struct tcp_ao_hdr *aoh, int l3index, u32 seq,
 725                          struct tcp_ao_key **key, char **traffic_key,
 726                          bool *allocated_traffic_key, u8 *keyid, u32 *sne)
 727 {
 728         const struct tcphdr *th = tcp_hdr(skb);
 729         struct tcp_ao_info *ao_info;
 730
 731         *allocated_traffic_key = false;
 732         /* If there's no socket - than initial sisn/disn are unknown.
 733          * Drop the segment. RFC5925 (7.7) advises to require graceful
 734          * restart [RFC4724]. Alternatively, the RFC5925 advises to
 735          * save/restore traffic keys before/after reboot.
 736          * Linux TCP-AO support provides TCP_AO_ADD_KEY and TCP_AO_REPAIR
 737          * options to restore a socket post-reboot.
 738          */
 739         if (!sk)
 740                 return -ENOTCONN;
 741
 742         if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_NEW_SYN_RECV)) {
 743                 unsigned int family = READ_ONCE(sk->sk_family);
 744                 union tcp_ao_addr *addr;
 745                 __be32 disn, sisn;
 746
 747                 if (sk->sk_state == TCP_NEW_SYN_RECV) {
 748                         struct request_sock *req = inet_reqsk(sk);
 749
 750                         sisn = htonl(tcp_rsk(req)->rcv_isn);
 751                         disn = htonl(tcp_rsk(req)->snt_isn);
 752                         *sne = tcp_ao_compute_sne(0, tcp_rsk(req)->snt_isn, seq);
 753                 } else {
 754                         sisn = th->seq;
 755                         disn = 0;
 756                 }
 757                 if (IS_ENABLED(CONFIG_IPV6) && family == AF_INET6)
 758                         addr = (union tcp_md5_addr *)&ipv6_hdr(skb)->saddr;
 759                 else
 760                         addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
 761 #if IS_ENABLED(CONFIG_IPV6)
 762                 if (family == AF_INET6 && ipv6_addr_v4mapped(&sk->sk_v6_daddr))
 763                         family = AF_INET;
 764 #endif
 765
 766                 sk = sk_const_to_full_sk(sk);
 767                 ao_info = rcu_dereference(tcp_sk(sk)->ao_info);
 768                 if (!ao_info)
 769                         return -ENOENT;
 770                 *key = tcp_ao_do_lookup(sk, l3index, addr, family,
 771                                         -1, aoh->rnext_keyid);
 772                 if (!*key)
 773                         return -ENOENT;
 774                 *traffic_key = kmalloc(tcp_ao_digest_size(*key), GFP_ATOMIC);
 775                 if (!*traffic_key)
 776                         return -ENOMEM;
 777                 *allocated_traffic_key = true;
 778                 if (tcp_ao_calc_key_skb(*key, *traffic_key, skb,
 779                                         sisn, disn, family))
 780                         return -1;
 781                 *keyid = (*key)->rcvid;
 782         } else {
 783                 struct tcp_ao_key *rnext_key;
 784                 u32 snd_basis;
 785
 786                 if (sk->sk_state == TCP_TIME_WAIT) {
 787                         ao_info = rcu_dereference(tcp_twsk(sk)->ao_info);
 788                         snd_basis = tcp_twsk(sk)->tw_snd_nxt;
 789                 } else {
 790                         ao_info = rcu_dereference(tcp_sk(sk)->ao_info);
 791                         snd_basis = tcp_sk(sk)->snd_una;
 792                 }
 793                 if (!ao_info)
 794                         return -ENOENT;
 795
 796                 *key = tcp_ao_established_key(ao_info, aoh->rnext_keyid, -1);
 797                 if (!*key)
 798                         return -ENOENT;
 799                 *traffic_key = snd_other_key(*key);
 800                 rnext_key = READ_ONCE(ao_info->rnext_key);
 801                 *keyid = rnext_key->rcvid;
 802                 *sne = tcp_ao_compute_sne(READ_ONCE(ao_info->snd_sne),
 803                                           snd_basis, seq);
 804         }
 805         return 0;
 806 }
 807
 808 int tcp_ao_transmit_skb(struct sock *sk, struct sk_buff *skb,
 809                         struct tcp_ao_key *key, struct tcphdr *th,
 810                         __u8 *hash_location)
 811 {
 812         struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
 813         struct tcp_sock *tp = tcp_sk(sk);
 814         struct tcp_ao_info *ao;
 815         void *tkey_buf = NULL;
 816         u8 *traffic_key;
 817         u32 sne;
 818
 819         ao = rcu_dereference_protected(tcp_sk(sk)->ao_info,
 820                                        lockdep_sock_is_held(sk));
 821         traffic_key = snd_other_key(key);
 822         if (unlikely(tcb->tcp_flags & TCPHDR_SYN)) {
 823                 __be32 disn;
 824
 825                 if (!(tcb->tcp_flags & TCPHDR_ACK)) {
 826                         disn = 0;
 827                         tkey_buf = kmalloc(tcp_ao_digest_size(key), GFP_ATOMIC);
 828                         if (!tkey_buf)
 829                                 return -ENOMEM;
 830                         traffic_key = tkey_buf;
 831                 } else {
 832                         disn = ao->risn;
 833                 }
 834                 tp->af_specific->ao_calc_key_sk(key, traffic_key,
 835                                                 sk, ao->lisn, disn, true);
 836         }
 837         sne = tcp_ao_compute_sne(READ_ONCE(ao->snd_sne), READ_ONCE(tp->snd_una),
 838                                  ntohl(th->seq));
 839         tp->af_specific->calc_ao_hash(hash_location, key, sk, skb, traffic_key,
 840                                       hash_location - (u8 *)th, sne);
 841         kfree(tkey_buf);
 842         return 0;
 843 }
 844
 845 static struct tcp_ao_key *tcp_ao_inbound_lookup(unsigned short int family,
 846                 const struct sock *sk, const struct sk_buff *skb,
 847                 int sndid, int rcvid, int l3index)
 848 {
 849         if (family == AF_INET) {
 850                 const struct iphdr *iph = ip_hdr(skb);
 851
 852                 return tcp_ao_do_lookup(sk, l3index,
 853                                         (union tcp_ao_addr *)&iph->saddr,
 854                                         AF_INET, sndid, rcvid);
 855         } else {
 856                 const struct ipv6hdr *iph = ipv6_hdr(skb);
 857
 858                 return tcp_ao_do_lookup(sk, l3index,
 859                                         (union tcp_ao_addr *)&iph->saddr,
 860                                         AF_INET6, sndid, rcvid);
 861         }
 862 }
 863
 864 void tcp_ao_syncookie(struct sock *sk, const struct sk_buff *skb,
 865                       struct request_sock *req, unsigned short int family)
 866 {
 867         struct tcp_request_sock *treq = tcp_rsk(req);
 868         const struct tcphdr *th = tcp_hdr(skb);
 869         const struct tcp_ao_hdr *aoh;
 870         struct tcp_ao_key *key;
 871         int l3index;
 872
 873         /* treq->af_specific is used to perform TCP_AO lookup
 874          * in tcp_create_openreq_child().
 875          */
 876 #if IS_ENABLED(CONFIG_IPV6)
 877         if (family == AF_INET6)
 878                 treq->af_specific = &tcp_request_sock_ipv6_ops;
 879         else
 880 #endif
 881                 treq->af_specific = &tcp_request_sock_ipv4_ops;
 882
 883         treq->used_tcp_ao = false;
 884
 885         if (tcp_parse_auth_options(th, NULL, &aoh) || !aoh)
 886                 return;
 887
 888         l3index = l3mdev_master_ifindex_by_index(sock_net(sk), inet_rsk(req)->ir_iif);
 889         key = tcp_ao_inbound_lookup(family, sk, skb, -1, aoh->keyid, l3index);
 890         if (!key)
 891                 /* Key not found, continue without TCP-AO */
 892                 return;
 893
 894         treq->ao_rcv_next = aoh->keyid;
 895         treq->ao_keyid = aoh->rnext_keyid;
 896         treq->used_tcp_ao = true;
 897 }
 898
 899 static enum skb_drop_reason
 900 tcp_ao_verify_hash(const struct sock *sk, const struct sk_buff *skb,
 901                    unsigned short int family, struct tcp_ao_info *info,
 902                    const struct tcp_ao_hdr *aoh, struct tcp_ao_key *key,
 903                    u8 *traffic_key, u8 *phash, u32 sne, int l3index)
 904 {
 905         const struct tcphdr *th = tcp_hdr(skb);
 906         u8 maclen = tcp_ao_hdr_maclen(aoh);
 907         void *hash_buf = NULL;
 908
 909         if (maclen != tcp_ao_maclen(key)) {
 910                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOBAD);
 911                 atomic64_inc(&info->counters.pkt_bad);
 912                 atomic64_inc(&key->pkt_bad);
 913                 trace_tcp_ao_wrong_maclen(sk, skb, aoh->keyid,
 914                                           aoh->rnext_keyid, maclen);
 915                 return SKB_DROP_REASON_TCP_AOFAILURE;
 916         }
 917
 918         hash_buf = kmalloc(tcp_ao_digest_size(key), GFP_ATOMIC);
 919         if (!hash_buf)
 920                 return SKB_DROP_REASON_NOT_SPECIFIED;
 921
 922         /* XXX: make it per-AF callback? */
 923         tcp_ao_hash_skb(family, hash_buf, key, sk, skb, traffic_key,
 924                         (phash - (u8 *)th), sne);
 925         if (memcmp(phash, hash_buf, maclen)) {
 926                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOBAD);
 927                 atomic64_inc(&info->counters.pkt_bad);
 928                 atomic64_inc(&key->pkt_bad);
 929                 trace_tcp_ao_mismatch(sk, skb, aoh->keyid,
 930                                       aoh->rnext_keyid, maclen);
 931                 kfree(hash_buf);
 932                 return SKB_DROP_REASON_TCP_AOFAILURE;
 933         }
 934         NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOGOOD);
 935         atomic64_inc(&info->counters.pkt_good);
 936         atomic64_inc(&key->pkt_good);
 937         kfree(hash_buf);
 938         return SKB_NOT_DROPPED_YET;
 939 }
 940
 941 enum skb_drop_reason
 942 tcp_inbound_ao_hash(struct sock *sk, const struct sk_buff *skb,
 943                     unsigned short int family, const struct request_sock *req,
 944                     int l3index, const struct tcp_ao_hdr *aoh)
 945 {
 946         const struct tcphdr *th = tcp_hdr(skb);
 947         u8 maclen = tcp_ao_hdr_maclen(aoh);
 948         u8 *phash = (u8 *)(aoh + 1); /* hash goes just after the header */
 949         struct tcp_ao_info *info;
 950         enum skb_drop_reason ret;
 951         struct tcp_ao_key *key;
 952         __be32 sisn, disn;
 953         u8 *traffic_key;
 954         int state;
 955         u32 sne = 0;
 956
 957         info = rcu_dereference(tcp_sk(sk)->ao_info);
 958         if (!info) {
 959                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOKEYNOTFOUND);
 960                 trace_tcp_ao_key_not_found(sk, skb, aoh->keyid,
 961                                            aoh->rnext_keyid, maclen);
 962                 return SKB_DROP_REASON_TCP_AOUNEXPECTED;
 963         }
 964
 965         if (unlikely(th->syn)) {
 966                 sisn = th->seq;
 967                 disn = 0;
 968         }
 969
 970         state = READ_ONCE(sk->sk_state);
 971         /* Fast-path */
 972         if (likely((1 << state) & TCP_AO_ESTABLISHED)) {
 973                 enum skb_drop_reason err;
 974                 struct tcp_ao_key *current_key;
 975
 976                 /* Check if this socket's rnext_key matches the keyid in the
 977                  * packet. If not we lookup the key based on the keyid
 978                  * matching the rcvid in the mkt.
 979                  */
 980                 key = READ_ONCE(info->rnext_key);
 981                 if (key->rcvid != aoh->keyid) {
 982                         key = tcp_ao_established_key(info, -1, aoh->keyid);
 983                         if (!key)
 984                                 goto key_not_found;
 985                 }
 986
 987                 /* Delayed retransmitted SYN */
 988                 if (unlikely(th->syn && !th->ack))
 989                         goto verify_hash;
 990
 991                 sne = tcp_ao_compute_sne(info->rcv_sne, tcp_sk(sk)->rcv_nxt,
 992                                          ntohl(th->seq));
 993                 /* Established socket, traffic key are cached */
 994                 traffic_key = rcv_other_key(key);
 995                 err = tcp_ao_verify_hash(sk, skb, family, info, aoh, key,
 996                                          traffic_key, phash, sne, l3index);
 997                 if (err)
 998                         return err;
 999                 current_key = READ_ONCE(info->current_key);
1000                 /* Key rotation: the peer asks us to use new key (RNext) */
1001                 if (unlikely(aoh->rnext_keyid != current_key->sndid)) {
1002                         trace_tcp_ao_rnext_request(sk, skb, current_key->sndid,
1003                                                    aoh->rnext_keyid,
1004                                                    tcp_ao_hdr_maclen(aoh));
1005                         /* If the key is not found we do nothing. */
1006                         key = tcp_ao_established_key(info, aoh->rnext_keyid, -1);
1007                         if (key)
1008                                 /* pairs with tcp_ao_del_cmd */
1009                                 WRITE_ONCE(info->current_key, key);
1010                 }
1011                 return SKB_NOT_DROPPED_YET;
1012         }
1013
1014         if (unlikely(state == TCP_CLOSE))
1015                 return SKB_DROP_REASON_TCP_CLOSE;
1016
1017         /* Lookup key based on peer address and keyid.
1018          * current_key and rnext_key must not be used on tcp listen
1019          * sockets as otherwise:
1020          * - request sockets would race on those key pointers
1021          * - tcp_ao_del_cmd() allows async key removal
1022          */
1023         key = tcp_ao_inbound_lookup(family, sk, skb, -1, aoh->keyid, l3index);
1024         if (!key)
1025                 goto key_not_found;
1026
1027         if (th->syn && !th->ack)
1028                 goto verify_hash;
1029
1030         if ((1 << state) & (TCPF_LISTEN | TCPF_NEW_SYN_RECV)) {
1031                 /* Make the initial syn the likely case here */
1032                 if (unlikely(req)) {
1033                         sne = tcp_ao_compute_sne(0, tcp_rsk(req)->rcv_isn,
1034                                                  ntohl(th->seq));
1035                         sisn = htonl(tcp_rsk(req)->rcv_isn);
1036                         disn = htonl(tcp_rsk(req)->snt_isn);
1037                 } else if (unlikely(th->ack && !th->syn)) {
1038                         /* Possible syncookie packet */
1039                         sisn = htonl(ntohl(th->seq) - 1);
1040                         disn = htonl(ntohl(th->ack_seq) - 1);
1041                         sne = tcp_ao_compute_sne(0, ntohl(sisn),
1042                                                  ntohl(th->seq));
1043                 } else if (unlikely(!th->syn)) {
1044                         /* no way to figure out initial sisn/disn - drop */
1045                         return SKB_DROP_REASON_TCP_FLAGS;
1046                 }
1047         } else if ((1 << state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) {
1048                 disn = info->lisn;
1049                 if (th->syn || th->rst)
1050                         sisn = th->seq;
1051                 else
1052                         sisn = info->risn;
1053         } else {
1054                 WARN_ONCE(1, "TCP-AO: Unexpected sk_state %d", state);
1055                 return SKB_DROP_REASON_TCP_AOFAILURE;
1056         }
1057 verify_hash:
1058         traffic_key = kmalloc(tcp_ao_digest_size(key), GFP_ATOMIC);
1059         if (!traffic_key)
1060                 return SKB_DROP_REASON_NOT_SPECIFIED;
1061         tcp_ao_calc_key_skb(key, traffic_key, skb, sisn, disn, family);
1062         ret = tcp_ao_verify_hash(sk, skb, family, info, aoh, key,
1063                                  traffic_key, phash, sne, l3index);
1064         kfree(traffic_key);
1065         return ret;
1066
1067 key_not_found:
1068         NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOKEYNOTFOUND);
1069         atomic64_inc(&info->counters.key_not_found);
1070         trace_tcp_ao_key_not_found(sk, skb, aoh->keyid,
1071                                    aoh->rnext_keyid, maclen);
1072         return SKB_DROP_REASON_TCP_AOKEYNOTFOUND;
1073 }
1074
1075 static int tcp_ao_cache_traffic_keys(const struct sock *sk,
1076                                      struct tcp_ao_info *ao,
1077                                      struct tcp_ao_key *ao_key)
1078 {
1079         u8 *traffic_key = snd_other_key(ao_key);
1080         int ret;
1081
1082         ret = tcp_ao_calc_key_sk(ao_key, traffic_key, sk,
1083                                  ao->lisn, ao->risn, true);
1084         if (ret)
1085                 return ret;
1086
1087         traffic_key = rcv_other_key(ao_key);
1088         ret = tcp_ao_calc_key_sk(ao_key, traffic_key, sk,
1089                                  ao->lisn, ao->risn, false);
1090         return ret;
1091 }
1092
1093 void tcp_ao_connect_init(struct sock *sk)
1094 {
1095         struct tcp_sock *tp = tcp_sk(sk);
1096         struct tcp_ao_info *ao_info;
1097         struct hlist_node *next;
1098         union tcp_ao_addr *addr;
1099         struct tcp_ao_key *key;
1100         int family, l3index;
1101
1102         ao_info = rcu_dereference_protected(tp->ao_info,
1103                                             lockdep_sock_is_held(sk));
1104         if (!ao_info)
1105                 return;
1106
1107         /* Remove all keys that don't match the peer */
1108         family = sk->sk_family;
1109         if (family == AF_INET)
1110                 addr = (union tcp_ao_addr *)&sk->sk_daddr;
1111 #if IS_ENABLED(CONFIG_IPV6)
1112         else if (family == AF_INET6)
1113                 addr = (union tcp_ao_addr *)&sk->sk_v6_daddr;
1114 #endif
1115         else
1116                 return;
1117         l3index = l3mdev_master_ifindex_by_index(sock_net(sk),
1118                                                  sk->sk_bound_dev_if);
1119
1120         hlist_for_each_entry_safe(key, next, &ao_info->head, node) {
1121                 if (!tcp_ao_key_cmp(key, l3index, addr, key->prefixlen, family, -1, -1))
1122                         continue;
1123
1124                 if (key == ao_info->current_key)
1125                         ao_info->current_key = NULL;
1126                 if (key == ao_info->rnext_key)
1127                         ao_info->rnext_key = NULL;
1128                 hlist_del_rcu(&key->node);
1129                 atomic_sub(tcp_ao_sizeof_key(key), &sk->sk_omem_alloc);
1130                 call_rcu(&key->rcu, tcp_ao_key_free_rcu);
1131         }
1132
1133         key = tp->af_specific->ao_lookup(sk, sk, -1, -1);
1134         if (key) {
1135                 /* if current_key or rnext_key were not provided,
1136                  * use the first key matching the peer
1137                  */
1138                 if (!ao_info->current_key)
1139                         ao_info->current_key = key;
1140                 if (!ao_info->rnext_key)
1141                         ao_info->rnext_key = key;
1142                 tp->tcp_header_len += tcp_ao_len_aligned(key);
1143
1144                 ao_info->lisn = htonl(tp->write_seq);
1145                 ao_info->snd_sne = 0;
1146         } else {
1147                 /* Can't happen: tcp_connect() verifies that there's
1148                  * at least one tcp-ao key that matches the remote peer.
1149                  */
1150                 WARN_ON_ONCE(1);
1151                 rcu_assign_pointer(tp->ao_info, NULL);
1152                 kfree(ao_info);
1153         }
1154 }
1155
1156 void tcp_ao_established(struct sock *sk)
1157 {
1158         struct tcp_ao_info *ao;
1159         struct tcp_ao_key *key;
1160
1161         ao = rcu_dereference_protected(tcp_sk(sk)->ao_info,
1162                                        lockdep_sock_is_held(sk));
1163         if (!ao)
1164                 return;
1165
1166         hlist_for_each_entry_rcu(key, &ao->head, node)
1167                 tcp_ao_cache_traffic_keys(sk, ao, key);
1168 }
1169
1170 void tcp_ao_finish_connect(struct sock *sk, struct sk_buff *skb)
1171 {
1172         struct tcp_ao_info *ao;
1173         struct tcp_ao_key *key;
1174
1175         ao = rcu_dereference_protected(tcp_sk(sk)->ao_info,
1176                                        lockdep_sock_is_held(sk));
1177         if (!ao)
1178                 return;
1179
1180         WRITE_ONCE(ao->risn, tcp_hdr(skb)->seq);
1181         ao->rcv_sne = 0;
1182
1183         hlist_for_each_entry_rcu(key, &ao->head, node)
1184                 tcp_ao_cache_traffic_keys(sk, ao, key);
1185 }
1186
1187 int tcp_ao_copy_all_matching(const struct sock *sk, struct sock *newsk,
1188                              struct request_sock *req, struct sk_buff *skb,
1189                              int family)
1190 {
1191         struct tcp_ao_key *key, *new_key, *first_key;
1192         struct tcp_ao_info *new_ao, *ao;
1193         struct hlist_node *key_head;
1194         int l3index, ret = -ENOMEM;
1195         union tcp_ao_addr *addr;
1196         bool match = false;
1197
1198         ao = rcu_dereference(tcp_sk(sk)->ao_info);
1199         if (!ao)
1200                 return 0;
1201
1202         /* New socket without TCP-AO on it */
1203         if (!tcp_rsk_used_ao(req))
1204                 return 0;
1205
1206         new_ao = tcp_ao_alloc_info(GFP_ATOMIC);
1207         if (!new_ao)
1208                 return -ENOMEM;
1209         new_ao->lisn = htonl(tcp_rsk(req)->snt_isn);
1210         new_ao->risn = htonl(tcp_rsk(req)->rcv_isn);
1211         new_ao->ao_required = ao->ao_required;
1212         new_ao->accept_icmps = ao->accept_icmps;
1213
1214         if (family == AF_INET) {
1215                 addr = (union tcp_ao_addr *)&newsk->sk_daddr;
1216 #if IS_ENABLED(CONFIG_IPV6)
1217         } else if (family == AF_INET6) {
1218                 addr = (union tcp_ao_addr *)&newsk->sk_v6_daddr;
1219 #endif
1220         } else {
1221                 ret = -EAFNOSUPPORT;
1222                 goto free_ao;
1223         }
1224         l3index = l3mdev_master_ifindex_by_index(sock_net(newsk),
1225                                                  newsk->sk_bound_dev_if);
1226
1227         hlist_for_each_entry_rcu(key, &ao->head, node) {
1228                 if (tcp_ao_key_cmp(key, l3index, addr, key->prefixlen, family, -1, -1))
1229                         continue;
1230
1231                 new_key = tcp_ao_copy_key(newsk, key);
1232                 if (!new_key)
1233                         goto free_and_exit;
1234
1235                 tcp_ao_cache_traffic_keys(newsk, new_ao, new_key);
1236                 tcp_ao_link_mkt(new_ao, new_key);
1237                 match = true;
1238         }
1239
1240         if (!match) {
1241                 /* RFC5925 (7.4.1) specifies that the TCP-AO status
1242                  * of a connection is determined on the initial SYN.
1243                  * At this point the connection was TCP-AO enabled, so
1244                  * it can't switch to being unsigned if peer's key
1245                  * disappears on the listening socket.
1246                  */
1247                 ret = -EKEYREJECTED;
1248                 goto free_and_exit;
1249         }
1250
1251         if (!static_key_fast_inc_not_disabled(&tcp_ao_needed.key.key)) {
1252                 ret = -EUSERS;
1253                 goto free_and_exit;
1254         }
1255
1256         key_head = rcu_dereference(hlist_first_rcu(&new_ao->head));
1257         first_key = hlist_entry_safe(key_head, struct tcp_ao_key, node);
1258
1259         key = tcp_ao_established_key(new_ao, tcp_rsk(req)->ao_keyid, -1);
1260         if (key)
1261                 new_ao->current_key = key;
1262         else
1263                 new_ao->current_key = first_key;
1264
1265         /* set rnext_key */
1266         key = tcp_ao_established_key(new_ao, -1, tcp_rsk(req)->ao_rcv_next);
1267         if (key)
1268                 new_ao->rnext_key = key;
1269         else
1270                 new_ao->rnext_key = first_key;
1271
1272         sk_gso_disable(newsk);
1273         rcu_assign_pointer(tcp_sk(newsk)->ao_info, new_ao);
1274
1275         return 0;
1276
1277 free_and_exit:
1278         hlist_for_each_entry_safe(key, key_head, &new_ao->head, node) {
1279                 hlist_del(&key->node);
1280                 tcp_sigpool_release(key->tcp_sigpool_id);
1281                 atomic_sub(tcp_ao_sizeof_key(key), &newsk->sk_omem_alloc);
1282                 kfree_sensitive(key);
1283         }
1284 free_ao:
1285         kfree(new_ao);
1286         return ret;
1287 }
1288
1289 static bool tcp_ao_can_set_current_rnext(struct sock *sk)
1290 {
1291         /* There aren't current/rnext keys on TCP_LISTEN sockets */
1292         if (sk->sk_state == TCP_LISTEN)
1293                 return false;
1294         return true;
1295 }
1296
1297 static int tcp_ao_verify_ipv4(struct sock *sk, struct tcp_ao_add *cmd,
1298                               union tcp_ao_addr **addr)
1299 {
1300         struct sockaddr_in *sin = (struct sockaddr_in *)&cmd->addr;
1301         struct inet_sock *inet = inet_sk(sk);
1302
1303         if (sin->sin_family != AF_INET)
1304                 return -EINVAL;
1305
1306         /* Currently matching is not performed on port (or port ranges) */
1307         if (sin->sin_port != 0)
1308                 return -EINVAL;
1309
1310         /* Check prefix and trailing 0's in addr */
1311         if (cmd->prefix != 0) {
1312                 __be32 mask;
1313
1314                 if (ntohl(sin->sin_addr.s_addr) == INADDR_ANY)
1315                         return -EINVAL;
1316                 if (cmd->prefix > 32)
1317                         return -EINVAL;
1318
1319                 mask = inet_make_mask(cmd->prefix);
1320                 if (sin->sin_addr.s_addr & ~mask)
1321                         return -EINVAL;
1322
1323                 /* Check that MKT address is consistent with socket */
1324                 if (ntohl(inet->inet_daddr) != INADDR_ANY &&
1325                     (inet->inet_daddr & mask) != sin->sin_addr.s_addr)
1326                         return -EINVAL;
1327         } else {
1328                 if (ntohl(sin->sin_addr.s_addr) != INADDR_ANY)
1329                         return -EINVAL;
1330         }
1331
1332         *addr = (union tcp_ao_addr *)&sin->sin_addr;
1333         return 0;
1334 }
1335
1336 static int tcp_ao_parse_crypto(struct tcp_ao_add *cmd, struct tcp_ao_key *key)
1337 {
1338         unsigned int syn_tcp_option_space;
1339         bool is_kdf_aes_128_cmac = false;
1340         struct crypto_ahash *tfm;
1341         struct tcp_sigpool hp;
1342         void *tmp_key = NULL;
1343         int err;
1344
1345         /* RFC5926, 3.1.1.2. KDF_AES_128_CMAC */
1346         if (!strcmp("cmac(aes128)", cmd->alg_name)) {
1347                 strscpy(cmd->alg_name, "cmac(aes)", sizeof(cmd->alg_name));
1348                 is_kdf_aes_128_cmac = (cmd->keylen != 16);
1349                 tmp_key = kmalloc(cmd->keylen, GFP_KERNEL);
1350                 if (!tmp_key)
1351                         return -ENOMEM;
1352         }
1353
1354         key->maclen = cmd->maclen ?: 12; /* 12 is the default in RFC5925 */
1355
1356         /* Check: maclen + tcp-ao header <= (MAX_TCP_OPTION_SPACE - mss
1357          *                                      - tstamp (including sackperm)
1358          *                                      - wscale),
1359          * see tcp_syn_options(), tcp_synack_options(), commit 33ad798c924b.
1360          *
1361          * In order to allow D-SACK with TCP-AO, the header size should be:
1362          * (MAX_TCP_OPTION_SPACE - TCPOLEN_TSTAMP_ALIGNED
1363          *                      - TCPOLEN_SACK_BASE_ALIGNED
1364          *                      - 2 * TCPOLEN_SACK_PERBLOCK) = 8 (maclen = 4),
1365          * see tcp_established_options().
1366          *
1367          * RFC5925, 2.2:
1368          * Typical MACs are 96-128 bits (12-16 bytes), but any length
1369          * that fits in the header of the segment being authenticated
1370          * is allowed.
1371          *
1372          * RFC5925, 7.6:
1373          * TCP-AO continues to consume 16 bytes in non-SYN segments,
1374          * leaving a total of 24 bytes for other options, of which
1375          * the timestamp consumes 10.  This leaves 14 bytes, of which 10
1376          * are used for a single SACK block. When two SACK blocks are used,
1377          * such as to handle D-SACK, a smaller TCP-AO MAC would be required
1378          * to make room for the additional SACK block (i.e., to leave 18
1379          * bytes for the D-SACK variant of the SACK option) [RFC2883].
1380          * Note that D-SACK is not supportable in TCP MD5 in the presence
1381          * of timestamps, because TCP MD5’s MAC length is fixed and too
1382          * large to leave sufficient option space.
1383          */
1384         syn_tcp_option_space = MAX_TCP_OPTION_SPACE;
1385         syn_tcp_option_space -= TCPOLEN_MSS_ALIGNED;
1386         syn_tcp_option_space -= TCPOLEN_TSTAMP_ALIGNED;
1387         syn_tcp_option_space -= TCPOLEN_WSCALE_ALIGNED;
1388         if (tcp_ao_len_aligned(key) > syn_tcp_option_space) {
1389                 err = -EMSGSIZE;
1390                 goto err_kfree;
1391         }
1392
1393         key->keylen = cmd->keylen;
1394         memcpy(key->key, cmd->key, cmd->keylen);
1395
1396         err = tcp_sigpool_start(key->tcp_sigpool_id, &hp);
1397         if (err)
1398                 goto err_kfree;
1399
1400         tfm = crypto_ahash_reqtfm(hp.req);
1401         if (is_kdf_aes_128_cmac) {
1402                 void *scratch = hp.scratch;
1403                 struct scatterlist sg;
1404
1405                 memcpy(tmp_key, cmd->key, cmd->keylen);
1406                 sg_init_one(&sg, tmp_key, cmd->keylen);
1407
1408                 /* Using zero-key of 16 bytes as described in RFC5926 */
1409                 memset(scratch, 0, 16);
1410                 err = crypto_ahash_setkey(tfm, scratch, 16);
1411                 if (err)
1412                         goto err_pool_end;
1413
1414                 err = crypto_ahash_init(hp.req);
1415                 if (err)
1416                         goto err_pool_end;
1417
1418                 ahash_request_set_crypt(hp.req, &sg, key->key, cmd->keylen);
1419                 err = crypto_ahash_update(hp.req);
1420                 if (err)
1421                         goto err_pool_end;
1422
1423                 err |= crypto_ahash_final(hp.req);
1424                 if (err)
1425                         goto err_pool_end;
1426                 key->keylen = 16;
1427         }
1428
1429         err = crypto_ahash_setkey(tfm, key->key, key->keylen);
1430         if (err)
1431                 goto err_pool_end;
1432
1433         tcp_sigpool_end(&hp);
1434         kfree_sensitive(tmp_key);
1435
1436         if (tcp_ao_maclen(key) > key->digest_size)
1437                 return -EINVAL;
1438
1439         return 0;
1440
1441 err_pool_end:
1442         tcp_sigpool_end(&hp);
1443 err_kfree:
1444         kfree_sensitive(tmp_key);
1445         return err;
1446 }
1447
1448 #if IS_ENABLED(CONFIG_IPV6)
1449 static int tcp_ao_verify_ipv6(struct sock *sk, struct tcp_ao_add *cmd,
1450                               union tcp_ao_addr **paddr,
1451                               unsigned short int *family)
1452 {
1453         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&cmd->addr;
1454         struct in6_addr *addr = &sin6->sin6_addr;
1455         u8 prefix = cmd->prefix;
1456
1457         if (sin6->sin6_family != AF_INET6)
1458                 return -EINVAL;
1459
1460         /* Currently matching is not performed on port (or port ranges) */
1461         if (sin6->sin6_port != 0)
1462                 return -EINVAL;
1463
1464         /* Check prefix and trailing 0's in addr */
1465         if (cmd->prefix != 0 && ipv6_addr_v4mapped(addr)) {
1466                 __be32 addr4 = addr->s6_addr32[3];
1467                 __be32 mask;
1468
1469                 if (prefix > 32 || ntohl(addr4) == INADDR_ANY)
1470                         return -EINVAL;
1471
1472                 mask = inet_make_mask(prefix);
1473                 if (addr4 & ~mask)
1474                         return -EINVAL;
1475
1476                 /* Check that MKT address is consistent with socket */
1477                 if (!ipv6_addr_any(&sk->sk_v6_daddr)) {
1478                         __be32 daddr4 = sk->sk_v6_daddr.s6_addr32[3];
1479
1480                         if (!ipv6_addr_v4mapped(&sk->sk_v6_daddr))
1481                                 return -EINVAL;
1482                         if ((daddr4 & mask) != addr4)
1483                                 return -EINVAL;
1484                 }
1485
1486                 *paddr = (union tcp_ao_addr *)&addr->s6_addr32[3];
1487                 *family = AF_INET;
1488                 return 0;
1489         } else if (cmd->prefix != 0) {
1490                 struct in6_addr pfx;
1491
1492                 if (ipv6_addr_any(addr) || prefix > 128)
1493                         return -EINVAL;
1494
1495                 ipv6_addr_prefix(&pfx, addr, prefix);
1496                 if (ipv6_addr_cmp(&pfx, addr))
1497                         return -EINVAL;
1498
1499                 /* Check that MKT address is consistent with socket */
1500                 if (!ipv6_addr_any(&sk->sk_v6_daddr) &&
1501                     !ipv6_prefix_equal(&sk->sk_v6_daddr, addr, prefix))
1502
1503                         return -EINVAL;
1504         } else {
1505                 if (!ipv6_addr_any(addr))
1506                         return -EINVAL;
1507         }
1508
1509         *paddr = (union tcp_ao_addr *)addr;
1510         return 0;
1511 }
1512 #else
1513 static int tcp_ao_verify_ipv6(struct sock *sk, struct tcp_ao_add *cmd,
1514                               union tcp_ao_addr **paddr,
1515                               unsigned short int *family)
1516 {
1517         return -EOPNOTSUPP;
1518 }
1519 #endif
1520
1521 static struct tcp_ao_info *setsockopt_ao_info(struct sock *sk)
1522 {
1523         if (sk_fullsock(sk)) {
1524                 return rcu_dereference_protected(tcp_sk(sk)->ao_info,
1525                                                  lockdep_sock_is_held(sk));
1526         } else if (sk->sk_state == TCP_TIME_WAIT) {
1527                 return rcu_dereference_protected(tcp_twsk(sk)->ao_info,
1528                                                  lockdep_sock_is_held(sk));
1529         }
1530         return ERR_PTR(-ESOCKTNOSUPPORT);
1531 }
1532
1533 static struct tcp_ao_info *getsockopt_ao_info(struct sock *sk)
1534 {
1535         if (sk_fullsock(sk))
1536                 return rcu_dereference(tcp_sk(sk)->ao_info);
1537         else if (sk->sk_state == TCP_TIME_WAIT)
1538                 return rcu_dereference(tcp_twsk(sk)->ao_info);
1539
1540         return ERR_PTR(-ESOCKTNOSUPPORT);
1541 }
1542
1543 #define TCP_AO_KEYF_ALL (TCP_AO_KEYF_IFINDEX | TCP_AO_KEYF_EXCLUDE_OPT)
1544 #define TCP_AO_GET_KEYF_VALID   (TCP_AO_KEYF_IFINDEX)
1545
1546 static struct tcp_ao_key *tcp_ao_key_alloc(struct sock *sk,
1547                                            struct tcp_ao_add *cmd)
1548 {
1549         const char *algo = cmd->alg_name;
1550         unsigned int digest_size;
1551         struct crypto_ahash *tfm;
1552         struct tcp_ao_key *key;
1553         struct tcp_sigpool hp;
1554         int err, pool_id;
1555         size_t size;
1556
1557         /* Force null-termination of alg_name */
1558         cmd->alg_name[ARRAY_SIZE(cmd->alg_name) - 1] = '\0';
1559
1560         /* RFC5926, 3.1.1.2. KDF_AES_128_CMAC */
1561         if (!strcmp("cmac(aes128)", algo))
1562                 algo = "cmac(aes)";
1563
1564         /* Full TCP header (th->doff << 2) should fit into scratch area,
1565          * see tcp_ao_hash_header().
1566          */
1567         pool_id = tcp_sigpool_alloc_ahash(algo, 60);
1568         if (pool_id < 0)
1569                 return ERR_PTR(pool_id);
1570
1571         err = tcp_sigpool_start(pool_id, &hp);
1572         if (err)
1573                 goto err_free_pool;
1574
1575         tfm = crypto_ahash_reqtfm(hp.req);
1576         digest_size = crypto_ahash_digestsize(tfm);
1577         tcp_sigpool_end(&hp);
1578
1579         size = sizeof(struct tcp_ao_key) + (digest_size << 1);
1580         key = sock_kmalloc(sk, size, GFP_KERNEL);
1581         if (!key) {
1582                 err = -ENOMEM;
1583                 goto err_free_pool;
1584         }
1585
1586         key->tcp_sigpool_id = pool_id;
1587         key->digest_size = digest_size;
1588         return key;
1589
1590 err_free_pool:
1591         tcp_sigpool_release(pool_id);
1592         return ERR_PTR(err);
1593 }
1594
1595 static int tcp_ao_add_cmd(struct sock *sk, unsigned short int family,
1596                           sockptr_t optval, int optlen)
1597 {
1598         struct tcp_ao_info *ao_info;
1599         union tcp_ao_addr *addr;
1600         struct tcp_ao_key *key;
1601         struct tcp_ao_add cmd;
1602         int ret, l3index = 0;
1603         bool first = false;
1604
1605         if (optlen < sizeof(cmd))
1606                 return -EINVAL;
1607
1608         ret = copy_struct_from_sockptr(&cmd, sizeof(cmd), optval, optlen);
1609         if (ret)
1610                 return ret;
1611
1612         if (cmd.keylen > TCP_AO_MAXKEYLEN)
1613                 return -EINVAL;
1614
1615         if (cmd.reserved != 0 || cmd.reserved2 != 0)
1616                 return -EINVAL;
1617
1618         if (family == AF_INET)
1619                 ret = tcp_ao_verify_ipv4(sk, &cmd, &addr);
1620         else
1621                 ret = tcp_ao_verify_ipv6(sk, &cmd, &addr, &family);
1622         if (ret)
1623                 return ret;
1624
1625         if (cmd.keyflags & ~TCP_AO_KEYF_ALL)
1626                 return -EINVAL;
1627
1628         if (cmd.set_current || cmd.set_rnext) {
1629                 if (!tcp_ao_can_set_current_rnext(sk))
1630                         return -EINVAL;
1631         }
1632
1633         if (cmd.ifindex && !(cmd.keyflags & TCP_AO_KEYF_IFINDEX))
1634                 return -EINVAL;
1635
1636         /* For cmd.tcp_ifindex = 0 the key will apply to the default VRF */
1637         if (cmd.keyflags & TCP_AO_KEYF_IFINDEX && cmd.ifindex) {
1638                 int bound_dev_if = READ_ONCE(sk->sk_bound_dev_if);
1639                 struct net_device *dev;
1640
1641                 rcu_read_lock();
1642                 dev = dev_get_by_index_rcu(sock_net(sk), cmd.ifindex);
1643                 if (dev && netif_is_l3_master(dev))
1644                         l3index = dev->ifindex;
1645                 rcu_read_unlock();
1646
1647                 if (!dev || !l3index)
1648                         return -EINVAL;
1649
1650                 if (!bound_dev_if || bound_dev_if != cmd.ifindex) {
1651                         /* tcp_ao_established_key() doesn't expect having
1652                          * non peer-matching key on an established TCP-AO
1653                          * connection.
1654                          */
1655                         if (!((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)))
1656                                 return -EINVAL;
1657                 }
1658
1659                 /* It's still possible to bind after adding keys or even
1660                  * re-bind to a different dev (with CAP_NET_RAW).
1661                  * So, no reason to return error here, rather try to be
1662                  * nice and warn the user.
1663                  */
1664                 if (bound_dev_if && bound_dev_if != cmd.ifindex)
1665                         net_warn_ratelimited("AO key ifindex %d != sk bound ifindex %d\n",
1666                                              cmd.ifindex, bound_dev_if);
1667         }
1668
1669         /* Don't allow keys for peers that have a matching TCP-MD5 key */
1670         if (cmd.keyflags & TCP_AO_KEYF_IFINDEX) {
1671                 /* Non-_exact version of tcp_md5_do_lookup() will
1672                  * as well match keys that aren't bound to a specific VRF
1673                  * (that will make them match AO key with
1674                  * sysctl_tcp_l3dev_accept = 1
1675                  */
1676                 if (tcp_md5_do_lookup(sk, l3index, addr, family))
1677                         return -EKEYREJECTED;
1678         } else {
1679                 if (tcp_md5_do_lookup_any_l3index(sk, addr, family))
1680                         return -EKEYREJECTED;
1681         }
1682
1683         ao_info = setsockopt_ao_info(sk);
1684         if (IS_ERR(ao_info))
1685                 return PTR_ERR(ao_info);
1686
1687         if (!ao_info) {
1688                 ao_info = tcp_ao_alloc_info(GFP_KERNEL);
1689                 if (!ao_info)
1690                         return -ENOMEM;
1691                 first = true;
1692         } else {
1693                 /* Check that neither RecvID nor SendID match any
1694                  * existing key for the peer, RFC5925 3.1:
1695                  * > The IDs of MKTs MUST NOT overlap where their
1696                  * > TCP connection identifiers overlap.
1697                  */
1698                 if (__tcp_ao_do_lookup(sk, l3index, addr, family, cmd.prefix, -1, cmd.rcvid))
1699                         return -EEXIST;
1700                 if (__tcp_ao_do_lookup(sk, l3index, addr, family,
1701                                        cmd.prefix, cmd.sndid, -1))
1702                         return -EEXIST;
1703         }
1704
1705         key = tcp_ao_key_alloc(sk, &cmd);
1706         if (IS_ERR(key)) {
1707                 ret = PTR_ERR(key);
1708                 goto err_free_ao;
1709         }
1710
1711         INIT_HLIST_NODE(&key->node);
1712         memcpy(&key->addr, addr, (family == AF_INET) ? sizeof(struct in_addr) :
1713                                                        sizeof(struct in6_addr));
1714         key->prefixlen  = cmd.prefix;
1715         key->family     = family;
1716         key->keyflags   = cmd.keyflags;
1717         key->sndid      = cmd.sndid;
1718         key->rcvid      = cmd.rcvid;
1719         key->l3index    = l3index;
1720         atomic64_set(&key->pkt_good, 0);
1721         atomic64_set(&key->pkt_bad, 0);
1722
1723         ret = tcp_ao_parse_crypto(&cmd, key);
1724         if (ret < 0)
1725                 goto err_free_sock;
1726
1727         if (!((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))) {
1728                 tcp_ao_cache_traffic_keys(sk, ao_info, key);
1729                 if (first) {
1730                         ao_info->current_key = key;
1731                         ao_info->rnext_key = key;
1732                 }
1733         }
1734
1735         tcp_ao_link_mkt(ao_info, key);
1736         if (first) {
1737                 if (!static_branch_inc(&tcp_ao_needed.key)) {
1738                         ret = -EUSERS;
1739                         goto err_free_sock;
1740                 }
1741                 sk_gso_disable(sk);
1742                 rcu_assign_pointer(tcp_sk(sk)->ao_info, ao_info);
1743         }
1744
1745         if (cmd.set_current)
1746                 WRITE_ONCE(ao_info->current_key, key);
1747         if (cmd.set_rnext)
1748                 WRITE_ONCE(ao_info->rnext_key, key);
1749         return 0;
1750
1751 err_free_sock:
1752         atomic_sub(tcp_ao_sizeof_key(key), &sk->sk_omem_alloc);
1753         tcp_sigpool_release(key->tcp_sigpool_id);
1754         kfree_sensitive(key);
1755 err_free_ao:
1756         if (first)
1757                 kfree(ao_info);
1758         return ret;
1759 }
1760
1761 static int tcp_ao_delete_key(struct sock *sk, struct tcp_ao_info *ao_info,
1762                              bool del_async, struct tcp_ao_key *key,
1763                              struct tcp_ao_key *new_current,
1764                              struct tcp_ao_key *new_rnext)
1765 {
1766         int err;
1767
1768         hlist_del_rcu(&key->node);
1769
1770         /* Support for async delete on listening sockets: as they don't
1771          * need current_key/rnext_key maintaining, we don't need to check
1772          * them and we can just free all resources in RCU fashion.
1773          */
1774         if (del_async) {
1775                 atomic_sub(tcp_ao_sizeof_key(key), &sk->sk_omem_alloc);
1776                 call_rcu(&key->rcu, tcp_ao_key_free_rcu);
1777                 return 0;
1778         }
1779
1780         /* At this moment another CPU could have looked this key up
1781          * while it was unlinked from the list. Wait for RCU grace period,
1782          * after which the key is off-list and can't be looked up again;
1783          * the rx path [just before RCU came] might have used it and set it
1784          * as current_key (very unlikely).
1785          * Free the key with next RCU grace period (in case it was
1786          * current_key before tcp_ao_current_rnext() might have
1787          * changed it in forced-delete).
1788          */
1789         synchronize_rcu();
1790         if (new_current)
1791                 WRITE_ONCE(ao_info->current_key, new_current);
1792         if (new_rnext)
1793                 WRITE_ONCE(ao_info->rnext_key, new_rnext);
1794
1795         if (unlikely(READ_ONCE(ao_info->current_key) == key ||
1796                      READ_ONCE(ao_info->rnext_key) == key)) {
1797                 err = -EBUSY;
1798                 goto add_key;
1799         }
1800
1801         atomic_sub(tcp_ao_sizeof_key(key), &sk->sk_omem_alloc);
1802         call_rcu(&key->rcu, tcp_ao_key_free_rcu);
1803
1804         return 0;
1805 add_key:
1806         hlist_add_head_rcu(&key->node, &ao_info->head);
1807         return err;
1808 }
1809
1810 #define TCP_AO_DEL_KEYF_ALL (TCP_AO_KEYF_IFINDEX)
1811 static int tcp_ao_del_cmd(struct sock *sk, unsigned short int family,
1812                           sockptr_t optval, int optlen)
1813 {
1814         struct tcp_ao_key *key, *new_current = NULL, *new_rnext = NULL;
1815         int err, addr_len, l3index = 0;
1816         struct tcp_ao_info *ao_info;
1817         union tcp_ao_addr *addr;
1818         struct tcp_ao_del cmd;
1819         __u8 prefix;
1820         u16 port;
1821
1822         if (optlen < sizeof(cmd))
1823                 return -EINVAL;
1824
1825         err = copy_struct_from_sockptr(&cmd, sizeof(cmd), optval, optlen);
1826         if (err)
1827                 return err;
1828
1829         if (cmd.reserved != 0 || cmd.reserved2 != 0)
1830                 return -EINVAL;
1831
1832         if (cmd.set_current || cmd.set_rnext) {
1833                 if (!tcp_ao_can_set_current_rnext(sk))
1834                         return -EINVAL;
1835         }
1836
1837         if (cmd.keyflags & ~TCP_AO_DEL_KEYF_ALL)
1838                 return -EINVAL;
1839
1840         /* No sanity check for TCP_AO_KEYF_IFINDEX as if a VRF
1841          * was destroyed, there still should be a way to delete keys,
1842          * that were bound to that l3intf. So, fail late at lookup stage
1843          * if there is no key for that ifindex.
1844          */
1845         if (cmd.ifindex && !(cmd.keyflags & TCP_AO_KEYF_IFINDEX))
1846                 return -EINVAL;
1847
1848         ao_info = setsockopt_ao_info(sk);
1849         if (IS_ERR(ao_info))
1850                 return PTR_ERR(ao_info);
1851         if (!ao_info)
1852                 return -ENOENT;
1853
1854         /* For sockets in TCP_CLOSED it's possible set keys that aren't
1855          * matching the future peer (address/VRF/etc),
1856          * tcp_ao_connect_init() will choose a correct matching MKT
1857          * if there's any.
1858          */
1859         if (cmd.set_current) {
1860                 new_current = tcp_ao_established_key(ao_info, cmd.current_key, -1);
1861                 if (!new_current)
1862                         return -ENOENT;
1863         }
1864         if (cmd.set_rnext) {
1865                 new_rnext = tcp_ao_established_key(ao_info, -1, cmd.rnext);
1866                 if (!new_rnext)
1867                         return -ENOENT;
1868         }
1869         if (cmd.del_async && sk->sk_state != TCP_LISTEN)
1870                 return -EINVAL;
1871
1872         if (family == AF_INET) {
1873                 struct sockaddr_in *sin = (struct sockaddr_in *)&cmd.addr;
1874
1875                 addr = (union tcp_ao_addr *)&sin->sin_addr;
1876                 addr_len = sizeof(struct in_addr);
1877                 port = ntohs(sin->sin_port);
1878         } else {
1879                 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&cmd.addr;
1880                 struct in6_addr *addr6 = &sin6->sin6_addr;
1881
1882                 if (ipv6_addr_v4mapped(addr6)) {
1883                         addr = (union tcp_ao_addr *)&addr6->s6_addr32[3];
1884                         addr_len = sizeof(struct in_addr);
1885                         family = AF_INET;
1886                 } else {
1887                         addr = (union tcp_ao_addr *)addr6;
1888                         addr_len = sizeof(struct in6_addr);
1889                 }
1890                 port = ntohs(sin6->sin6_port);
1891         }
1892         prefix = cmd.prefix;
1893
1894         /* Currently matching is not performed on port (or port ranges) */
1895         if (port != 0)
1896                 return -EINVAL;
1897
1898         /* We could choose random present key here for current/rnext
1899          * but that's less predictable. Let's be strict and don't
1900          * allow removing a key that's in use. RFC5925 doesn't
1901          * specify how-to coordinate key removal, but says:
1902          * "It is presumed that an MKT affecting a particular
1903          * connection cannot be destroyed during an active connection"
1904          */
1905         hlist_for_each_entry_rcu(key, &ao_info->head, node) {
1906                 if (cmd.sndid != key->sndid ||
1907                     cmd.rcvid != key->rcvid)
1908                         continue;
1909
1910                 if (family != key->family ||
1911                     prefix != key->prefixlen ||
1912                     memcmp(addr, &key->addr, addr_len))
1913                         continue;
1914
1915                 if ((cmd.keyflags & TCP_AO_KEYF_IFINDEX) !=
1916                     (key->keyflags & TCP_AO_KEYF_IFINDEX))
1917                         continue;
1918
1919                 if (key->l3index != l3index)
1920                         continue;
1921
1922                 if (key == new_current || key == new_rnext)
1923                         continue;
1924
1925                 return tcp_ao_delete_key(sk, ao_info, cmd.del_async, key,
1926                                          new_current, new_rnext);
1927         }
1928         return -ENOENT;
1929 }
1930
1931 /* cmd.ao_required makes a socket TCP-AO only.
1932  * Don't allow any md5 keys for any l3intf on the socket together with it.
1933  * Restricting it early in setsockopt() removes a check for
1934  * ao_info->ao_required on inbound tcp segment fast-path.
1935  */
1936 static int tcp_ao_required_verify(struct sock *sk)
1937 {
1938 #ifdef CONFIG_TCP_MD5SIG
1939         const struct tcp_md5sig_info *md5sig;
1940
1941         if (!static_branch_unlikely(&tcp_md5_needed.key))
1942                 return 0;
1943
1944         md5sig = rcu_dereference_check(tcp_sk(sk)->md5sig_info,
1945                                        lockdep_sock_is_held(sk));
1946         if (!md5sig)
1947                 return 0;
1948
1949         if (rcu_dereference_check(hlist_first_rcu(&md5sig->head),
1950                                   lockdep_sock_is_held(sk)))
1951                 return 1;
1952 #endif
1953         return 0;
1954 }
1955
1956 static int tcp_ao_info_cmd(struct sock *sk, unsigned short int family,
1957                            sockptr_t optval, int optlen)
1958 {
1959         struct tcp_ao_key *new_current = NULL, *new_rnext = NULL;
1960         struct tcp_ao_info *ao_info;
1961         struct tcp_ao_info_opt cmd;
1962         bool first = false;
1963         int err;
1964
1965         if (optlen < sizeof(cmd))
1966                 return -EINVAL;
1967
1968         err = copy_struct_from_sockptr(&cmd, sizeof(cmd), optval, optlen);
1969         if (err)
1970                 return err;
1971
1972         if (cmd.set_current || cmd.set_rnext) {
1973                 if (!tcp_ao_can_set_current_rnext(sk))
1974                         return -EINVAL;
1975         }
1976
1977         if (cmd.reserved != 0 || cmd.reserved2 != 0)
1978                 return -EINVAL;
1979
1980         ao_info = setsockopt_ao_info(sk);
1981         if (IS_ERR(ao_info))
1982                 return PTR_ERR(ao_info);
1983         if (!ao_info) {
1984                 if (!((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)))
1985                         return -EINVAL;
1986                 ao_info = tcp_ao_alloc_info(GFP_KERNEL);
1987                 if (!ao_info)
1988                         return -ENOMEM;
1989                 first = true;
1990         }
1991
1992         if (cmd.ao_required && tcp_ao_required_verify(sk)) {
1993                 err = -EKEYREJECTED;
1994                 goto out;
1995         }
1996
1997         /* For sockets in TCP_CLOSED it's possible set keys that aren't
1998          * matching the future peer (address/port/VRF/etc),
1999          * tcp_ao_connect_init() will choose a correct matching MKT
2000          * if there's any.
2001          */
2002         if (cmd.set_current) {
2003                 new_current = tcp_ao_established_key(ao_info, cmd.current_key, -1);
2004                 if (!new_current) {
2005                         err = -ENOENT;
2006                         goto out;
2007                 }
2008         }
2009         if (cmd.set_rnext) {
2010                 new_rnext = tcp_ao_established_key(ao_info, -1, cmd.rnext);
2011                 if (!new_rnext) {
2012                         err = -ENOENT;
2013                         goto out;
2014                 }
2015         }
2016         if (cmd.set_counters) {
2017                 atomic64_set(&ao_info->counters.pkt_good, cmd.pkt_good);
2018                 atomic64_set(&ao_info->counters.pkt_bad, cmd.pkt_bad);
2019                 atomic64_set(&ao_info->counters.key_not_found, cmd.pkt_key_not_found);
2020                 atomic64_set(&ao_info->counters.ao_required, cmd.pkt_ao_required);
2021                 atomic64_set(&ao_info->counters.dropped_icmp, cmd.pkt_dropped_icmp);
2022         }
2023
2024         ao_info->ao_required = cmd.ao_required;
2025         ao_info->accept_icmps = cmd.accept_icmps;
2026         if (new_current)
2027                 WRITE_ONCE(ao_info->current_key, new_current);
2028         if (new_rnext)
2029                 WRITE_ONCE(ao_info->rnext_key, new_rnext);
2030         if (first) {
2031                 if (!static_branch_inc(&tcp_ao_needed.key)) {
2032                         err = -EUSERS;
2033                         goto out;
2034                 }
2035                 sk_gso_disable(sk);
2036                 rcu_assign_pointer(tcp_sk(sk)->ao_info, ao_info);
2037         }
2038         return 0;
2039 out:
2040         if (first)
2041                 kfree(ao_info);
2042         return err;
2043 }
2044
2045 int tcp_parse_ao(struct sock *sk, int cmd, unsigned short int family,
2046                  sockptr_t optval, int optlen)
2047 {
2048         if (WARN_ON_ONCE(family != AF_INET && family != AF_INET6))
2049                 return -EAFNOSUPPORT;
2050
2051         switch (cmd) {
2052         case TCP_AO_ADD_KEY:
2053                 return tcp_ao_add_cmd(sk, family, optval, optlen);
2054         case TCP_AO_DEL_KEY:
2055                 return tcp_ao_del_cmd(sk, family, optval, optlen);
2056         case TCP_AO_INFO:
2057                 return tcp_ao_info_cmd(sk, family, optval, optlen);
2058         default:
2059                 WARN_ON_ONCE(1);
2060                 return -EINVAL;
2061         }
2062 }
2063
2064 int tcp_v4_parse_ao(struct sock *sk, int cmd, sockptr_t optval, int optlen)
2065 {
2066         return tcp_parse_ao(sk, cmd, AF_INET, optval, optlen);
2067 }
2068
2069 /* tcp_ao_copy_mkts_to_user(ao_info, optval, optlen)
2070  *
2071  * @ao_info:    struct tcp_ao_info on the socket that
2072  *              socket getsockopt(TCP_AO_GET_KEYS) is executed on
2073  * @optval:     pointer to array of tcp_ao_getsockopt structures in user space.
2074  *              Must be != NULL.
2075  * @optlen:     pointer to size of tcp_ao_getsockopt structure.
2076  *              Must be != NULL.
2077  *
2078  * Return value: 0 on success, a negative error number otherwise.
2079  *
2080  * optval points to an array of tcp_ao_getsockopt structures in user space.
2081  * optval[0] is used as both input and output to getsockopt. It determines
2082  * which keys are returned by the kernel.
2083  * optval[0].nkeys is the size of the array in user space. On return it contains
2084  * the number of keys matching the search criteria.
2085  * If tcp_ao_getsockopt::get_all is set, then all keys in the socket are
2086  * returned, otherwise only keys matching <addr, prefix, sndid, rcvid>
2087  * in optval[0] are returned.
2088  * optlen is also used as both input and output. The user provides the size
2089  * of struct tcp_ao_getsockopt in user space, and the kernel returns the size
2090  * of the structure in kernel space.
2091  * The size of struct tcp_ao_getsockopt may differ between user and kernel.
2092  * There are three cases to consider:
2093  *  * If usize == ksize, then keys are copied verbatim.
2094  *  * If usize < ksize, then the userspace has passed an old struct to a
2095  *    newer kernel. The rest of the trailing bytes in optval[0]
2096  *    (ksize - usize) are interpreted as 0 by the kernel.
2097  *  * If usize > ksize, then the userspace has passed a new struct to an
2098  *    older kernel. The trailing bytes unknown to the kernel (usize - ksize)
2099  *    are checked to ensure they are zeroed, otherwise -E2BIG is returned.
2100  * On return the kernel fills in min(usize, ksize) in each entry of the array.
2101  * The layout of the fields in the user and kernel structures is expected to
2102  * be the same (including in the 32bit vs 64bit case).
2103  */
2104 static int tcp_ao_copy_mkts_to_user(struct tcp_ao_info *ao_info,
2105                                     sockptr_t optval, sockptr_t optlen)
2106 {
2107         struct tcp_ao_getsockopt opt_in, opt_out;
2108         struct tcp_ao_key *key, *current_key;
2109         bool do_address_matching = true;
2110         union tcp_ao_addr *addr = NULL;
2111         int err, l3index, user_len;
2112         unsigned int max_keys;  /* maximum number of keys to copy to user */
2113         size_t out_offset = 0;
2114         size_t bytes_to_write;  /* number of bytes to write to user level */
2115         u32 matched_keys;       /* keys from ao_info matched so far */
2116         int optlen_out;
2117         __be16 port = 0;
2118
2119         if (copy_from_sockptr(&user_len, optlen, sizeof(int)))
2120                 return -EFAULT;
2121
2122         if (user_len <= 0)
2123                 return -EINVAL;
2124
2125         memset(&opt_in, 0, sizeof(struct tcp_ao_getsockopt));
2126         err = copy_struct_from_sockptr(&opt_in, sizeof(opt_in),
2127                                        optval, user_len);
2128         if (err < 0)
2129                 return err;
2130
2131         if (opt_in.pkt_good || opt_in.pkt_bad)
2132                 return -EINVAL;
2133         if (opt_in.keyflags & ~TCP_AO_GET_KEYF_VALID)
2134                 return -EINVAL;
2135         if (opt_in.ifindex && !(opt_in.keyflags & TCP_AO_KEYF_IFINDEX))
2136                 return -EINVAL;
2137
2138         if (opt_in.reserved != 0)
2139                 return -EINVAL;
2140
2141         max_keys = opt_in.nkeys;
2142         l3index = (opt_in.keyflags & TCP_AO_KEYF_IFINDEX) ? opt_in.ifindex : -1;
2143
2144         if (opt_in.get_all || opt_in.is_current || opt_in.is_rnext) {
2145                 if (opt_in.get_all && (opt_in.is_current || opt_in.is_rnext))
2146                         return -EINVAL;
2147                 do_address_matching = false;
2148         }
2149
2150         switch (opt_in.addr.ss_family) {
2151         case AF_INET: {
2152                 struct sockaddr_in *sin;
2153                 __be32 mask;
2154
2155                 sin = (struct sockaddr_in *)&opt_in.addr;
2156                 port = sin->sin_port;
2157                 addr = (union tcp_ao_addr *)&sin->sin_addr;
2158
2159                 if (opt_in.prefix > 32)
2160                         return -EINVAL;
2161
2162                 if (ntohl(sin->sin_addr.s_addr) == INADDR_ANY &&
2163                     opt_in.prefix != 0)
2164                         return -EINVAL;
2165
2166                 mask = inet_make_mask(opt_in.prefix);
2167                 if (sin->sin_addr.s_addr & ~mask)
2168                         return -EINVAL;
2169
2170                 break;
2171         }
2172         case AF_INET6: {
2173                 struct sockaddr_in6 *sin6;
2174                 struct in6_addr *addr6;
2175
2176                 sin6 = (struct sockaddr_in6 *)&opt_in.addr;
2177                 addr = (union tcp_ao_addr *)&sin6->sin6_addr;
2178                 addr6 = &sin6->sin6_addr;
2179                 port = sin6->sin6_port;
2180
2181                 /* We don't have to change family and @addr here if
2182                  * ipv6_addr_v4mapped() like in key adding:
2183                  * tcp_ao_key_cmp() does it. Do the sanity checks though.
2184                  */
2185                 if (opt_in.prefix != 0) {
2186                         if (ipv6_addr_v4mapped(addr6)) {
2187                                 __be32 mask, addr4 = addr6->s6_addr32[3];
2188
2189                                 if (opt_in.prefix > 32 ||
2190                                     ntohl(addr4) == INADDR_ANY)
2191                                         return -EINVAL;
2192                                 mask = inet_make_mask(opt_in.prefix);
2193                                 if (addr4 & ~mask)
2194                                         return -EINVAL;
2195                         } else {
2196                                 struct in6_addr pfx;
2197
2198                                 if (ipv6_addr_any(addr6) ||
2199                                     opt_in.prefix > 128)
2200                                         return -EINVAL;
2201
2202                                 ipv6_addr_prefix(&pfx, addr6, opt_in.prefix);
2203                                 if (ipv6_addr_cmp(&pfx, addr6))
2204                                         return -EINVAL;
2205                         }
2206                 } else if (!ipv6_addr_any(addr6)) {
2207                         return -EINVAL;
2208                 }
2209                 break;
2210         }
2211         case 0:
2212                 if (!do_address_matching)
2213                         break;
2214                 fallthrough;
2215         default:
2216                 return -EAFNOSUPPORT;
2217         }
2218
2219         if (!do_address_matching) {
2220                 /* We could just ignore those, but let's do stricter checks */
2221                 if (addr || port)
2222                         return -EINVAL;
2223                 if (opt_in.prefix || opt_in.sndid || opt_in.rcvid)
2224                         return -EINVAL;
2225         }
2226
2227         bytes_to_write = min_t(int, user_len, sizeof(struct tcp_ao_getsockopt));
2228         matched_keys = 0;
2229         /* May change in RX, while we're dumping, pre-fetch it */
2230         current_key = READ_ONCE(ao_info->current_key);
2231
2232         hlist_for_each_entry_rcu(key, &ao_info->head, node) {
2233                 if (opt_in.get_all)
2234                         goto match;
2235
2236                 if (opt_in.is_current || opt_in.is_rnext) {
2237                         if (opt_in.is_current && key == current_key)
2238                                 goto match;
2239                         if (opt_in.is_rnext && key == ao_info->rnext_key)
2240                                 goto match;
2241                         continue;
2242                 }
2243
2244                 if (tcp_ao_key_cmp(key, l3index, addr, opt_in.prefix,
2245                                    opt_in.addr.ss_family,
2246                                    opt_in.sndid, opt_in.rcvid) != 0)
2247                         continue;
2248 match:
2249                 matched_keys++;
2250                 if (matched_keys > max_keys)
2251                         continue;
2252
2253                 memset(&opt_out, 0, sizeof(struct tcp_ao_getsockopt));
2254
2255                 if (key->family == AF_INET) {
2256                         struct sockaddr_in *sin_out = (struct sockaddr_in *)&opt_out.addr;
2257
2258                         sin_out->sin_family = key->family;
2259                         sin_out->sin_port = 0;
2260                         memcpy(&sin_out->sin_addr, &key->addr, sizeof(struct in_addr));
2261                 } else {
2262                         struct sockaddr_in6 *sin6_out = (struct sockaddr_in6 *)&opt_out.addr;
2263
2264                         sin6_out->sin6_family = key->family;
2265                         sin6_out->sin6_port = 0;
2266                         memcpy(&sin6_out->sin6_addr, &key->addr, sizeof(struct in6_addr));
2267                 }
2268                 opt_out.sndid = key->sndid;
2269                 opt_out.rcvid = key->rcvid;
2270                 opt_out.prefix = key->prefixlen;
2271                 opt_out.keyflags = key->keyflags;
2272                 opt_out.is_current = (key == current_key);
2273                 opt_out.is_rnext = (key == ao_info->rnext_key);
2274                 opt_out.nkeys = 0;
2275                 opt_out.maclen = key->maclen;
2276                 opt_out.keylen = key->keylen;
2277                 opt_out.ifindex = key->l3index;
2278                 opt_out.pkt_good = atomic64_read(&key->pkt_good);
2279                 opt_out.pkt_bad = atomic64_read(&key->pkt_bad);
2280                 memcpy(&opt_out.key, key->key, key->keylen);
2281                 tcp_sigpool_algo(key->tcp_sigpool_id, opt_out.alg_name, 64);
2282
2283                 /* Copy key to user */
2284                 if (copy_to_sockptr_offset(optval, out_offset,
2285                                            &opt_out, bytes_to_write))
2286                         return -EFAULT;
2287                 out_offset += user_len;
2288         }
2289
2290         optlen_out = (int)sizeof(struct tcp_ao_getsockopt);
2291         if (copy_to_sockptr(optlen, &optlen_out, sizeof(int)))
2292                 return -EFAULT;
2293
2294         out_offset = offsetof(struct tcp_ao_getsockopt, nkeys);
2295         if (copy_to_sockptr_offset(optval, out_offset,
2296                                    &matched_keys, sizeof(u32)))
2297                 return -EFAULT;
2298
2299         return 0;
2300 }
2301
2302 int tcp_ao_get_mkts(struct sock *sk, sockptr_t optval, sockptr_t optlen)
2303 {
2304         struct tcp_ao_info *ao_info;
2305
2306         ao_info = setsockopt_ao_info(sk);
2307         if (IS_ERR(ao_info))
2308                 return PTR_ERR(ao_info);
2309         if (!ao_info)
2310                 return -ENOENT;
2311
2312         return tcp_ao_copy_mkts_to_user(ao_info, optval, optlen);
2313 }
2314
2315 int tcp_ao_get_sock_info(struct sock *sk, sockptr_t optval, sockptr_t optlen)
2316 {
2317         struct tcp_ao_info_opt out, in = {};
2318         struct tcp_ao_key *current_key;
2319         struct tcp_ao_info *ao;
2320         int err, len;
2321
2322         if (copy_from_sockptr(&len, optlen, sizeof(int)))
2323                 return -EFAULT;
2324
2325         if (len <= 0)
2326                 return -EINVAL;
2327
2328         /* Copying this "in" only to check ::reserved, ::reserved2,
2329          * that may be needed to extend (struct tcp_ao_info_opt) and
2330          * what getsockopt() provides in future.
2331          */
2332         err = copy_struct_from_sockptr(&in, sizeof(in), optval, len);
2333         if (err)
2334                 return err;
2335
2336         if (in.reserved != 0 || in.reserved2 != 0)
2337                 return -EINVAL;
2338
2339         ao = setsockopt_ao_info(sk);
2340         if (IS_ERR(ao))
2341                 return PTR_ERR(ao);
2342         if (!ao)
2343                 return -ENOENT;
2344
2345         memset(&out, 0, sizeof(out));
2346         out.ao_required         = ao->ao_required;
2347         out.accept_icmps        = ao->accept_icmps;
2348         out.pkt_good            = atomic64_read(&ao->counters.pkt_good);
2349         out.pkt_bad             = atomic64_read(&ao->counters.pkt_bad);
2350         out.pkt_key_not_found   = atomic64_read(&ao->counters.key_not_found);
2351         out.pkt_ao_required     = atomic64_read(&ao->counters.ao_required);
2352         out.pkt_dropped_icmp    = atomic64_read(&ao->counters.dropped_icmp);
2353
2354         current_key = READ_ONCE(ao->current_key);
2355         if (current_key) {
2356                 out.set_current = 1;
2357                 out.current_key = current_key->sndid;
2358         }
2359         if (ao->rnext_key) {
2360                 out.set_rnext = 1;
2361                 out.rnext = ao->rnext_key->rcvid;
2362         }
2363
2364         if (copy_to_sockptr(optval, &out, min_t(int, len, sizeof(out))))
2365                 return -EFAULT;
2366
2367         return 0;
2368 }
2369
2370 int tcp_ao_set_repair(struct sock *sk, sockptr_t optval, unsigned int optlen)
2371 {
2372         struct tcp_sock *tp = tcp_sk(sk);
2373         struct tcp_ao_repair cmd;
2374         struct tcp_ao_key *key;
2375         struct tcp_ao_info *ao;
2376         int err;
2377
2378         if (optlen < sizeof(cmd))
2379                 return -EINVAL;
2380
2381         err = copy_struct_from_sockptr(&cmd, sizeof(cmd), optval, optlen);
2382         if (err)
2383                 return err;
2384
2385         if (!tp->repair)
2386                 return -EPERM;
2387
2388         ao = setsockopt_ao_info(sk);
2389         if (IS_ERR(ao))
2390                 return PTR_ERR(ao);
2391         if (!ao)
2392                 return -ENOENT;
2393
2394         WRITE_ONCE(ao->lisn, cmd.snt_isn);
2395         WRITE_ONCE(ao->risn, cmd.rcv_isn);
2396         WRITE_ONCE(ao->snd_sne, cmd.snd_sne);
2397         WRITE_ONCE(ao->rcv_sne, cmd.rcv_sne);
2398
2399         hlist_for_each_entry_rcu(key, &ao->head, node)
2400                 tcp_ao_cache_traffic_keys(sk, ao, key);
2401
2402         return 0;
2403 }
2404
2405 int tcp_ao_get_repair(struct sock *sk, sockptr_t optval, sockptr_t optlen)
2406 {
2407         struct tcp_sock *tp = tcp_sk(sk);
2408         struct tcp_ao_repair opt;
2409         struct tcp_ao_info *ao;
2410         int len;
2411
2412         if (copy_from_sockptr(&len, optlen, sizeof(int)))
2413                 return -EFAULT;
2414
2415         if (len <= 0)
2416                 return -EINVAL;
2417
2418         if (!tp->repair)
2419                 return -EPERM;
2420
2421         rcu_read_lock();
2422         ao = getsockopt_ao_info(sk);
2423         if (IS_ERR_OR_NULL(ao)) {
2424                 rcu_read_unlock();
2425                 return ao ? PTR_ERR(ao) : -ENOENT;
2426         }
2427
2428         opt.snt_isn     = ao->lisn;
2429         opt.rcv_isn     = ao->risn;
2430         opt.snd_sne     = READ_ONCE(ao->snd_sne);
2431         opt.rcv_sne     = READ_ONCE(ao->rcv_sne);
2432         rcu_read_unlock();
2433
2434         if (copy_to_sockptr(optval, &opt, min_t(int, len, sizeof(opt))))
2435                 return -EFAULT;
2436         return 0;
2437 }