net/smc/af_smc.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  Shared Memory Communications over RDMA (SMC-R) and RoCE
   4  *
   5  *  AF_SMC protocol family socket handler keeping the AF_INET sock address type
   6  *  applies to SOCK_STREAM sockets only
   7  *  offers an alternative communication option for TCP-protocol sockets
   8  *  applicable with RoCE-cards only
   9  *
  10  *  Initial restrictions:
  11  *    - support for alternate links postponed
  12  *
  13  *  Copyright IBM Corp. 2016, 2018
  14  *
  15  *  Author(s):  Ursula Braun <[email protected]>
  16  *              based on prototype from Frank Blaschka
  17  */
  18
  19 #define KMSG_COMPONENT "smc"
  20 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  21
  22 #include <linux/module.h>
  23 #include <linux/socket.h>
  24 #include <linux/workqueue.h>
  25 #include <linux/in.h>
  26 #include <linux/sched/signal.h>
  27 #include <linux/if_vlan.h>
  28 #include <linux/rcupdate_wait.h>
  29 #include <linux/ctype.h>
  30
  31 #include <net/sock.h>
  32 #include <net/tcp.h>
  33 #include <net/smc.h>
  34 #include <asm/ioctls.h>
  35
  36 #include <net/net_namespace.h>
  37 #include <net/netns/generic.h>
  38 #include "smc_netns.h"
  39
  40 #include "smc.h"
  41 #include "smc_clc.h"
  42 #include "smc_llc.h"
  43 #include "smc_cdc.h"
  44 #include "smc_core.h"
  45 #include "smc_ib.h"
  46 #include "smc_ism.h"
  47 #include "smc_pnet.h"
  48 #include "smc_netlink.h"
  49 #include "smc_tx.h"
  50 #include "smc_rx.h"
  51 #include "smc_close.h"
  52 #include "smc_stats.h"
  53 #include "smc_tracepoint.h"
  54 #include "smc_sysctl.h"
  55
  56 static DEFINE_MUTEX(smc_server_lgr_pending);    /* serialize link group
  57                                                  * creation on server
  58                                                  */
  59 static DEFINE_MUTEX(smc_client_lgr_pending);    /* serialize link group
  60                                                  * creation on client
  61                                                  */
  62
  63 static struct workqueue_struct  *smc_tcp_ls_wq; /* wq for tcp listen work */
  64 struct workqueue_struct *smc_hs_wq;     /* wq for handshake work */
  65 struct workqueue_struct *smc_close_wq;  /* wq for close work */
  66
  67 static void smc_tcp_listen_work(struct work_struct *);
  68 static void smc_connect_work(struct work_struct *);
  69
  70 int smc_nl_dump_hs_limitation(struct sk_buff *skb, struct netlink_callback *cb)
  71 {
  72         struct smc_nl_dmp_ctx *cb_ctx = smc_nl_dmp_ctx(cb);
  73         void *hdr;
  74
  75         if (cb_ctx->pos[0])
  76                 goto out;
  77
  78         hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
  79                           &smc_gen_nl_family, NLM_F_MULTI,
  80                           SMC_NETLINK_DUMP_HS_LIMITATION);
  81         if (!hdr)
  82                 return -ENOMEM;
  83
  84         if (nla_put_u8(skb, SMC_NLA_HS_LIMITATION_ENABLED,
  85                        sock_net(skb->sk)->smc.limit_smc_hs))
  86                 goto err;
  87
  88         genlmsg_end(skb, hdr);
  89         cb_ctx->pos[0] = 1;
  90 out:
  91         return skb->len;
  92 err:
  93         genlmsg_cancel(skb, hdr);
  94         return -EMSGSIZE;
  95 }
  96
  97 int smc_nl_enable_hs_limitation(struct sk_buff *skb, struct genl_info *info)
  98 {
  99         sock_net(skb->sk)->smc.limit_smc_hs = true;
 100         return 0;
 101 }
 102
 103 int smc_nl_disable_hs_limitation(struct sk_buff *skb, struct genl_info *info)
 104 {
 105         sock_net(skb->sk)->smc.limit_smc_hs = false;
 106         return 0;
 107 }
 108
 109 static void smc_set_keepalive(struct sock *sk, int val)
 110 {
 111         struct smc_sock *smc = smc_sk(sk);
 112
 113         smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val);
 114 }
 115
 116 static struct sock *smc_tcp_syn_recv_sock(const struct sock *sk,
 117                                           struct sk_buff *skb,
 118                                           struct request_sock *req,
 119                                           struct dst_entry *dst,
 120                                           struct request_sock *req_unhash,
 121                                           bool *own_req)
 122 {
 123         struct smc_sock *smc;
 124
 125         smc = smc_clcsock_user_data(sk);
 126
 127         if (READ_ONCE(sk->sk_ack_backlog) + atomic_read(&smc->queued_smc_hs) >
 128                                 sk->sk_max_ack_backlog)
 129                 goto drop;
 130
 131         if (sk_acceptq_is_full(&smc->sk)) {
 132                 NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
 133                 goto drop;
 134         }
 135
 136         /* passthrough to original syn recv sock fct */
 137         return smc->ori_af_ops->syn_recv_sock(sk, skb, req, dst, req_unhash,
 138                                               own_req);
 139
 140 drop:
 141         dst_release(dst);
 142         tcp_listendrop(sk);
 143         return NULL;
 144 }
 145
 146 static bool smc_hs_congested(const struct sock *sk)
 147 {
 148         const struct smc_sock *smc;
 149
 150         smc = smc_clcsock_user_data(sk);
 151
 152         if (!smc)
 153                 return true;
 154
 155         if (workqueue_congested(WORK_CPU_UNBOUND, smc_hs_wq))
 156                 return true;
 157
 158         return false;
 159 }
 160
 161 static struct smc_hashinfo smc_v4_hashinfo = {
 162         .lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock),
 163 };
 164
 165 static struct smc_hashinfo smc_v6_hashinfo = {
 166         .lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock),
 167 };
 168
 169 int smc_hash_sk(struct sock *sk)
 170 {
 171         struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
 172         struct hlist_head *head;
 173
 174         head = &h->ht;
 175
 176         write_lock_bh(&h->lock);
 177         sk_add_node(sk, head);
 178         write_unlock_bh(&h->lock);
 179         sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
 180
 181         return 0;
 182 }
 183 EXPORT_SYMBOL_GPL(smc_hash_sk);
 184
 185 void smc_unhash_sk(struct sock *sk)
 186 {
 187         struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
 188
 189         write_lock_bh(&h->lock);
 190         if (sk_del_node_init(sk))
 191                 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
 192         write_unlock_bh(&h->lock);
 193 }
 194 EXPORT_SYMBOL_GPL(smc_unhash_sk);
 195
 196 /* This will be called before user really release sock_lock. So do the
 197  * work which we didn't do because of user hold the sock_lock in the
 198  * BH context
 199  */
 200 static void smc_release_cb(struct sock *sk)
 201 {
 202         struct smc_sock *smc = smc_sk(sk);
 203
 204         if (smc->conn.tx_in_release_sock) {
 205                 smc_tx_pending(&smc->conn);
 206                 smc->conn.tx_in_release_sock = false;
 207         }
 208 }
 209
 210 struct proto smc_proto = {
 211         .name           = "SMC",
 212         .owner          = THIS_MODULE,
 213         .keepalive      = smc_set_keepalive,
 214         .hash           = smc_hash_sk,
 215         .unhash         = smc_unhash_sk,
 216         .release_cb     = smc_release_cb,
 217         .obj_size       = sizeof(struct smc_sock),
 218         .h.smc_hash     = &smc_v4_hashinfo,
 219         .slab_flags     = SLAB_TYPESAFE_BY_RCU,
 220 };
 221 EXPORT_SYMBOL_GPL(smc_proto);
 222
 223 struct proto smc_proto6 = {
 224         .name           = "SMC6",
 225         .owner          = THIS_MODULE,
 226         .keepalive      = smc_set_keepalive,
 227         .hash           = smc_hash_sk,
 228         .unhash         = smc_unhash_sk,
 229         .release_cb     = smc_release_cb,
 230         .obj_size       = sizeof(struct smc_sock),
 231         .h.smc_hash     = &smc_v6_hashinfo,
 232         .slab_flags     = SLAB_TYPESAFE_BY_RCU,
 233 };
 234 EXPORT_SYMBOL_GPL(smc_proto6);
 235
 236 static void smc_restore_fallback_changes(struct smc_sock *smc)
 237 {
 238         if (smc->clcsock->file) { /* non-accepted sockets have no file yet */
 239                 smc->clcsock->file->private_data = smc->sk.sk_socket;
 240                 smc->clcsock->file = NULL;
 241         }
 242 }
 243
 244 static int __smc_release(struct smc_sock *smc)
 245 {
 246         struct sock *sk = &smc->sk;
 247         int rc = 0;
 248
 249         if (!smc->use_fallback) {
 250                 rc = smc_close_active(smc);
 251                 sock_set_flag(sk, SOCK_DEAD);
 252                 sk->sk_shutdown |= SHUTDOWN_MASK;
 253         } else {
 254                 if (sk->sk_state != SMC_CLOSED) {
 255                         if (sk->sk_state != SMC_LISTEN &&
 256                             sk->sk_state != SMC_INIT)
 257                                 sock_put(sk); /* passive closing */
 258                         if (sk->sk_state == SMC_LISTEN) {
 259                                 /* wake up clcsock accept */
 260                                 rc = kernel_sock_shutdown(smc->clcsock,
 261                                                           SHUT_RDWR);
 262                         }
 263                         sk->sk_state = SMC_CLOSED;
 264                         sk->sk_state_change(sk);
 265                 }
 266                 smc_restore_fallback_changes(smc);
 267         }
 268
 269         sk->sk_prot->unhash(sk);
 270
 271         if (sk->sk_state == SMC_CLOSED) {
 272                 if (smc->clcsock) {
 273                         release_sock(sk);
 274                         smc_clcsock_release(smc);
 275                         lock_sock(sk);
 276                 }
 277                 if (!smc->use_fallback)
 278                         smc_conn_free(&smc->conn);
 279         }
 280
 281         return rc;
 282 }
 283
 284 static int smc_release(struct socket *sock)
 285 {
 286         struct sock *sk = sock->sk;
 287         struct smc_sock *smc;
 288         int old_state, rc = 0;
 289
 290         if (!sk)
 291                 goto out;
 292
 293         sock_hold(sk); /* sock_put below */
 294         smc = smc_sk(sk);
 295
 296         old_state = sk->sk_state;
 297
 298         /* cleanup for a dangling non-blocking connect */
 299         if (smc->connect_nonblock && old_state == SMC_INIT)
 300                 tcp_abort(smc->clcsock->sk, ECONNABORTED);
 301
 302         if (cancel_work_sync(&smc->connect_work))
 303                 sock_put(&smc->sk); /* sock_hold in smc_connect for passive closing */
 304
 305         if (sk->sk_state == SMC_LISTEN)
 306                 /* smc_close_non_accepted() is called and acquires
 307                  * sock lock for child sockets again
 308                  */
 309                 lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
 310         else
 311                 lock_sock(sk);
 312
 313         if (old_state == SMC_INIT && sk->sk_state == SMC_ACTIVE &&
 314             !smc->use_fallback)
 315                 smc_close_active_abort(smc);
 316
 317         rc = __smc_release(smc);
 318
 319         /* detach socket */
 320         sock_orphan(sk);
 321         sock->sk = NULL;
 322         release_sock(sk);
 323
 324         sock_put(sk); /* sock_hold above */
 325         sock_put(sk); /* final sock_put */
 326 out:
 327         return rc;
 328 }
 329
 330 static void smc_destruct(struct sock *sk)
 331 {
 332         if (sk->sk_state != SMC_CLOSED)
 333                 return;
 334         if (!sock_flag(sk, SOCK_DEAD))
 335                 return;
 336
 337         sk_refcnt_debug_dec(sk);
 338 }
 339
 340 static struct sock *smc_sock_alloc(struct net *net, struct socket *sock,
 341                                    int protocol)
 342 {
 343         struct smc_sock *smc;
 344         struct proto *prot;
 345         struct sock *sk;
 346
 347         prot = (protocol == SMCPROTO_SMC6) ? &smc_proto6 : &smc_proto;
 348         sk = sk_alloc(net, PF_SMC, GFP_KERNEL, prot, 0);
 349         if (!sk)
 350                 return NULL;
 351
 352         sock_init_data(sock, sk); /* sets sk_refcnt to 1 */
 353         sk->sk_state = SMC_INIT;
 354         sk->sk_destruct = smc_destruct;
 355         sk->sk_protocol = protocol;
 356         smc = smc_sk(sk);
 357         INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
 358         INIT_WORK(&smc->connect_work, smc_connect_work);
 359         INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
 360         INIT_LIST_HEAD(&smc->accept_q);
 361         spin_lock_init(&smc->accept_q_lock);
 362         spin_lock_init(&smc->conn.send_lock);
 363         sk->sk_prot->hash(sk);
 364         sk_refcnt_debug_inc(sk);
 365         mutex_init(&smc->clcsock_release_lock);
 366
 367         return sk;
 368 }
 369
 370 static int smc_bind(struct socket *sock, struct sockaddr *uaddr,
 371                     int addr_len)
 372 {
 373         struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
 374         struct sock *sk = sock->sk;
 375         struct smc_sock *smc;
 376         int rc;
 377
 378         smc = smc_sk(sk);
 379
 380         /* replicate tests from inet_bind(), to be safe wrt. future changes */
 381         rc = -EINVAL;
 382         if (addr_len < sizeof(struct sockaddr_in))
 383                 goto out;
 384
 385         rc = -EAFNOSUPPORT;
 386         if (addr->sin_family != AF_INET &&
 387             addr->sin_family != AF_INET6 &&
 388             addr->sin_family != AF_UNSPEC)
 389                 goto out;
 390         /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
 391         if (addr->sin_family == AF_UNSPEC &&
 392             addr->sin_addr.s_addr != htonl(INADDR_ANY))
 393                 goto out;
 394
 395         lock_sock(sk);
 396
 397         /* Check if socket is already active */
 398         rc = -EINVAL;
 399         if (sk->sk_state != SMC_INIT || smc->connect_nonblock)
 400                 goto out_rel;
 401
 402         smc->clcsock->sk->sk_reuse = sk->sk_reuse;
 403         rc = kernel_bind(smc->clcsock, uaddr, addr_len);
 404
 405 out_rel:
 406         release_sock(sk);
 407 out:
 408         return rc;
 409 }
 410
 411 static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk,
 412                                    unsigned long mask)
 413 {
 414         /* options we don't get control via setsockopt for */
 415         nsk->sk_type = osk->sk_type;
 416         nsk->sk_sndbuf = osk->sk_sndbuf;
 417         nsk->sk_rcvbuf = osk->sk_rcvbuf;
 418         nsk->sk_sndtimeo = osk->sk_sndtimeo;
 419         nsk->sk_rcvtimeo = osk->sk_rcvtimeo;
 420         nsk->sk_mark = osk->sk_mark;
 421         nsk->sk_priority = osk->sk_priority;
 422         nsk->sk_rcvlowat = osk->sk_rcvlowat;
 423         nsk->sk_bound_dev_if = osk->sk_bound_dev_if;
 424         nsk->sk_err = osk->sk_err;
 425
 426         nsk->sk_flags &= ~mask;
 427         nsk->sk_flags |= osk->sk_flags & mask;
 428 }
 429
 430 #define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
 431                              (1UL << SOCK_KEEPOPEN) | \
 432                              (1UL << SOCK_LINGER) | \
 433                              (1UL << SOCK_BROADCAST) | \
 434                              (1UL << SOCK_TIMESTAMP) | \
 435                              (1UL << SOCK_DBG) | \
 436                              (1UL << SOCK_RCVTSTAMP) | \
 437                              (1UL << SOCK_RCVTSTAMPNS) | \
 438                              (1UL << SOCK_LOCALROUTE) | \
 439                              (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
 440                              (1UL << SOCK_RXQ_OVFL) | \
 441                              (1UL << SOCK_WIFI_STATUS) | \
 442                              (1UL << SOCK_NOFCS) | \
 443                              (1UL << SOCK_FILTER_LOCKED) | \
 444                              (1UL << SOCK_TSTAMP_NEW))
 445 /* copy only relevant settings and flags of SOL_SOCKET level from smc to
 446  * clc socket (since smc is not called for these options from net/core)
 447  */
 448 static void smc_copy_sock_settings_to_clc(struct smc_sock *smc)
 449 {
 450         smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC);
 451 }
 452
 453 #define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
 454                              (1UL << SOCK_KEEPOPEN) | \
 455                              (1UL << SOCK_LINGER) | \
 456                              (1UL << SOCK_DBG))
 457 /* copy only settings and flags relevant for smc from clc to smc socket */
 458 static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
 459 {
 460         smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
 461 }
 462
 463 /* register the new rmb on all links */
 464 static int smcr_lgr_reg_rmbs(struct smc_link *link,
 465                              struct smc_buf_desc *rmb_desc)
 466 {
 467         struct smc_link_group *lgr = link->lgr;
 468         int i, rc = 0;
 469
 470         rc = smc_llc_flow_initiate(lgr, SMC_LLC_FLOW_RKEY);
 471         if (rc)
 472                 return rc;
 473         /* protect against parallel smc_llc_cli_rkey_exchange() and
 474          * parallel smcr_link_reg_rmb()
 475          */
 476         mutex_lock(&lgr->llc_conf_mutex);
 477         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
 478                 if (!smc_link_active(&lgr->lnk[i]))
 479                         continue;
 480                 rc = smcr_link_reg_rmb(&lgr->lnk[i], rmb_desc);
 481                 if (rc)
 482                         goto out;
 483         }
 484
 485         /* exchange confirm_rkey msg with peer */
 486         rc = smc_llc_do_confirm_rkey(link, rmb_desc);
 487         if (rc) {
 488                 rc = -EFAULT;
 489                 goto out;
 490         }
 491         rmb_desc->is_conf_rkey = true;
 492 out:
 493         mutex_unlock(&lgr->llc_conf_mutex);
 494         smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
 495         return rc;
 496 }
 497
 498 static int smcr_clnt_conf_first_link(struct smc_sock *smc)
 499 {
 500         struct smc_link *link = smc->conn.lnk;
 501         struct smc_llc_qentry *qentry;
 502         int rc;
 503
 504         /* receive CONFIRM LINK request from server over RoCE fabric */
 505         qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME,
 506                               SMC_LLC_CONFIRM_LINK);
 507         if (!qentry) {
 508                 struct smc_clc_msg_decline dclc;
 509
 510                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 511                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
 512                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
 513         }
 514         smc_llc_save_peer_uid(qentry);
 515         rc = smc_llc_eval_conf_link(qentry, SMC_LLC_REQ);
 516         smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl);
 517         if (rc)
 518                 return SMC_CLC_DECL_RMBE_EC;
 519
 520         rc = smc_ib_modify_qp_rts(link);
 521         if (rc)
 522                 return SMC_CLC_DECL_ERR_RDYLNK;
 523
 524         smc_wr_remember_qp_attr(link);
 525
 526         if (smcr_link_reg_rmb(link, smc->conn.rmb_desc))
 527                 return SMC_CLC_DECL_ERR_REGRMB;
 528
 529         /* confirm_rkey is implicit on 1st contact */
 530         smc->conn.rmb_desc->is_conf_rkey = true;
 531
 532         /* send CONFIRM LINK response over RoCE fabric */
 533         rc = smc_llc_send_confirm_link(link, SMC_LLC_RESP);
 534         if (rc < 0)
 535                 return SMC_CLC_DECL_TIMEOUT_CL;
 536
 537         smc_llc_link_active(link);
 538         smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE);
 539
 540         /* optional 2nd link, receive ADD LINK request from server */
 541         qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME,
 542                               SMC_LLC_ADD_LINK);
 543         if (!qentry) {
 544                 struct smc_clc_msg_decline dclc;
 545
 546                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 547                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
 548                 if (rc == -EAGAIN)
 549                         rc = 0; /* no DECLINE received, go with one link */
 550                 return rc;
 551         }
 552         smc_llc_flow_qentry_clr(&link->lgr->llc_flow_lcl);
 553         smc_llc_cli_add_link(link, qentry);
 554         return 0;
 555 }
 556
 557 static bool smc_isascii(char *hostname)
 558 {
 559         int i;
 560
 561         for (i = 0; i < SMC_MAX_HOSTNAME_LEN; i++)
 562                 if (!isascii(hostname[i]))
 563                         return false;
 564         return true;
 565 }
 566
 567 static void smc_conn_save_peer_info_fce(struct smc_sock *smc,
 568                                         struct smc_clc_msg_accept_confirm *clc)
 569 {
 570         struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
 571                 (struct smc_clc_msg_accept_confirm_v2 *)clc;
 572         struct smc_clc_first_contact_ext *fce;
 573         int clc_v2_len;
 574
 575         if (clc->hdr.version == SMC_V1 ||
 576             !(clc->hdr.typev2 & SMC_FIRST_CONTACT_MASK))
 577                 return;
 578
 579         if (smc->conn.lgr->is_smcd) {
 580                 memcpy(smc->conn.lgr->negotiated_eid, clc_v2->d1.eid,
 581                        SMC_MAX_EID_LEN);
 582                 clc_v2_len = offsetofend(struct smc_clc_msg_accept_confirm_v2,
 583                                          d1);
 584         } else {
 585                 memcpy(smc->conn.lgr->negotiated_eid, clc_v2->r1.eid,
 586                        SMC_MAX_EID_LEN);
 587                 clc_v2_len = offsetofend(struct smc_clc_msg_accept_confirm_v2,
 588                                          r1);
 589         }
 590         fce = (struct smc_clc_first_contact_ext *)(((u8 *)clc_v2) + clc_v2_len);
 591         smc->conn.lgr->peer_os = fce->os_type;
 592         smc->conn.lgr->peer_smc_release = fce->release;
 593         if (smc_isascii(fce->hostname))
 594                 memcpy(smc->conn.lgr->peer_hostname, fce->hostname,
 595                        SMC_MAX_HOSTNAME_LEN);
 596 }
 597
 598 static void smcr_conn_save_peer_info(struct smc_sock *smc,
 599                                      struct smc_clc_msg_accept_confirm *clc)
 600 {
 601         int bufsize = smc_uncompress_bufsize(clc->r0.rmbe_size);
 602
 603         smc->conn.peer_rmbe_idx = clc->r0.rmbe_idx;
 604         smc->conn.local_tx_ctrl.token = ntohl(clc->r0.rmbe_alert_token);
 605         smc->conn.peer_rmbe_size = bufsize;
 606         atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 607         smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1);
 608 }
 609
 610 static void smcd_conn_save_peer_info(struct smc_sock *smc,
 611                                      struct smc_clc_msg_accept_confirm *clc)
 612 {
 613         int bufsize = smc_uncompress_bufsize(clc->d0.dmbe_size);
 614
 615         smc->conn.peer_rmbe_idx = clc->d0.dmbe_idx;
 616         smc->conn.peer_token = clc->d0.token;
 617         /* msg header takes up space in the buffer */
 618         smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg);
 619         atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 620         smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx;
 621 }
 622
 623 static void smc_conn_save_peer_info(struct smc_sock *smc,
 624                                     struct smc_clc_msg_accept_confirm *clc)
 625 {
 626         if (smc->conn.lgr->is_smcd)
 627                 smcd_conn_save_peer_info(smc, clc);
 628         else
 629                 smcr_conn_save_peer_info(smc, clc);
 630         smc_conn_save_peer_info_fce(smc, clc);
 631 }
 632
 633 static void smc_link_save_peer_info(struct smc_link *link,
 634                                     struct smc_clc_msg_accept_confirm *clc,
 635                                     struct smc_init_info *ini)
 636 {
 637         link->peer_qpn = ntoh24(clc->r0.qpn);
 638         memcpy(link->peer_gid, ini->peer_gid, SMC_GID_SIZE);
 639         memcpy(link->peer_mac, ini->peer_mac, sizeof(link->peer_mac));
 640         link->peer_psn = ntoh24(clc->r0.psn);
 641         link->peer_mtu = clc->r0.qp_mtu;
 642 }
 643
 644 static void smc_stat_inc_fback_rsn_cnt(struct smc_sock *smc,
 645                                        struct smc_stats_fback *fback_arr)
 646 {
 647         int cnt;
 648
 649         for (cnt = 0; cnt < SMC_MAX_FBACK_RSN_CNT; cnt++) {
 650                 if (fback_arr[cnt].fback_code == smc->fallback_rsn) {
 651                         fback_arr[cnt].count++;
 652                         break;
 653                 }
 654                 if (!fback_arr[cnt].fback_code) {
 655                         fback_arr[cnt].fback_code = smc->fallback_rsn;
 656                         fback_arr[cnt].count++;
 657                         break;
 658                 }
 659         }
 660 }
 661
 662 static void smc_stat_fallback(struct smc_sock *smc)
 663 {
 664         struct net *net = sock_net(&smc->sk);
 665
 666         mutex_lock(&net->smc.mutex_fback_rsn);
 667         if (smc->listen_smc) {
 668                 smc_stat_inc_fback_rsn_cnt(smc, net->smc.fback_rsn->srv);
 669                 net->smc.fback_rsn->srv_fback_cnt++;
 670         } else {
 671                 smc_stat_inc_fback_rsn_cnt(smc, net->smc.fback_rsn->clnt);
 672                 net->smc.fback_rsn->clnt_fback_cnt++;
 673         }
 674         mutex_unlock(&net->smc.mutex_fback_rsn);
 675 }
 676
 677 /* must be called under rcu read lock */
 678 static void smc_fback_wakeup_waitqueue(struct smc_sock *smc, void *key)
 679 {
 680         struct socket_wq *wq;
 681         __poll_t flags;
 682
 683         wq = rcu_dereference(smc->sk.sk_wq);
 684         if (!skwq_has_sleeper(wq))
 685                 return;
 686
 687         /* wake up smc sk->sk_wq */
 688         if (!key) {
 689                 /* sk_state_change */
 690                 wake_up_interruptible_all(&wq->wait);
 691         } else {
 692                 flags = key_to_poll(key);
 693                 if (flags & (EPOLLIN | EPOLLOUT))
 694                         /* sk_data_ready or sk_write_space */
 695                         wake_up_interruptible_sync_poll(&wq->wait, flags);
 696                 else if (flags & EPOLLERR)
 697                         /* sk_error_report */
 698                         wake_up_interruptible_poll(&wq->wait, flags);
 699         }
 700 }
 701
 702 static int smc_fback_mark_woken(wait_queue_entry_t *wait,
 703                                 unsigned int mode, int sync, void *key)
 704 {
 705         struct smc_mark_woken *mark =
 706                 container_of(wait, struct smc_mark_woken, wait_entry);
 707
 708         mark->woken = true;
 709         mark->key = key;
 710         return 0;
 711 }
 712
 713 static void smc_fback_forward_wakeup(struct smc_sock *smc, struct sock *clcsk,
 714                                      void (*clcsock_callback)(struct sock *sk))
 715 {
 716         struct smc_mark_woken mark = { .woken = false };
 717         struct socket_wq *wq;
 718
 719         init_waitqueue_func_entry(&mark.wait_entry,
 720                                   smc_fback_mark_woken);
 721         rcu_read_lock();
 722         wq = rcu_dereference(clcsk->sk_wq);
 723         if (!wq)
 724                 goto out;
 725         add_wait_queue(sk_sleep(clcsk), &mark.wait_entry);
 726         clcsock_callback(clcsk);
 727         remove_wait_queue(sk_sleep(clcsk), &mark.wait_entry);
 728
 729         if (mark.woken)
 730                 smc_fback_wakeup_waitqueue(smc, mark.key);
 731 out:
 732         rcu_read_unlock();
 733 }
 734
 735 static void smc_fback_state_change(struct sock *clcsk)
 736 {
 737         struct smc_sock *smc =
 738                 smc_clcsock_user_data(clcsk);
 739
 740         if (!smc)
 741                 return;
 742         smc_fback_forward_wakeup(smc, clcsk, smc->clcsk_state_change);
 743 }
 744
 745 static void smc_fback_data_ready(struct sock *clcsk)
 746 {
 747         struct smc_sock *smc =
 748                 smc_clcsock_user_data(clcsk);
 749
 750         if (!smc)
 751                 return;
 752         smc_fback_forward_wakeup(smc, clcsk, smc->clcsk_data_ready);
 753 }
 754
 755 static void smc_fback_write_space(struct sock *clcsk)
 756 {
 757         struct smc_sock *smc =
 758                 smc_clcsock_user_data(clcsk);
 759
 760         if (!smc)
 761                 return;
 762         smc_fback_forward_wakeup(smc, clcsk, smc->clcsk_write_space);
 763 }
 764
 765 static void smc_fback_error_report(struct sock *clcsk)
 766 {
 767         struct smc_sock *smc =
 768                 smc_clcsock_user_data(clcsk);
 769
 770         if (!smc)
 771                 return;
 772         smc_fback_forward_wakeup(smc, clcsk, smc->clcsk_error_report);
 773 }
 774
 775 static int smc_switch_to_fallback(struct smc_sock *smc, int reason_code)
 776 {
 777         struct sock *clcsk;
 778         int rc = 0;
 779
 780         mutex_lock(&smc->clcsock_release_lock);
 781         if (!smc->clcsock) {
 782                 rc = -EBADF;
 783                 goto out;
 784         }
 785         clcsk = smc->clcsock->sk;
 786
 787         if (smc->use_fallback)
 788                 goto out;
 789         smc->use_fallback = true;
 790         smc->fallback_rsn = reason_code;
 791         smc_stat_fallback(smc);
 792         trace_smc_switch_to_fallback(smc, reason_code);
 793         if (smc->sk.sk_socket && smc->sk.sk_socket->file) {
 794                 smc->clcsock->file = smc->sk.sk_socket->file;
 795                 smc->clcsock->file->private_data = smc->clcsock;
 796                 smc->clcsock->wq.fasync_list =
 797                         smc->sk.sk_socket->wq.fasync_list;
 798
 799                 /* There might be some wait entries remaining
 800                  * in smc sk->sk_wq and they should be woken up
 801                  * as clcsock's wait queue is woken up.
 802                  */
 803                 smc->clcsk_state_change = clcsk->sk_state_change;
 804                 smc->clcsk_data_ready = clcsk->sk_data_ready;
 805                 smc->clcsk_write_space = clcsk->sk_write_space;
 806                 smc->clcsk_error_report = clcsk->sk_error_report;
 807
 808                 clcsk->sk_state_change = smc_fback_state_change;
 809                 clcsk->sk_data_ready = smc_fback_data_ready;
 810                 clcsk->sk_write_space = smc_fback_write_space;
 811                 clcsk->sk_error_report = smc_fback_error_report;
 812
 813                 smc->clcsock->sk->sk_user_data =
 814                         (void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY);
 815         }
 816 out:
 817         mutex_unlock(&smc->clcsock_release_lock);
 818         return rc;
 819 }
 820
 821 /* fall back during connect */
 822 static int smc_connect_fallback(struct smc_sock *smc, int reason_code)
 823 {
 824         struct net *net = sock_net(&smc->sk);
 825         int rc = 0;
 826
 827         rc = smc_switch_to_fallback(smc, reason_code);
 828         if (rc) { /* fallback fails */
 829                 this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt);
 830                 if (smc->sk.sk_state == SMC_INIT)
 831                         sock_put(&smc->sk); /* passive closing */
 832                 return rc;
 833         }
 834         smc_copy_sock_settings_to_clc(smc);
 835         smc->connect_nonblock = 0;
 836         if (smc->sk.sk_state == SMC_INIT)
 837                 smc->sk.sk_state = SMC_ACTIVE;
 838         return 0;
 839 }
 840
 841 /* decline and fall back during connect */
 842 static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code,
 843                                         u8 version)
 844 {
 845         struct net *net = sock_net(&smc->sk);
 846         int rc;
 847
 848         if (reason_code < 0) { /* error, fallback is not possible */
 849                 this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt);
 850                 if (smc->sk.sk_state == SMC_INIT)
 851                         sock_put(&smc->sk); /* passive closing */
 852                 return reason_code;
 853         }
 854         if (reason_code != SMC_CLC_DECL_PEERDECL) {
 855                 rc = smc_clc_send_decline(smc, reason_code, version);
 856                 if (rc < 0) {
 857                         this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt);
 858                         if (smc->sk.sk_state == SMC_INIT)
 859                                 sock_put(&smc->sk); /* passive closing */
 860                         return rc;
 861                 }
 862         }
 863         return smc_connect_fallback(smc, reason_code);
 864 }
 865
 866 static void smc_conn_abort(struct smc_sock *smc, int local_first)
 867 {
 868         struct smc_connection *conn = &smc->conn;
 869         struct smc_link_group *lgr = conn->lgr;
 870         bool lgr_valid = false;
 871
 872         if (smc_conn_lgr_valid(conn))
 873                 lgr_valid = true;
 874
 875         smc_conn_free(conn);
 876         if (local_first && lgr_valid)
 877                 smc_lgr_cleanup_early(lgr);
 878 }
 879
 880 /* check if there is a rdma device available for this connection. */
 881 /* called for connect and listen */
 882 static int smc_find_rdma_device(struct smc_sock *smc, struct smc_init_info *ini)
 883 {
 884         /* PNET table look up: search active ib_device and port
 885          * within same PNETID that also contains the ethernet device
 886          * used for the internal TCP socket
 887          */
 888         smc_pnet_find_roce_resource(smc->clcsock->sk, ini);
 889         if (!ini->check_smcrv2 && !ini->ib_dev)
 890                 return SMC_CLC_DECL_NOSMCRDEV;
 891         if (ini->check_smcrv2 && !ini->smcrv2.ib_dev_v2)
 892                 return SMC_CLC_DECL_NOSMCRDEV;
 893         return 0;
 894 }
 895
 896 /* check if there is an ISM device available for this connection. */
 897 /* called for connect and listen */
 898 static int smc_find_ism_device(struct smc_sock *smc, struct smc_init_info *ini)
 899 {
 900         /* Find ISM device with same PNETID as connecting interface  */
 901         smc_pnet_find_ism_resource(smc->clcsock->sk, ini);
 902         if (!ini->ism_dev[0])
 903                 return SMC_CLC_DECL_NOSMCDDEV;
 904         else
 905                 ini->ism_chid[0] = smc_ism_get_chid(ini->ism_dev[0]);
 906         return 0;
 907 }
 908
 909 /* is chid unique for the ism devices that are already determined? */
 910 static bool smc_find_ism_v2_is_unique_chid(u16 chid, struct smc_init_info *ini,
 911                                            int cnt)
 912 {
 913         int i = (!ini->ism_dev[0]) ? 1 : 0;
 914
 915         for (; i < cnt; i++)
 916                 if (ini->ism_chid[i] == chid)
 917                         return false;
 918         return true;
 919 }
 920
 921 /* determine possible V2 ISM devices (either without PNETID or with PNETID plus
 922  * PNETID matching net_device)
 923  */
 924 static int smc_find_ism_v2_device_clnt(struct smc_sock *smc,
 925                                        struct smc_init_info *ini)
 926 {
 927         int rc = SMC_CLC_DECL_NOSMCDDEV;
 928         struct smcd_dev *smcd;
 929         int i = 1;
 930         u16 chid;
 931
 932         if (smcd_indicated(ini->smc_type_v1))
 933                 rc = 0;         /* already initialized for V1 */
 934         mutex_lock(&smcd_dev_list.mutex);
 935         list_for_each_entry(smcd, &smcd_dev_list.list, list) {
 936                 if (smcd->going_away || smcd == ini->ism_dev[0])
 937                         continue;
 938                 chid = smc_ism_get_chid(smcd);
 939                 if (!smc_find_ism_v2_is_unique_chid(chid, ini, i))
 940                         continue;
 941                 if (!smc_pnet_is_pnetid_set(smcd->pnetid) ||
 942                     smc_pnet_is_ndev_pnetid(sock_net(&smc->sk), smcd->pnetid)) {
 943                         ini->ism_dev[i] = smcd;
 944                         ini->ism_chid[i] = chid;
 945                         ini->is_smcd = true;
 946                         rc = 0;
 947                         i++;
 948                         if (i > SMC_MAX_ISM_DEVS)
 949                                 break;
 950                 }
 951         }
 952         mutex_unlock(&smcd_dev_list.mutex);
 953         ini->ism_offered_cnt = i - 1;
 954         if (!ini->ism_dev[0] && !ini->ism_dev[1])
 955                 ini->smcd_version = 0;
 956
 957         return rc;
 958 }
 959
 960 /* Check for VLAN ID and register it on ISM device just for CLC handshake */
 961 static int smc_connect_ism_vlan_setup(struct smc_sock *smc,
 962                                       struct smc_init_info *ini)
 963 {
 964         if (ini->vlan_id && smc_ism_get_vlan(ini->ism_dev[0], ini->vlan_id))
 965                 return SMC_CLC_DECL_ISMVLANERR;
 966         return 0;
 967 }
 968
 969 static int smc_find_proposal_devices(struct smc_sock *smc,
 970                                      struct smc_init_info *ini)
 971 {
 972         int rc = 0;
 973
 974         /* check if there is an ism device available */
 975         if (!(ini->smcd_version & SMC_V1) ||
 976             smc_find_ism_device(smc, ini) ||
 977             smc_connect_ism_vlan_setup(smc, ini))
 978                 ini->smcd_version &= ~SMC_V1;
 979         /* else ISM V1 is supported for this connection */
 980
 981         /* check if there is an rdma device available */
 982         if (!(ini->smcr_version & SMC_V1) ||
 983             smc_find_rdma_device(smc, ini))
 984                 ini->smcr_version &= ~SMC_V1;
 985         /* else RDMA is supported for this connection */
 986
 987         ini->smc_type_v1 = smc_indicated_type(ini->smcd_version & SMC_V1,
 988                                               ini->smcr_version & SMC_V1);
 989
 990         /* check if there is an ism v2 device available */
 991         if (!(ini->smcd_version & SMC_V2) ||
 992             !smc_ism_is_v2_capable() ||
 993             smc_find_ism_v2_device_clnt(smc, ini))
 994                 ini->smcd_version &= ~SMC_V2;
 995
 996         /* check if there is an rdma v2 device available */
 997         ini->check_smcrv2 = true;
 998         ini->smcrv2.saddr = smc->clcsock->sk->sk_rcv_saddr;
 999         if (!(ini->smcr_version & SMC_V2) ||
1000             smc->clcsock->sk->sk_family != AF_INET ||
1001             !smc_clc_ueid_count() ||
1002             smc_find_rdma_device(smc, ini))
1003                 ini->smcr_version &= ~SMC_V2;
1004         ini->check_smcrv2 = false;
1005
1006         ini->smc_type_v2 = smc_indicated_type(ini->smcd_version & SMC_V2,
1007                                               ini->smcr_version & SMC_V2);
1008
1009         /* if neither ISM nor RDMA are supported, fallback */
1010         if (ini->smc_type_v1 == SMC_TYPE_N && ini->smc_type_v2 == SMC_TYPE_N)
1011                 rc = SMC_CLC_DECL_NOSMCDEV;
1012
1013         return rc;
1014 }
1015
1016 /* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
1017  * used, the VLAN ID will be registered again during the connection setup.
1018  */
1019 static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc,
1020                                         struct smc_init_info *ini)
1021 {
1022         if (!smcd_indicated(ini->smc_type_v1))
1023                 return 0;
1024         if (ini->vlan_id && smc_ism_put_vlan(ini->ism_dev[0], ini->vlan_id))
1025                 return SMC_CLC_DECL_CNFERR;
1026         return 0;
1027 }
1028
1029 #define SMC_CLC_MAX_ACCEPT_LEN \
1030         (sizeof(struct smc_clc_msg_accept_confirm_v2) + \
1031          sizeof(struct smc_clc_first_contact_ext) + \
1032          sizeof(struct smc_clc_msg_trail))
1033
1034 /* CLC handshake during connect */
1035 static int smc_connect_clc(struct smc_sock *smc,
1036                            struct smc_clc_msg_accept_confirm_v2 *aclc2,
1037                            struct smc_init_info *ini)
1038 {
1039         int rc = 0;
1040
1041         /* do inband token exchange */
1042         rc = smc_clc_send_proposal(smc, ini);
1043         if (rc)
1044                 return rc;
1045         /* receive SMC Accept CLC message */
1046         return smc_clc_wait_msg(smc, aclc2, SMC_CLC_MAX_ACCEPT_LEN,
1047                                 SMC_CLC_ACCEPT, CLC_WAIT_TIME);
1048 }
1049
1050 void smc_fill_gid_list(struct smc_link_group *lgr,
1051                        struct smc_gidlist *gidlist,
1052                        struct smc_ib_device *known_dev, u8 *known_gid)
1053 {
1054         struct smc_init_info *alt_ini = NULL;
1055
1056         memset(gidlist, 0, sizeof(*gidlist));
1057         memcpy(gidlist->list[gidlist->len++], known_gid, SMC_GID_SIZE);
1058
1059         alt_ini = kzalloc(sizeof(*alt_ini), GFP_KERNEL);
1060         if (!alt_ini)
1061                 goto out;
1062
1063         alt_ini->vlan_id = lgr->vlan_id;
1064         alt_ini->check_smcrv2 = true;
1065         alt_ini->smcrv2.saddr = lgr->saddr;
1066         smc_pnet_find_alt_roce(lgr, alt_ini, known_dev);
1067
1068         if (!alt_ini->smcrv2.ib_dev_v2)
1069                 goto out;
1070
1071         memcpy(gidlist->list[gidlist->len++], alt_ini->smcrv2.ib_gid_v2,
1072                SMC_GID_SIZE);
1073
1074 out:
1075         kfree(alt_ini);
1076 }
1077
1078 static int smc_connect_rdma_v2_prepare(struct smc_sock *smc,
1079                                        struct smc_clc_msg_accept_confirm *aclc,
1080                                        struct smc_init_info *ini)
1081 {
1082         struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
1083                 (struct smc_clc_msg_accept_confirm_v2 *)aclc;
1084         struct smc_clc_first_contact_ext *fce =
1085                 (struct smc_clc_first_contact_ext *)
1086                         (((u8 *)clc_v2) + sizeof(*clc_v2));
1087
1088         if (!ini->first_contact_peer || aclc->hdr.version == SMC_V1)
1089                 return 0;
1090
1091         if (fce->v2_direct) {
1092                 memcpy(ini->smcrv2.nexthop_mac, &aclc->r0.lcl.mac, ETH_ALEN);
1093                 ini->smcrv2.uses_gateway = false;
1094         } else {
1095                 if (smc_ib_find_route(smc->clcsock->sk->sk_rcv_saddr,
1096                                       smc_ib_gid_to_ipv4(aclc->r0.lcl.gid),
1097                                       ini->smcrv2.nexthop_mac,
1098                                       &ini->smcrv2.uses_gateway))
1099                         return SMC_CLC_DECL_NOROUTE;
1100                 if (!ini->smcrv2.uses_gateway) {
1101                         /* mismatch: peer claims indirect, but its direct */
1102                         return SMC_CLC_DECL_NOINDIRECT;
1103                 }
1104         }
1105         return 0;
1106 }
1107
1108 /* setup for RDMA connection of client */
1109 static int smc_connect_rdma(struct smc_sock *smc,
1110                             struct smc_clc_msg_accept_confirm *aclc,
1111                             struct smc_init_info *ini)
1112 {
1113         int i, reason_code = 0;
1114         struct smc_link *link;
1115         u8 *eid = NULL;
1116
1117         ini->is_smcd = false;
1118         ini->ib_clcqpn = ntoh24(aclc->r0.qpn);
1119         ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK;
1120         memcpy(ini->peer_systemid, aclc->r0.lcl.id_for_peer, SMC_SYSTEMID_LEN);
1121         memcpy(ini->peer_gid, aclc->r0.lcl.gid, SMC_GID_SIZE);
1122         memcpy(ini->peer_mac, aclc->r0.lcl.mac, ETH_ALEN);
1123
1124         reason_code = smc_connect_rdma_v2_prepare(smc, aclc, ini);
1125         if (reason_code)
1126                 return reason_code;
1127
1128         mutex_lock(&smc_client_lgr_pending);
1129         reason_code = smc_conn_create(smc, ini);
1130         if (reason_code) {
1131                 mutex_unlock(&smc_client_lgr_pending);
1132                 return reason_code;
1133         }
1134
1135         smc_conn_save_peer_info(smc, aclc);
1136
1137         if (ini->first_contact_local) {
1138                 link = smc->conn.lnk;
1139         } else {
1140                 /* set link that was assigned by server */
1141                 link = NULL;
1142                 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1143                         struct smc_link *l = &smc->conn.lgr->lnk[i];
1144
1145                         if (l->peer_qpn == ntoh24(aclc->r0.qpn) &&
1146                             !memcmp(l->peer_gid, &aclc->r0.lcl.gid,
1147                                     SMC_GID_SIZE) &&
1148                             (aclc->hdr.version > SMC_V1 ||
1149                              !memcmp(l->peer_mac, &aclc->r0.lcl.mac,
1150                                      sizeof(l->peer_mac)))) {
1151                                 link = l;
1152                                 break;
1153                         }
1154                 }
1155                 if (!link) {
1156                         reason_code = SMC_CLC_DECL_NOSRVLINK;
1157                         goto connect_abort;
1158                 }
1159                 smc_switch_link_and_count(&smc->conn, link);
1160         }
1161
1162         /* create send buffer and rmb */
1163         if (smc_buf_create(smc, false)) {
1164                 reason_code = SMC_CLC_DECL_MEM;
1165                 goto connect_abort;
1166         }
1167
1168         if (ini->first_contact_local)
1169                 smc_link_save_peer_info(link, aclc, ini);
1170
1171         if (smc_rmb_rtoken_handling(&smc->conn, link, aclc)) {
1172                 reason_code = SMC_CLC_DECL_ERR_RTOK;
1173                 goto connect_abort;
1174         }
1175
1176         smc_close_init(smc);
1177         smc_rx_init(smc);
1178
1179         if (ini->first_contact_local) {
1180                 if (smc_ib_ready_link(link)) {
1181                         reason_code = SMC_CLC_DECL_ERR_RDYLNK;
1182                         goto connect_abort;
1183                 }
1184         } else {
1185                 if (smcr_lgr_reg_rmbs(link, smc->conn.rmb_desc)) {
1186                         reason_code = SMC_CLC_DECL_ERR_REGRMB;
1187                         goto connect_abort;
1188                 }
1189         }
1190         smc_rmb_sync_sg_for_device(&smc->conn);
1191
1192         if (aclc->hdr.version > SMC_V1) {
1193                 struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
1194                         (struct smc_clc_msg_accept_confirm_v2 *)aclc;
1195
1196                 eid = clc_v2->r1.eid;
1197                 if (ini->first_contact_local)
1198                         smc_fill_gid_list(link->lgr, &ini->smcrv2.gidlist,
1199                                           link->smcibdev, link->gid);
1200         }
1201
1202         reason_code = smc_clc_send_confirm(smc, ini->first_contact_local,
1203                                            aclc->hdr.version, eid, ini);
1204         if (reason_code)
1205                 goto connect_abort;
1206
1207         smc_tx_init(smc);
1208
1209         if (ini->first_contact_local) {
1210                 /* QP confirmation over RoCE fabric */
1211                 smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK);
1212                 reason_code = smcr_clnt_conf_first_link(smc);
1213                 smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl);
1214                 if (reason_code)
1215                         goto connect_abort;
1216         }
1217         mutex_unlock(&smc_client_lgr_pending);
1218
1219         smc_copy_sock_settings_to_clc(smc);
1220         smc->connect_nonblock = 0;
1221         if (smc->sk.sk_state == SMC_INIT)
1222                 smc->sk.sk_state = SMC_ACTIVE;
1223
1224         return 0;
1225 connect_abort:
1226         smc_conn_abort(smc, ini->first_contact_local);
1227         mutex_unlock(&smc_client_lgr_pending);
1228         smc->connect_nonblock = 0;
1229
1230         return reason_code;
1231 }
1232
1233 /* The server has chosen one of the proposed ISM devices for the communication.
1234  * Determine from the CHID of the received CLC ACCEPT the ISM device chosen.
1235  */
1236 static int
1237 smc_v2_determine_accepted_chid(struct smc_clc_msg_accept_confirm_v2 *aclc,
1238                                struct smc_init_info *ini)
1239 {
1240         int i;
1241
1242         for (i = 0; i < ini->ism_offered_cnt + 1; i++) {
1243                 if (ini->ism_chid[i] == ntohs(aclc->d1.chid)) {
1244                         ini->ism_selected = i;
1245                         return 0;
1246                 }
1247         }
1248
1249         return -EPROTO;
1250 }
1251
1252 /* setup for ISM connection of client */
1253 static int smc_connect_ism(struct smc_sock *smc,
1254                            struct smc_clc_msg_accept_confirm *aclc,
1255                            struct smc_init_info *ini)
1256 {
1257         u8 *eid = NULL;
1258         int rc = 0;
1259
1260         ini->is_smcd = true;
1261         ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK;
1262
1263         if (aclc->hdr.version == SMC_V2) {
1264                 struct smc_clc_msg_accept_confirm_v2 *aclc_v2 =
1265                         (struct smc_clc_msg_accept_confirm_v2 *)aclc;
1266
1267                 rc = smc_v2_determine_accepted_chid(aclc_v2, ini);
1268                 if (rc)
1269                         return rc;
1270         }
1271         ini->ism_peer_gid[ini->ism_selected] = aclc->d0.gid;
1272
1273         /* there is only one lgr role for SMC-D; use server lock */
1274         mutex_lock(&smc_server_lgr_pending);
1275         rc = smc_conn_create(smc, ini);
1276         if (rc) {
1277                 mutex_unlock(&smc_server_lgr_pending);
1278                 return rc;
1279         }
1280
1281         /* Create send and receive buffers */
1282         rc = smc_buf_create(smc, true);
1283         if (rc) {
1284                 rc = (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB : SMC_CLC_DECL_MEM;
1285                 goto connect_abort;
1286         }
1287
1288         smc_conn_save_peer_info(smc, aclc);
1289         smc_close_init(smc);
1290         smc_rx_init(smc);
1291         smc_tx_init(smc);
1292
1293         if (aclc->hdr.version > SMC_V1) {
1294                 struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
1295                         (struct smc_clc_msg_accept_confirm_v2 *)aclc;
1296
1297                 eid = clc_v2->d1.eid;
1298         }
1299
1300         rc = smc_clc_send_confirm(smc, ini->first_contact_local,
1301                                   aclc->hdr.version, eid, NULL);
1302         if (rc)
1303                 goto connect_abort;
1304         mutex_unlock(&smc_server_lgr_pending);
1305
1306         smc_copy_sock_settings_to_clc(smc);
1307         smc->connect_nonblock = 0;
1308         if (smc->sk.sk_state == SMC_INIT)
1309                 smc->sk.sk_state = SMC_ACTIVE;
1310
1311         return 0;
1312 connect_abort:
1313         smc_conn_abort(smc, ini->first_contact_local);
1314         mutex_unlock(&smc_server_lgr_pending);
1315         smc->connect_nonblock = 0;
1316
1317         return rc;
1318 }
1319
1320 /* check if received accept type and version matches a proposed one */
1321 static int smc_connect_check_aclc(struct smc_init_info *ini,
1322                                   struct smc_clc_msg_accept_confirm *aclc)
1323 {
1324         if (aclc->hdr.typev1 != SMC_TYPE_R &&
1325             aclc->hdr.typev1 != SMC_TYPE_D)
1326                 return SMC_CLC_DECL_MODEUNSUPP;
1327
1328         if (aclc->hdr.version >= SMC_V2) {
1329                 if ((aclc->hdr.typev1 == SMC_TYPE_R &&
1330                      !smcr_indicated(ini->smc_type_v2)) ||
1331                     (aclc->hdr.typev1 == SMC_TYPE_D &&
1332                      !smcd_indicated(ini->smc_type_v2)))
1333                         return SMC_CLC_DECL_MODEUNSUPP;
1334         } else {
1335                 if ((aclc->hdr.typev1 == SMC_TYPE_R &&
1336                      !smcr_indicated(ini->smc_type_v1)) ||
1337                     (aclc->hdr.typev1 == SMC_TYPE_D &&
1338                      !smcd_indicated(ini->smc_type_v1)))
1339                         return SMC_CLC_DECL_MODEUNSUPP;
1340         }
1341
1342         return 0;
1343 }
1344
1345 /* perform steps before actually connecting */
1346 static int __smc_connect(struct smc_sock *smc)
1347 {
1348         u8 version = smc_ism_is_v2_capable() ? SMC_V2 : SMC_V1;
1349         struct smc_clc_msg_accept_confirm_v2 *aclc2;
1350         struct smc_clc_msg_accept_confirm *aclc;
1351         struct smc_init_info *ini = NULL;
1352         u8 *buf = NULL;
1353         int rc = 0;
1354
1355         if (smc->use_fallback)
1356                 return smc_connect_fallback(smc, smc->fallback_rsn);
1357
1358         /* if peer has not signalled SMC-capability, fall back */
1359         if (!tcp_sk(smc->clcsock->sk)->syn_smc)
1360                 return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC);
1361
1362         /* IPSec connections opt out of SMC optimizations */
1363         if (using_ipsec(smc))
1364                 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC,
1365                                                     version);
1366
1367         ini = kzalloc(sizeof(*ini), GFP_KERNEL);
1368         if (!ini)
1369                 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_MEM,
1370                                                     version);
1371
1372         ini->smcd_version = SMC_V1 | SMC_V2;
1373         ini->smcr_version = SMC_V1 | SMC_V2;
1374         ini->smc_type_v1 = SMC_TYPE_B;
1375         ini->smc_type_v2 = SMC_TYPE_B;
1376
1377         /* get vlan id from IP device */
1378         if (smc_vlan_by_tcpsk(smc->clcsock, ini)) {
1379                 ini->smcd_version &= ~SMC_V1;
1380                 ini->smcr_version = 0;
1381                 ini->smc_type_v1 = SMC_TYPE_N;
1382                 if (!ini->smcd_version) {
1383                         rc = SMC_CLC_DECL_GETVLANERR;
1384                         goto fallback;
1385                 }
1386         }
1387
1388         rc = smc_find_proposal_devices(smc, ini);
1389         if (rc)
1390                 goto fallback;
1391
1392         buf = kzalloc(SMC_CLC_MAX_ACCEPT_LEN, GFP_KERNEL);
1393         if (!buf) {
1394                 rc = SMC_CLC_DECL_MEM;
1395                 goto fallback;
1396         }
1397         aclc2 = (struct smc_clc_msg_accept_confirm_v2 *)buf;
1398         aclc = (struct smc_clc_msg_accept_confirm *)aclc2;
1399
1400         /* perform CLC handshake */
1401         rc = smc_connect_clc(smc, aclc2, ini);
1402         if (rc) {
1403                 /* -EAGAIN on timeout, see tcp_recvmsg() */
1404                 if (rc == -EAGAIN) {
1405                         rc = -ETIMEDOUT;
1406                         smc->sk.sk_err = ETIMEDOUT;
1407                 }
1408                 goto vlan_cleanup;
1409         }
1410
1411         /* check if smc modes and versions of CLC proposal and accept match */
1412         rc = smc_connect_check_aclc(ini, aclc);
1413         version = aclc->hdr.version == SMC_V1 ? SMC_V1 : SMC_V2;
1414         if (rc)
1415                 goto vlan_cleanup;
1416
1417         /* depending on previous steps, connect using rdma or ism */
1418         if (aclc->hdr.typev1 == SMC_TYPE_R) {
1419                 ini->smcr_version = version;
1420                 rc = smc_connect_rdma(smc, aclc, ini);
1421         } else if (aclc->hdr.typev1 == SMC_TYPE_D) {
1422                 ini->smcd_version = version;
1423                 rc = smc_connect_ism(smc, aclc, ini);
1424         }
1425         if (rc)
1426                 goto vlan_cleanup;
1427
1428         SMC_STAT_CLNT_SUCC_INC(sock_net(smc->clcsock->sk), aclc);
1429         smc_connect_ism_vlan_cleanup(smc, ini);
1430         kfree(buf);
1431         kfree(ini);
1432         return 0;
1433
1434 vlan_cleanup:
1435         smc_connect_ism_vlan_cleanup(smc, ini);
1436         kfree(buf);
1437 fallback:
1438         kfree(ini);
1439         return smc_connect_decline_fallback(smc, rc, version);
1440 }
1441
1442 static void smc_connect_work(struct work_struct *work)
1443 {
1444         struct smc_sock *smc = container_of(work, struct smc_sock,
1445                                             connect_work);
1446         long timeo = smc->sk.sk_sndtimeo;
1447         int rc = 0;
1448
1449         if (!timeo)
1450                 timeo = MAX_SCHEDULE_TIMEOUT;
1451         lock_sock(smc->clcsock->sk);
1452         if (smc->clcsock->sk->sk_err) {
1453                 smc->sk.sk_err = smc->clcsock->sk->sk_err;
1454         } else if ((1 << smc->clcsock->sk->sk_state) &
1455                                         (TCPF_SYN_SENT | TCPF_SYN_RECV)) {
1456                 rc = sk_stream_wait_connect(smc->clcsock->sk, &timeo);
1457                 if ((rc == -EPIPE) &&
1458                     ((1 << smc->clcsock->sk->sk_state) &
1459                                         (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)))
1460                         rc = 0;
1461         }
1462         release_sock(smc->clcsock->sk);
1463         lock_sock(&smc->sk);
1464         if (rc != 0 || smc->sk.sk_err) {
1465                 smc->sk.sk_state = SMC_CLOSED;
1466                 if (rc == -EPIPE || rc == -EAGAIN)
1467                         smc->sk.sk_err = EPIPE;
1468                 else if (signal_pending(current))
1469                         smc->sk.sk_err = -sock_intr_errno(timeo);
1470                 sock_put(&smc->sk); /* passive closing */
1471                 goto out;
1472         }
1473
1474         rc = __smc_connect(smc);
1475         if (rc < 0)
1476                 smc->sk.sk_err = -rc;
1477
1478 out:
1479         if (!sock_flag(&smc->sk, SOCK_DEAD)) {
1480                 if (smc->sk.sk_err) {
1481                         smc->sk.sk_state_change(&smc->sk);
1482                 } else { /* allow polling before and after fallback decision */
1483                         smc->clcsock->sk->sk_write_space(smc->clcsock->sk);
1484                         smc->sk.sk_write_space(&smc->sk);
1485                 }
1486         }
1487         release_sock(&smc->sk);
1488 }
1489
1490 static int smc_connect(struct socket *sock, struct sockaddr *addr,
1491                        int alen, int flags)
1492 {
1493         struct sock *sk = sock->sk;
1494         struct smc_sock *smc;
1495         int rc = -EINVAL;
1496
1497         smc = smc_sk(sk);
1498
1499         /* separate smc parameter checking to be safe */
1500         if (alen < sizeof(addr->sa_family))
1501                 goto out_err;
1502         if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
1503                 goto out_err;
1504
1505         lock_sock(sk);
1506         switch (sk->sk_state) {
1507         default:
1508                 goto out;
1509         case SMC_ACTIVE:
1510                 rc = -EISCONN;
1511                 goto out;
1512         case SMC_INIT:
1513                 break;
1514         }
1515
1516         smc_copy_sock_settings_to_clc(smc);
1517         tcp_sk(smc->clcsock->sk)->syn_smc = 1;
1518         if (smc->connect_nonblock) {
1519                 rc = -EALREADY;
1520                 goto out;
1521         }
1522         rc = kernel_connect(smc->clcsock, addr, alen, flags);
1523         if (rc && rc != -EINPROGRESS)
1524                 goto out;
1525
1526         sock_hold(&smc->sk); /* sock put in passive closing */
1527         if (smc->use_fallback)
1528                 goto out;
1529         if (flags & O_NONBLOCK) {
1530                 if (queue_work(smc_hs_wq, &smc->connect_work))
1531                         smc->connect_nonblock = 1;
1532                 rc = -EINPROGRESS;
1533         } else {
1534                 rc = __smc_connect(smc);
1535                 if (rc < 0)
1536                         goto out;
1537                 else
1538                         rc = 0; /* success cases including fallback */
1539         }
1540
1541 out:
1542         release_sock(sk);
1543 out_err:
1544         return rc;
1545 }
1546
1547 static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc)
1548 {
1549         struct socket *new_clcsock = NULL;
1550         struct sock *lsk = &lsmc->sk;
1551         struct sock *new_sk;
1552         int rc = -EINVAL;
1553
1554         release_sock(lsk);
1555         new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol);
1556         if (!new_sk) {
1557                 rc = -ENOMEM;
1558                 lsk->sk_err = ENOMEM;
1559                 *new_smc = NULL;
1560                 lock_sock(lsk);
1561                 goto out;
1562         }
1563         *new_smc = smc_sk(new_sk);
1564
1565         mutex_lock(&lsmc->clcsock_release_lock);
1566         if (lsmc->clcsock)
1567                 rc = kernel_accept(lsmc->clcsock, &new_clcsock, SOCK_NONBLOCK);
1568         mutex_unlock(&lsmc->clcsock_release_lock);
1569         lock_sock(lsk);
1570         if  (rc < 0 && rc != -EAGAIN)
1571                 lsk->sk_err = -rc;
1572         if (rc < 0 || lsk->sk_state == SMC_CLOSED) {
1573                 new_sk->sk_prot->unhash(new_sk);
1574                 if (new_clcsock)
1575                         sock_release(new_clcsock);
1576                 new_sk->sk_state = SMC_CLOSED;
1577                 sock_set_flag(new_sk, SOCK_DEAD);
1578                 sock_put(new_sk); /* final */
1579                 *new_smc = NULL;
1580                 goto out;
1581         }
1582
1583         /* new clcsock has inherited the smc listen-specific sk_data_ready
1584          * function; switch it back to the original sk_data_ready function
1585          */
1586         new_clcsock->sk->sk_data_ready = lsmc->clcsk_data_ready;
1587         (*new_smc)->clcsock = new_clcsock;
1588 out:
1589         return rc;
1590 }
1591
1592 /* add a just created sock to the accept queue of the listen sock as
1593  * candidate for a following socket accept call from user space
1594  */
1595 static void smc_accept_enqueue(struct sock *parent, struct sock *sk)
1596 {
1597         struct smc_sock *par = smc_sk(parent);
1598
1599         sock_hold(sk); /* sock_put in smc_accept_unlink () */
1600         spin_lock(&par->accept_q_lock);
1601         list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q);
1602         spin_unlock(&par->accept_q_lock);
1603         sk_acceptq_added(parent);
1604 }
1605
1606 /* remove a socket from the accept queue of its parental listening socket */
1607 static void smc_accept_unlink(struct sock *sk)
1608 {
1609         struct smc_sock *par = smc_sk(sk)->listen_smc;
1610
1611         spin_lock(&par->accept_q_lock);
1612         list_del_init(&smc_sk(sk)->accept_q);
1613         spin_unlock(&par->accept_q_lock);
1614         sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk);
1615         sock_put(sk); /* sock_hold in smc_accept_enqueue */
1616 }
1617
1618 /* remove a sock from the accept queue to bind it to a new socket created
1619  * for a socket accept call from user space
1620  */
1621 struct sock *smc_accept_dequeue(struct sock *parent,
1622                                 struct socket *new_sock)
1623 {
1624         struct smc_sock *isk, *n;
1625         struct sock *new_sk;
1626
1627         list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) {
1628                 new_sk = (struct sock *)isk;
1629
1630                 smc_accept_unlink(new_sk);
1631                 if (new_sk->sk_state == SMC_CLOSED) {
1632                         new_sk->sk_prot->unhash(new_sk);
1633                         if (isk->clcsock) {
1634                                 sock_release(isk->clcsock);
1635                                 isk->clcsock = NULL;
1636                         }
1637                         sock_put(new_sk); /* final */
1638                         continue;
1639                 }
1640                 if (new_sock) {
1641                         sock_graft(new_sk, new_sock);
1642                         if (isk->use_fallback) {
1643                                 smc_sk(new_sk)->clcsock->file = new_sock->file;
1644                                 isk->clcsock->file->private_data = isk->clcsock;
1645                         }
1646                 }
1647                 return new_sk;
1648         }
1649         return NULL;
1650 }
1651
1652 /* clean up for a created but never accepted sock */
1653 void smc_close_non_accepted(struct sock *sk)
1654 {
1655         struct smc_sock *smc = smc_sk(sk);
1656
1657         sock_hold(sk); /* sock_put below */
1658         lock_sock(sk);
1659         if (!sk->sk_lingertime)
1660                 /* wait for peer closing */
1661                 sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT;
1662         __smc_release(smc);
1663         release_sock(sk);
1664         sock_put(sk); /* sock_hold above */
1665         sock_put(sk); /* final sock_put */
1666 }
1667
1668 static int smcr_serv_conf_first_link(struct smc_sock *smc)
1669 {
1670         struct smc_link *link = smc->conn.lnk;
1671         struct smc_llc_qentry *qentry;
1672         int rc;
1673
1674         if (smcr_link_reg_rmb(link, smc->conn.rmb_desc))
1675                 return SMC_CLC_DECL_ERR_REGRMB;
1676
1677         /* send CONFIRM LINK request to client over the RoCE fabric */
1678         rc = smc_llc_send_confirm_link(link, SMC_LLC_REQ);
1679         if (rc < 0)
1680                 return SMC_CLC_DECL_TIMEOUT_CL;
1681
1682         /* receive CONFIRM LINK response from client over the RoCE fabric */
1683         qentry = smc_llc_wait(link->lgr, link, SMC_LLC_WAIT_TIME,
1684                               SMC_LLC_CONFIRM_LINK);
1685         if (!qentry) {
1686                 struct smc_clc_msg_decline dclc;
1687
1688                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
1689                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
1690                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
1691         }
1692         smc_llc_save_peer_uid(qentry);
1693         rc = smc_llc_eval_conf_link(qentry, SMC_LLC_RESP);
1694         smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl);
1695         if (rc)
1696                 return SMC_CLC_DECL_RMBE_EC;
1697
1698         /* confirm_rkey is implicit on 1st contact */
1699         smc->conn.rmb_desc->is_conf_rkey = true;
1700
1701         smc_llc_link_active(link);
1702         smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE);
1703
1704         /* initial contact - try to establish second link */
1705         smc_llc_srv_add_link(link, NULL);
1706         return 0;
1707 }
1708
1709 /* listen worker: finish */
1710 static void smc_listen_out(struct smc_sock *new_smc)
1711 {
1712         struct smc_sock *lsmc = new_smc->listen_smc;
1713         struct sock *newsmcsk = &new_smc->sk;
1714
1715         if (tcp_sk(new_smc->clcsock->sk)->syn_smc)
1716                 atomic_dec(&lsmc->queued_smc_hs);
1717
1718         if (lsmc->sk.sk_state == SMC_LISTEN) {
1719                 lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING);
1720                 smc_accept_enqueue(&lsmc->sk, newsmcsk);
1721                 release_sock(&lsmc->sk);
1722         } else { /* no longer listening */
1723                 smc_close_non_accepted(newsmcsk);
1724         }
1725
1726         /* Wake up accept */
1727         lsmc->sk.sk_data_ready(&lsmc->sk);
1728         sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */
1729 }
1730
1731 /* listen worker: finish in state connected */
1732 static void smc_listen_out_connected(struct smc_sock *new_smc)
1733 {
1734         struct sock *newsmcsk = &new_smc->sk;
1735
1736         sk_refcnt_debug_inc(newsmcsk);
1737         if (newsmcsk->sk_state == SMC_INIT)
1738                 newsmcsk->sk_state = SMC_ACTIVE;
1739
1740         smc_listen_out(new_smc);
1741 }
1742
1743 /* listen worker: finish in error state */
1744 static void smc_listen_out_err(struct smc_sock *new_smc)
1745 {
1746         struct sock *newsmcsk = &new_smc->sk;
1747         struct net *net = sock_net(newsmcsk);
1748
1749         this_cpu_inc(net->smc.smc_stats->srv_hshake_err_cnt);
1750         if (newsmcsk->sk_state == SMC_INIT)
1751                 sock_put(&new_smc->sk); /* passive closing */
1752         newsmcsk->sk_state = SMC_CLOSED;
1753
1754         smc_listen_out(new_smc);
1755 }
1756
1757 /* listen worker: decline and fall back if possible */
1758 static void smc_listen_decline(struct smc_sock *new_smc, int reason_code,
1759                                int local_first, u8 version)
1760 {
1761         /* RDMA setup failed, switch back to TCP */
1762         smc_conn_abort(new_smc, local_first);
1763         if (reason_code < 0 ||
1764             smc_switch_to_fallback(new_smc, reason_code)) {
1765                 /* error, no fallback possible */
1766                 smc_listen_out_err(new_smc);
1767                 return;
1768         }
1769         if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) {
1770                 if (smc_clc_send_decline(new_smc, reason_code, version) < 0) {
1771                         smc_listen_out_err(new_smc);
1772                         return;
1773                 }
1774         }
1775         smc_listen_out_connected(new_smc);
1776 }
1777
1778 /* listen worker: version checking */
1779 static int smc_listen_v2_check(struct smc_sock *new_smc,
1780                                struct smc_clc_msg_proposal *pclc,
1781                                struct smc_init_info *ini)
1782 {
1783         struct smc_clc_smcd_v2_extension *pclc_smcd_v2_ext;
1784         struct smc_clc_v2_extension *pclc_v2_ext;
1785         int rc = SMC_CLC_DECL_PEERNOSMC;
1786
1787         ini->smc_type_v1 = pclc->hdr.typev1;
1788         ini->smc_type_v2 = pclc->hdr.typev2;
1789         ini->smcd_version = smcd_indicated(ini->smc_type_v1) ? SMC_V1 : 0;
1790         ini->smcr_version = smcr_indicated(ini->smc_type_v1) ? SMC_V1 : 0;
1791         if (pclc->hdr.version > SMC_V1) {
1792                 if (smcd_indicated(ini->smc_type_v2))
1793                         ini->smcd_version |= SMC_V2;
1794                 if (smcr_indicated(ini->smc_type_v2))
1795                         ini->smcr_version |= SMC_V2;
1796         }
1797         if (!(ini->smcd_version & SMC_V2) && !(ini->smcr_version & SMC_V2)) {
1798                 rc = SMC_CLC_DECL_PEERNOSMC;
1799                 goto out;
1800         }
1801         pclc_v2_ext = smc_get_clc_v2_ext(pclc);
1802         if (!pclc_v2_ext) {
1803                 ini->smcd_version &= ~SMC_V2;
1804                 ini->smcr_version &= ~SMC_V2;
1805                 rc = SMC_CLC_DECL_NOV2EXT;
1806                 goto out;
1807         }
1808         pclc_smcd_v2_ext = smc_get_clc_smcd_v2_ext(pclc_v2_ext);
1809         if (ini->smcd_version & SMC_V2) {
1810                 if (!smc_ism_is_v2_capable()) {
1811                         ini->smcd_version &= ~SMC_V2;
1812                         rc = SMC_CLC_DECL_NOISM2SUPP;
1813                 } else if (!pclc_smcd_v2_ext) {
1814                         ini->smcd_version &= ~SMC_V2;
1815                         rc = SMC_CLC_DECL_NOV2DEXT;
1816                 } else if (!pclc_v2_ext->hdr.eid_cnt &&
1817                            !pclc_v2_ext->hdr.flag.seid) {
1818                         ini->smcd_version &= ~SMC_V2;
1819                         rc = SMC_CLC_DECL_NOUEID;
1820                 }
1821         }
1822         if (ini->smcr_version & SMC_V2) {
1823                 if (!pclc_v2_ext->hdr.eid_cnt) {
1824                         ini->smcr_version &= ~SMC_V2;
1825                         rc = SMC_CLC_DECL_NOUEID;
1826                 }
1827         }
1828
1829 out:
1830         if (!ini->smcd_version && !ini->smcr_version)
1831                 return rc;
1832
1833         return 0;
1834 }
1835
1836 /* listen worker: check prefixes */
1837 static int smc_listen_prfx_check(struct smc_sock *new_smc,
1838                                  struct smc_clc_msg_proposal *pclc)
1839 {
1840         struct smc_clc_msg_proposal_prefix *pclc_prfx;
1841         struct socket *newclcsock = new_smc->clcsock;
1842
1843         if (pclc->hdr.typev1 == SMC_TYPE_N)
1844                 return 0;
1845         pclc_prfx = smc_clc_proposal_get_prefix(pclc);
1846         if (smc_clc_prfx_match(newclcsock, pclc_prfx))
1847                 return SMC_CLC_DECL_DIFFPREFIX;
1848
1849         return 0;
1850 }
1851
1852 /* listen worker: initialize connection and buffers */
1853 static int smc_listen_rdma_init(struct smc_sock *new_smc,
1854                                 struct smc_init_info *ini)
1855 {
1856         int rc;
1857
1858         /* allocate connection / link group */
1859         rc = smc_conn_create(new_smc, ini);
1860         if (rc)
1861                 return rc;
1862
1863         /* create send buffer and rmb */
1864         if (smc_buf_create(new_smc, false))
1865                 return SMC_CLC_DECL_MEM;
1866
1867         return 0;
1868 }
1869
1870 /* listen worker: initialize connection and buffers for SMC-D */
1871 static int smc_listen_ism_init(struct smc_sock *new_smc,
1872                                struct smc_init_info *ini)
1873 {
1874         int rc;
1875
1876         rc = smc_conn_create(new_smc, ini);
1877         if (rc)
1878                 return rc;
1879
1880         /* Create send and receive buffers */
1881         rc = smc_buf_create(new_smc, true);
1882         if (rc) {
1883                 smc_conn_abort(new_smc, ini->first_contact_local);
1884                 return (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB :
1885                                          SMC_CLC_DECL_MEM;
1886         }
1887
1888         return 0;
1889 }
1890
1891 static bool smc_is_already_selected(struct smcd_dev *smcd,
1892                                     struct smc_init_info *ini,
1893                                     int matches)
1894 {
1895         int i;
1896
1897         for (i = 0; i < matches; i++)
1898                 if (smcd == ini->ism_dev[i])
1899                         return true;
1900
1901         return false;
1902 }
1903
1904 /* check for ISM devices matching proposed ISM devices */
1905 static void smc_check_ism_v2_match(struct smc_init_info *ini,
1906                                    u16 proposed_chid, u64 proposed_gid,
1907                                    unsigned int *matches)
1908 {
1909         struct smcd_dev *smcd;
1910
1911         list_for_each_entry(smcd, &smcd_dev_list.list, list) {
1912                 if (smcd->going_away)
1913                         continue;
1914                 if (smc_is_already_selected(smcd, ini, *matches))
1915                         continue;
1916                 if (smc_ism_get_chid(smcd) == proposed_chid &&
1917                     !smc_ism_cantalk(proposed_gid, ISM_RESERVED_VLANID, smcd)) {
1918                         ini->ism_peer_gid[*matches] = proposed_gid;
1919                         ini->ism_dev[*matches] = smcd;
1920                         (*matches)++;
1921                         break;
1922                 }
1923         }
1924 }
1925
1926 static void smc_find_ism_store_rc(u32 rc, struct smc_init_info *ini)
1927 {
1928         if (!ini->rc)
1929                 ini->rc = rc;
1930 }
1931
1932 static void smc_find_ism_v2_device_serv(struct smc_sock *new_smc,
1933                                         struct smc_clc_msg_proposal *pclc,
1934                                         struct smc_init_info *ini)
1935 {
1936         struct smc_clc_smcd_v2_extension *smcd_v2_ext;
1937         struct smc_clc_v2_extension *smc_v2_ext;
1938         struct smc_clc_msg_smcd *pclc_smcd;
1939         unsigned int matches = 0;
1940         u8 smcd_version;
1941         u8 *eid = NULL;
1942         int i, rc;
1943
1944         if (!(ini->smcd_version & SMC_V2) || !smcd_indicated(ini->smc_type_v2))
1945                 goto not_found;
1946
1947         pclc_smcd = smc_get_clc_msg_smcd(pclc);
1948         smc_v2_ext = smc_get_clc_v2_ext(pclc);
1949         smcd_v2_ext = smc_get_clc_smcd_v2_ext(smc_v2_ext);
1950
1951         mutex_lock(&smcd_dev_list.mutex);
1952         if (pclc_smcd->ism.chid)
1953                 /* check for ISM device matching proposed native ISM device */
1954                 smc_check_ism_v2_match(ini, ntohs(pclc_smcd->ism.chid),
1955                                        ntohll(pclc_smcd->ism.gid), &matches);
1956         for (i = 1; i <= smc_v2_ext->hdr.ism_gid_cnt; i++) {
1957                 /* check for ISM devices matching proposed non-native ISM
1958                  * devices
1959                  */
1960                 smc_check_ism_v2_match(ini,
1961                                        ntohs(smcd_v2_ext->gidchid[i - 1].chid),
1962                                        ntohll(smcd_v2_ext->gidchid[i - 1].gid),
1963                                        &matches);
1964         }
1965         mutex_unlock(&smcd_dev_list.mutex);
1966
1967         if (!ini->ism_dev[0]) {
1968                 smc_find_ism_store_rc(SMC_CLC_DECL_NOSMCD2DEV, ini);
1969                 goto not_found;
1970         }
1971
1972         smc_ism_get_system_eid(&eid);
1973         if (!smc_clc_match_eid(ini->negotiated_eid, smc_v2_ext,
1974                                smcd_v2_ext->system_eid, eid))
1975                 goto not_found;
1976
1977         /* separate - outside the smcd_dev_list.lock */
1978         smcd_version = ini->smcd_version;
1979         for (i = 0; i < matches; i++) {
1980                 ini->smcd_version = SMC_V2;
1981                 ini->is_smcd = true;
1982                 ini->ism_selected = i;
1983                 rc = smc_listen_ism_init(new_smc, ini);
1984                 if (rc) {
1985                         smc_find_ism_store_rc(rc, ini);
1986                         /* try next active ISM device */
1987                         continue;
1988                 }
1989                 return; /* matching and usable V2 ISM device found */
1990         }
1991         /* no V2 ISM device could be initialized */
1992         ini->smcd_version = smcd_version;       /* restore original value */
1993         ini->negotiated_eid[0] = 0;
1994
1995 not_found:
1996         ini->smcd_version &= ~SMC_V2;
1997         ini->ism_dev[0] = NULL;
1998         ini->is_smcd = false;
1999 }
2000
2001 static void smc_find_ism_v1_device_serv(struct smc_sock *new_smc,
2002                                         struct smc_clc_msg_proposal *pclc,
2003                                         struct smc_init_info *ini)
2004 {
2005         struct smc_clc_msg_smcd *pclc_smcd = smc_get_clc_msg_smcd(pclc);
2006         int rc = 0;
2007
2008         /* check if ISM V1 is available */
2009         if (!(ini->smcd_version & SMC_V1) || !smcd_indicated(ini->smc_type_v1))
2010                 goto not_found;
2011         ini->is_smcd = true; /* prepare ISM check */
2012         ini->ism_peer_gid[0] = ntohll(pclc_smcd->ism.gid);
2013         rc = smc_find_ism_device(new_smc, ini);
2014         if (rc)
2015                 goto not_found;
2016         ini->ism_selected = 0;
2017         rc = smc_listen_ism_init(new_smc, ini);
2018         if (!rc)
2019                 return;         /* V1 ISM device found */
2020
2021 not_found:
2022         smc_find_ism_store_rc(rc, ini);
2023         ini->smcd_version &= ~SMC_V1;
2024         ini->ism_dev[0] = NULL;
2025         ini->is_smcd = false;
2026 }
2027
2028 /* listen worker: register buffers */
2029 static int smc_listen_rdma_reg(struct smc_sock *new_smc, bool local_first)
2030 {
2031         struct smc_connection *conn = &new_smc->conn;
2032
2033         if (!local_first) {
2034                 if (smcr_lgr_reg_rmbs(conn->lnk, conn->rmb_desc))
2035                         return SMC_CLC_DECL_ERR_REGRMB;
2036         }
2037         smc_rmb_sync_sg_for_device(&new_smc->conn);
2038
2039         return 0;
2040 }
2041
2042 static void smc_find_rdma_v2_device_serv(struct smc_sock *new_smc,
2043                                          struct smc_clc_msg_proposal *pclc,
2044                                          struct smc_init_info *ini)
2045 {
2046         struct smc_clc_v2_extension *smc_v2_ext;
2047         u8 smcr_version;
2048         int rc;
2049
2050         if (!(ini->smcr_version & SMC_V2) || !smcr_indicated(ini->smc_type_v2))
2051                 goto not_found;
2052
2053         smc_v2_ext = smc_get_clc_v2_ext(pclc);
2054         if (!smc_clc_match_eid(ini->negotiated_eid, smc_v2_ext, NULL, NULL))
2055                 goto not_found;
2056
2057         /* prepare RDMA check */
2058         memcpy(ini->peer_systemid, pclc->lcl.id_for_peer, SMC_SYSTEMID_LEN);
2059         memcpy(ini->peer_gid, smc_v2_ext->roce, SMC_GID_SIZE);
2060         memcpy(ini->peer_mac, pclc->lcl.mac, ETH_ALEN);
2061         ini->check_smcrv2 = true;
2062         ini->smcrv2.clc_sk = new_smc->clcsock->sk;
2063         ini->smcrv2.saddr = new_smc->clcsock->sk->sk_rcv_saddr;
2064         ini->smcrv2.daddr = smc_ib_gid_to_ipv4(smc_v2_ext->roce);
2065         rc = smc_find_rdma_device(new_smc, ini);
2066         if (rc) {
2067                 smc_find_ism_store_rc(rc, ini);
2068                 goto not_found;
2069         }
2070         if (!ini->smcrv2.uses_gateway)
2071                 memcpy(ini->smcrv2.nexthop_mac, pclc->lcl.mac, ETH_ALEN);
2072
2073         smcr_version = ini->smcr_version;
2074         ini->smcr_version = SMC_V2;
2075         rc = smc_listen_rdma_init(new_smc, ini);
2076         if (!rc)
2077                 rc = smc_listen_rdma_reg(new_smc, ini->first_contact_local);
2078         if (!rc)
2079                 return;
2080         ini->smcr_version = smcr_version;
2081         smc_find_ism_store_rc(rc, ini);
2082
2083 not_found:
2084         ini->smcr_version &= ~SMC_V2;
2085         ini->check_smcrv2 = false;
2086 }
2087
2088 static int smc_find_rdma_v1_device_serv(struct smc_sock *new_smc,
2089                                         struct smc_clc_msg_proposal *pclc,
2090                                         struct smc_init_info *ini)
2091 {
2092         int rc;
2093
2094         if (!(ini->smcr_version & SMC_V1) || !smcr_indicated(ini->smc_type_v1))
2095                 return SMC_CLC_DECL_NOSMCDEV;
2096
2097         /* prepare RDMA check */
2098         memcpy(ini->peer_systemid, pclc->lcl.id_for_peer, SMC_SYSTEMID_LEN);
2099         memcpy(ini->peer_gid, pclc->lcl.gid, SMC_GID_SIZE);
2100         memcpy(ini->peer_mac, pclc->lcl.mac, ETH_ALEN);
2101         rc = smc_find_rdma_device(new_smc, ini);
2102         if (rc) {
2103                 /* no RDMA device found */
2104                 return SMC_CLC_DECL_NOSMCDEV;
2105         }
2106         rc = smc_listen_rdma_init(new_smc, ini);
2107         if (rc)
2108                 return rc;
2109         return smc_listen_rdma_reg(new_smc, ini->first_contact_local);
2110 }
2111
2112 /* determine the local device matching to proposal */
2113 static int smc_listen_find_device(struct smc_sock *new_smc,
2114                                   struct smc_clc_msg_proposal *pclc,
2115                                   struct smc_init_info *ini)
2116 {
2117         int prfx_rc;
2118
2119         /* check for ISM device matching V2 proposed device */
2120         smc_find_ism_v2_device_serv(new_smc, pclc, ini);
2121         if (ini->ism_dev[0])
2122                 return 0;
2123
2124         /* check for matching IP prefix and subnet length (V1) */
2125         prfx_rc = smc_listen_prfx_check(new_smc, pclc);
2126         if (prfx_rc)
2127                 smc_find_ism_store_rc(prfx_rc, ini);
2128
2129         /* get vlan id from IP device */
2130         if (smc_vlan_by_tcpsk(new_smc->clcsock, ini))
2131                 return ini->rc ?: SMC_CLC_DECL_GETVLANERR;
2132
2133         /* check for ISM device matching V1 proposed device */
2134         if (!prfx_rc)
2135                 smc_find_ism_v1_device_serv(new_smc, pclc, ini);
2136         if (ini->ism_dev[0])
2137                 return 0;
2138
2139         if (!smcr_indicated(pclc->hdr.typev1) &&
2140             !smcr_indicated(pclc->hdr.typev2))
2141                 /* skip RDMA and decline */
2142                 return ini->rc ?: SMC_CLC_DECL_NOSMCDDEV;
2143
2144         /* check if RDMA V2 is available */
2145         smc_find_rdma_v2_device_serv(new_smc, pclc, ini);
2146         if (ini->smcrv2.ib_dev_v2)
2147                 return 0;
2148
2149         /* check if RDMA V1 is available */
2150         if (!prfx_rc) {
2151                 int rc;
2152
2153                 rc = smc_find_rdma_v1_device_serv(new_smc, pclc, ini);
2154                 smc_find_ism_store_rc(rc, ini);
2155                 return (!rc) ? 0 : ini->rc;
2156         }
2157         return SMC_CLC_DECL_NOSMCDEV;
2158 }
2159
2160 /* listen worker: finish RDMA setup */
2161 static int smc_listen_rdma_finish(struct smc_sock *new_smc,
2162                                   struct smc_clc_msg_accept_confirm *cclc,
2163                                   bool local_first,
2164                                   struct smc_init_info *ini)
2165 {
2166         struct smc_link *link = new_smc->conn.lnk;
2167         int reason_code = 0;
2168
2169         if (local_first)
2170                 smc_link_save_peer_info(link, cclc, ini);
2171
2172         if (smc_rmb_rtoken_handling(&new_smc->conn, link, cclc))
2173                 return SMC_CLC_DECL_ERR_RTOK;
2174
2175         if (local_first) {
2176                 if (smc_ib_ready_link(link))
2177                         return SMC_CLC_DECL_ERR_RDYLNK;
2178                 /* QP confirmation over RoCE fabric */
2179                 smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK);
2180                 reason_code = smcr_serv_conf_first_link(new_smc);
2181                 smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl);
2182         }
2183         return reason_code;
2184 }
2185
2186 /* setup for connection of server */
2187 static void smc_listen_work(struct work_struct *work)
2188 {
2189         struct smc_sock *new_smc = container_of(work, struct smc_sock,
2190                                                 smc_listen_work);
2191         struct socket *newclcsock = new_smc->clcsock;
2192         struct smc_clc_msg_accept_confirm *cclc;
2193         struct smc_clc_msg_proposal_area *buf;
2194         struct smc_clc_msg_proposal *pclc;
2195         struct smc_init_info *ini = NULL;
2196         u8 proposal_version = SMC_V1;
2197         u8 accept_version;
2198         int rc = 0;
2199
2200         if (new_smc->listen_smc->sk.sk_state != SMC_LISTEN)
2201                 return smc_listen_out_err(new_smc);
2202
2203         if (new_smc->use_fallback) {
2204                 smc_listen_out_connected(new_smc);
2205                 return;
2206         }
2207
2208         /* check if peer is smc capable */
2209         if (!tcp_sk(newclcsock->sk)->syn_smc) {
2210                 rc = smc_switch_to_fallback(new_smc, SMC_CLC_DECL_PEERNOSMC);
2211                 if (rc)
2212                         smc_listen_out_err(new_smc);
2213                 else
2214                         smc_listen_out_connected(new_smc);
2215                 return;
2216         }
2217
2218         /* do inband token exchange -
2219          * wait for and receive SMC Proposal CLC message
2220          */
2221         buf = kzalloc(sizeof(*buf), GFP_KERNEL);
2222         if (!buf) {
2223                 rc = SMC_CLC_DECL_MEM;
2224                 goto out_decl;
2225         }
2226         pclc = (struct smc_clc_msg_proposal *)buf;
2227         rc = smc_clc_wait_msg(new_smc, pclc, sizeof(*buf),
2228                               SMC_CLC_PROPOSAL, CLC_WAIT_TIME);
2229         if (rc)
2230                 goto out_decl;
2231
2232         if (pclc->hdr.version > SMC_V1)
2233                 proposal_version = SMC_V2;
2234
2235         /* IPSec connections opt out of SMC optimizations */
2236         if (using_ipsec(new_smc)) {
2237                 rc = SMC_CLC_DECL_IPSEC;
2238                 goto out_decl;
2239         }
2240
2241         ini = kzalloc(sizeof(*ini), GFP_KERNEL);
2242         if (!ini) {
2243                 rc = SMC_CLC_DECL_MEM;
2244                 goto out_decl;
2245         }
2246
2247         /* initial version checking */
2248         rc = smc_listen_v2_check(new_smc, pclc, ini);
2249         if (rc)
2250                 goto out_decl;
2251
2252         mutex_lock(&smc_server_lgr_pending);
2253         smc_close_init(new_smc);
2254         smc_rx_init(new_smc);
2255         smc_tx_init(new_smc);
2256
2257         /* determine ISM or RoCE device used for connection */
2258         rc = smc_listen_find_device(new_smc, pclc, ini);
2259         if (rc)
2260                 goto out_unlock;
2261
2262         /* send SMC Accept CLC message */
2263         accept_version = ini->is_smcd ? ini->smcd_version : ini->smcr_version;
2264         rc = smc_clc_send_accept(new_smc, ini->first_contact_local,
2265                                  accept_version, ini->negotiated_eid);
2266         if (rc)
2267                 goto out_unlock;
2268
2269         /* SMC-D does not need this lock any more */
2270         if (ini->is_smcd)
2271                 mutex_unlock(&smc_server_lgr_pending);
2272
2273         /* receive SMC Confirm CLC message */
2274         memset(buf, 0, sizeof(*buf));
2275         cclc = (struct smc_clc_msg_accept_confirm *)buf;
2276         rc = smc_clc_wait_msg(new_smc, cclc, sizeof(*buf),
2277                               SMC_CLC_CONFIRM, CLC_WAIT_TIME);
2278         if (rc) {
2279                 if (!ini->is_smcd)
2280                         goto out_unlock;
2281                 goto out_decl;
2282         }
2283
2284         /* finish worker */
2285         if (!ini->is_smcd) {
2286                 rc = smc_listen_rdma_finish(new_smc, cclc,
2287                                             ini->first_contact_local, ini);
2288                 if (rc)
2289                         goto out_unlock;
2290                 mutex_unlock(&smc_server_lgr_pending);
2291         }
2292         smc_conn_save_peer_info(new_smc, cclc);
2293         smc_listen_out_connected(new_smc);
2294         SMC_STAT_SERV_SUCC_INC(sock_net(newclcsock->sk), ini);
2295         goto out_free;
2296
2297 out_unlock:
2298         mutex_unlock(&smc_server_lgr_pending);
2299 out_decl:
2300         smc_listen_decline(new_smc, rc, ini ? ini->first_contact_local : 0,
2301                            proposal_version);
2302 out_free:
2303         kfree(ini);
2304         kfree(buf);
2305 }
2306
2307 static void smc_tcp_listen_work(struct work_struct *work)
2308 {
2309         struct smc_sock *lsmc = container_of(work, struct smc_sock,
2310                                              tcp_listen_work);
2311         struct sock *lsk = &lsmc->sk;
2312         struct smc_sock *new_smc;
2313         int rc = 0;
2314
2315         lock_sock(lsk);
2316         while (lsk->sk_state == SMC_LISTEN) {
2317                 rc = smc_clcsock_accept(lsmc, &new_smc);
2318                 if (rc) /* clcsock accept queue empty or error */
2319                         goto out;
2320                 if (!new_smc)
2321                         continue;
2322
2323                 if (tcp_sk(new_smc->clcsock->sk)->syn_smc)
2324                         atomic_inc(&lsmc->queued_smc_hs);
2325
2326                 new_smc->listen_smc = lsmc;
2327                 new_smc->use_fallback = lsmc->use_fallback;
2328                 new_smc->fallback_rsn = lsmc->fallback_rsn;
2329                 sock_hold(lsk); /* sock_put in smc_listen_work */
2330                 INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
2331                 smc_copy_sock_settings_to_smc(new_smc);
2332                 new_smc->sk.sk_sndbuf = lsmc->sk.sk_sndbuf;
2333                 new_smc->sk.sk_rcvbuf = lsmc->sk.sk_rcvbuf;
2334                 sock_hold(&new_smc->sk); /* sock_put in passive closing */
2335                 if (!queue_work(smc_hs_wq, &new_smc->smc_listen_work))
2336                         sock_put(&new_smc->sk);
2337         }
2338
2339 out:
2340         release_sock(lsk);
2341         sock_put(&lsmc->sk); /* sock_hold in smc_clcsock_data_ready() */
2342 }
2343
2344 static void smc_clcsock_data_ready(struct sock *listen_clcsock)
2345 {
2346         struct smc_sock *lsmc =
2347                 smc_clcsock_user_data(listen_clcsock);
2348
2349         if (!lsmc)
2350                 return;
2351         lsmc->clcsk_data_ready(listen_clcsock);
2352         if (lsmc->sk.sk_state == SMC_LISTEN) {
2353                 sock_hold(&lsmc->sk); /* sock_put in smc_tcp_listen_work() */
2354                 if (!queue_work(smc_tcp_ls_wq, &lsmc->tcp_listen_work))
2355                         sock_put(&lsmc->sk);
2356         }
2357 }
2358
2359 static int smc_listen(struct socket *sock, int backlog)
2360 {
2361         struct sock *sk = sock->sk;
2362         struct smc_sock *smc;
2363         int rc;
2364
2365         smc = smc_sk(sk);
2366         lock_sock(sk);
2367
2368         rc = -EINVAL;
2369         if ((sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) ||
2370             smc->connect_nonblock)
2371                 goto out;
2372
2373         rc = 0;
2374         if (sk->sk_state == SMC_LISTEN) {
2375                 sk->sk_max_ack_backlog = backlog;
2376                 goto out;
2377         }
2378         /* some socket options are handled in core, so we could not apply
2379          * them to the clc socket -- copy smc socket options to clc socket
2380          */
2381         smc_copy_sock_settings_to_clc(smc);
2382         if (!smc->use_fallback)
2383                 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
2384
2385         /* save original sk_data_ready function and establish
2386          * smc-specific sk_data_ready function
2387          */
2388         smc->clcsk_data_ready = smc->clcsock->sk->sk_data_ready;
2389         smc->clcsock->sk->sk_data_ready = smc_clcsock_data_ready;
2390         smc->clcsock->sk->sk_user_data =
2391                 (void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY);
2392
2393         /* save original ops */
2394         smc->ori_af_ops = inet_csk(smc->clcsock->sk)->icsk_af_ops;
2395
2396         smc->af_ops = *smc->ori_af_ops;
2397         smc->af_ops.syn_recv_sock = smc_tcp_syn_recv_sock;
2398
2399         inet_csk(smc->clcsock->sk)->icsk_af_ops = &smc->af_ops;
2400
2401         if (smc->limit_smc_hs)
2402                 tcp_sk(smc->clcsock->sk)->smc_hs_congested = smc_hs_congested;
2403
2404         rc = kernel_listen(smc->clcsock, backlog);
2405         if (rc) {
2406                 smc->clcsock->sk->sk_data_ready = smc->clcsk_data_ready;
2407                 goto out;
2408         }
2409         sk->sk_max_ack_backlog = backlog;
2410         sk->sk_ack_backlog = 0;
2411         sk->sk_state = SMC_LISTEN;
2412
2413 out:
2414         release_sock(sk);
2415         return rc;
2416 }
2417
2418 static int smc_accept(struct socket *sock, struct socket *new_sock,
2419                       int flags, bool kern)
2420 {
2421         struct sock *sk = sock->sk, *nsk;
2422         DECLARE_WAITQUEUE(wait, current);
2423         struct smc_sock *lsmc;
2424         long timeo;
2425         int rc = 0;
2426
2427         lsmc = smc_sk(sk);
2428         sock_hold(sk); /* sock_put below */
2429         lock_sock(sk);
2430
2431         if (lsmc->sk.sk_state != SMC_LISTEN) {
2432                 rc = -EINVAL;
2433                 release_sock(sk);
2434                 goto out;
2435         }
2436
2437         /* Wait for an incoming connection */
2438         timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
2439         add_wait_queue_exclusive(sk_sleep(sk), &wait);
2440         while (!(nsk = smc_accept_dequeue(sk, new_sock))) {
2441                 set_current_state(TASK_INTERRUPTIBLE);
2442                 if (!timeo) {
2443                         rc = -EAGAIN;
2444                         break;
2445                 }
2446                 release_sock(sk);
2447                 timeo = schedule_timeout(timeo);
2448                 /* wakeup by sk_data_ready in smc_listen_work() */
2449                 sched_annotate_sleep();
2450                 lock_sock(sk);
2451                 if (signal_pending(current)) {
2452                         rc = sock_intr_errno(timeo);
2453                         break;
2454                 }
2455         }
2456         set_current_state(TASK_RUNNING);
2457         remove_wait_queue(sk_sleep(sk), &wait);
2458
2459         if (!rc)
2460                 rc = sock_error(nsk);
2461         release_sock(sk);
2462         if (rc)
2463                 goto out;
2464
2465         if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) {
2466                 /* wait till data arrives on the socket */
2467                 timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept *
2468                                                                 MSEC_PER_SEC);
2469                 if (smc_sk(nsk)->use_fallback) {
2470                         struct sock *clcsk = smc_sk(nsk)->clcsock->sk;
2471
2472                         lock_sock(clcsk);
2473                         if (skb_queue_empty(&clcsk->sk_receive_queue))
2474                                 sk_wait_data(clcsk, &timeo, NULL);
2475                         release_sock(clcsk);
2476                 } else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) {
2477                         lock_sock(nsk);
2478                         smc_rx_wait(smc_sk(nsk), &timeo, smc_rx_data_available);
2479                         release_sock(nsk);
2480                 }
2481         }
2482
2483 out:
2484         sock_put(sk); /* sock_hold above */
2485         return rc;
2486 }
2487
2488 static int smc_getname(struct socket *sock, struct sockaddr *addr,
2489                        int peer)
2490 {
2491         struct smc_sock *smc;
2492
2493         if (peer && (sock->sk->sk_state != SMC_ACTIVE) &&
2494             (sock->sk->sk_state != SMC_APPCLOSEWAIT1))
2495                 return -ENOTCONN;
2496
2497         smc = smc_sk(sock->sk);
2498
2499         return smc->clcsock->ops->getname(smc->clcsock, addr, peer);
2500 }
2501
2502 static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
2503 {
2504         struct sock *sk = sock->sk;
2505         struct smc_sock *smc;
2506         int rc = -EPIPE;
2507
2508         smc = smc_sk(sk);
2509         lock_sock(sk);
2510         if ((sk->sk_state != SMC_ACTIVE) &&
2511             (sk->sk_state != SMC_APPCLOSEWAIT1) &&
2512             (sk->sk_state != SMC_INIT))
2513                 goto out;
2514
2515         if (msg->msg_flags & MSG_FASTOPEN) {
2516                 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
2517                         rc = smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP);
2518                         if (rc)
2519                                 goto out;
2520                 } else {
2521                         rc = -EINVAL;
2522                         goto out;
2523                 }
2524         }
2525
2526         if (smc->use_fallback) {
2527                 rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len);
2528         } else {
2529                 rc = smc_tx_sendmsg(smc, msg, len);
2530                 SMC_STAT_TX_PAYLOAD(smc, len, rc);
2531         }
2532 out:
2533         release_sock(sk);
2534         return rc;
2535 }
2536
2537 static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
2538                        int flags)
2539 {
2540         struct sock *sk = sock->sk;
2541         struct smc_sock *smc;
2542         int rc = -ENOTCONN;
2543
2544         smc = smc_sk(sk);
2545         lock_sock(sk);
2546         if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
2547                 /* socket was connected before, no more data to read */
2548                 rc = 0;
2549                 goto out;
2550         }
2551         if ((sk->sk_state == SMC_INIT) ||
2552             (sk->sk_state == SMC_LISTEN) ||
2553             (sk->sk_state == SMC_CLOSED))
2554                 goto out;
2555
2556         if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
2557                 rc = 0;
2558                 goto out;
2559         }
2560
2561         if (smc->use_fallback) {
2562                 rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags);
2563         } else {
2564                 msg->msg_namelen = 0;
2565                 rc = smc_rx_recvmsg(smc, msg, NULL, len, flags);
2566                 SMC_STAT_RX_PAYLOAD(smc, rc, rc);
2567         }
2568
2569 out:
2570         release_sock(sk);
2571         return rc;
2572 }
2573
2574 static __poll_t smc_accept_poll(struct sock *parent)
2575 {
2576         struct smc_sock *isk = smc_sk(parent);
2577         __poll_t mask = 0;
2578
2579         spin_lock(&isk->accept_q_lock);
2580         if (!list_empty(&isk->accept_q))
2581                 mask = EPOLLIN | EPOLLRDNORM;
2582         spin_unlock(&isk->accept_q_lock);
2583
2584         return mask;
2585 }
2586
2587 static __poll_t smc_poll(struct file *file, struct socket *sock,
2588                              poll_table *wait)
2589 {
2590         struct sock *sk = sock->sk;
2591         struct smc_sock *smc;
2592         __poll_t mask = 0;
2593
2594         if (!sk)
2595                 return EPOLLNVAL;
2596
2597         smc = smc_sk(sock->sk);
2598         if (smc->use_fallback) {
2599                 /* delegate to CLC child sock */
2600                 mask = smc->clcsock->ops->poll(file, smc->clcsock, wait);
2601                 sk->sk_err = smc->clcsock->sk->sk_err;
2602         } else {
2603                 if (sk->sk_state != SMC_CLOSED)
2604                         sock_poll_wait(file, sock, wait);
2605                 if (sk->sk_err)
2606                         mask |= EPOLLERR;
2607                 if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
2608                     (sk->sk_state == SMC_CLOSED))
2609                         mask |= EPOLLHUP;
2610                 if (sk->sk_state == SMC_LISTEN) {
2611                         /* woken up by sk_data_ready in smc_listen_work() */
2612                         mask |= smc_accept_poll(sk);
2613                 } else if (smc->use_fallback) { /* as result of connect_work()*/
2614                         mask |= smc->clcsock->ops->poll(file, smc->clcsock,
2615                                                            wait);
2616                         sk->sk_err = smc->clcsock->sk->sk_err;
2617                 } else {
2618                         if ((sk->sk_state != SMC_INIT &&
2619                              atomic_read(&smc->conn.sndbuf_space)) ||
2620                             sk->sk_shutdown & SEND_SHUTDOWN) {
2621                                 mask |= EPOLLOUT | EPOLLWRNORM;
2622                         } else {
2623                                 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
2624                                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
2625                         }
2626                         if (atomic_read(&smc->conn.bytes_to_rcv))
2627                                 mask |= EPOLLIN | EPOLLRDNORM;
2628                         if (sk->sk_shutdown & RCV_SHUTDOWN)
2629                                 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2630                         if (sk->sk_state == SMC_APPCLOSEWAIT1)
2631                                 mask |= EPOLLIN;
2632                         if (smc->conn.urg_state == SMC_URG_VALID)
2633                                 mask |= EPOLLPRI;
2634                 }
2635         }
2636
2637         return mask;
2638 }
2639
2640 static int smc_shutdown(struct socket *sock, int how)
2641 {
2642         struct sock *sk = sock->sk;
2643         bool do_shutdown = true;
2644         struct smc_sock *smc;
2645         int rc = -EINVAL;
2646         int old_state;
2647         int rc1 = 0;
2648
2649         smc = smc_sk(sk);
2650
2651         if ((how < SHUT_RD) || (how > SHUT_RDWR))
2652                 return rc;
2653
2654         lock_sock(sk);
2655
2656         rc = -ENOTCONN;
2657         if ((sk->sk_state != SMC_ACTIVE) &&
2658             (sk->sk_state != SMC_PEERCLOSEWAIT1) &&
2659             (sk->sk_state != SMC_PEERCLOSEWAIT2) &&
2660             (sk->sk_state != SMC_APPCLOSEWAIT1) &&
2661             (sk->sk_state != SMC_APPCLOSEWAIT2) &&
2662             (sk->sk_state != SMC_APPFINCLOSEWAIT))
2663                 goto out;
2664         if (smc->use_fallback) {
2665                 rc = kernel_sock_shutdown(smc->clcsock, how);
2666                 sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
2667                 if (sk->sk_shutdown == SHUTDOWN_MASK)
2668                         sk->sk_state = SMC_CLOSED;
2669                 goto out;
2670         }
2671         switch (how) {
2672         case SHUT_RDWR:         /* shutdown in both directions */
2673                 old_state = sk->sk_state;
2674                 rc = smc_close_active(smc);
2675                 if (old_state == SMC_ACTIVE &&
2676                     sk->sk_state == SMC_PEERCLOSEWAIT1)
2677                         do_shutdown = false;
2678                 break;
2679         case SHUT_WR:
2680                 rc = smc_close_shutdown_write(smc);
2681                 break;
2682         case SHUT_RD:
2683                 rc = 0;
2684                 /* nothing more to do because peer is not involved */
2685                 break;
2686         }
2687         if (do_shutdown && smc->clcsock)
2688                 rc1 = kernel_sock_shutdown(smc->clcsock, how);
2689         /* map sock_shutdown_cmd constants to sk_shutdown value range */
2690         sk->sk_shutdown |= how + 1;
2691
2692 out:
2693         release_sock(sk);
2694         return rc ? rc : rc1;
2695 }
2696
2697 static int __smc_getsockopt(struct socket *sock, int level, int optname,
2698                             char __user *optval, int __user *optlen)
2699 {
2700         struct smc_sock *smc;
2701         int val, len;
2702
2703         smc = smc_sk(sock->sk);
2704
2705         if (get_user(len, optlen))
2706                 return -EFAULT;
2707
2708         len = min_t(int, len, sizeof(int));
2709
2710         if (len < 0)
2711                 return -EINVAL;
2712
2713         switch (optname) {
2714         case SMC_LIMIT_HS:
2715                 val = smc->limit_smc_hs;
2716                 break;
2717         default:
2718                 return -EOPNOTSUPP;
2719         }
2720
2721         if (put_user(len, optlen))
2722                 return -EFAULT;
2723         if (copy_to_user(optval, &val, len))
2724                 return -EFAULT;
2725
2726         return 0;
2727 }
2728
2729 static int __smc_setsockopt(struct socket *sock, int level, int optname,
2730                             sockptr_t optval, unsigned int optlen)
2731 {
2732         struct sock *sk = sock->sk;
2733         struct smc_sock *smc;
2734         int val, rc;
2735
2736         smc = smc_sk(sk);
2737
2738         lock_sock(sk);
2739         switch (optname) {
2740         case SMC_LIMIT_HS:
2741                 if (optlen < sizeof(int)) {
2742                         rc = -EINVAL;
2743                         break;
2744                 }
2745                 if (copy_from_sockptr(&val, optval, sizeof(int))) {
2746                         rc = -EFAULT;
2747                         break;
2748                 }
2749
2750                 smc->limit_smc_hs = !!val;
2751                 rc = 0;
2752                 break;
2753         default:
2754                 rc = -EOPNOTSUPP;
2755                 break;
2756         }
2757         release_sock(sk);
2758
2759         return rc;
2760 }
2761
2762 static int smc_setsockopt(struct socket *sock, int level, int optname,
2763                           sockptr_t optval, unsigned int optlen)
2764 {
2765         struct sock *sk = sock->sk;
2766         struct smc_sock *smc;
2767         int val, rc;
2768
2769         if (level == SOL_TCP && optname == TCP_ULP)
2770                 return -EOPNOTSUPP;
2771         else if (level == SOL_SMC)
2772                 return __smc_setsockopt(sock, level, optname, optval, optlen);
2773
2774         smc = smc_sk(sk);
2775
2776         /* generic setsockopts reaching us here always apply to the
2777          * CLC socket
2778          */
2779         mutex_lock(&smc->clcsock_release_lock);
2780         if (!smc->clcsock) {
2781                 mutex_unlock(&smc->clcsock_release_lock);
2782                 return -EBADF;
2783         }
2784         if (unlikely(!smc->clcsock->ops->setsockopt))
2785                 rc = -EOPNOTSUPP;
2786         else
2787                 rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname,
2788                                                    optval, optlen);
2789         if (smc->clcsock->sk->sk_err) {
2790                 sk->sk_err = smc->clcsock->sk->sk_err;
2791                 sk_error_report(sk);
2792         }
2793         mutex_unlock(&smc->clcsock_release_lock);
2794
2795         if (optlen < sizeof(int))
2796                 return -EINVAL;
2797         if (copy_from_sockptr(&val, optval, sizeof(int)))
2798                 return -EFAULT;
2799
2800         lock_sock(sk);
2801         if (rc || smc->use_fallback)
2802                 goto out;
2803         switch (optname) {
2804         case TCP_FASTOPEN:
2805         case TCP_FASTOPEN_CONNECT:
2806         case TCP_FASTOPEN_KEY:
2807         case TCP_FASTOPEN_NO_COOKIE:
2808                 /* option not supported by SMC */
2809                 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
2810                         rc = smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP);
2811                 } else {
2812                         rc = -EINVAL;
2813                 }
2814                 break;
2815         case TCP_NODELAY:
2816                 if (sk->sk_state != SMC_INIT &&
2817                     sk->sk_state != SMC_LISTEN &&
2818                     sk->sk_state != SMC_CLOSED) {
2819                         if (val) {
2820                                 SMC_STAT_INC(smc, ndly_cnt);
2821                                 smc_tx_pending(&smc->conn);
2822                                 cancel_delayed_work(&smc->conn.tx_work);
2823                         }
2824                 }
2825                 break;
2826         case TCP_CORK:
2827                 if (sk->sk_state != SMC_INIT &&
2828                     sk->sk_state != SMC_LISTEN &&
2829                     sk->sk_state != SMC_CLOSED) {
2830                         if (!val) {
2831                                 SMC_STAT_INC(smc, cork_cnt);
2832                                 smc_tx_pending(&smc->conn);
2833                                 cancel_delayed_work(&smc->conn.tx_work);
2834                         }
2835                 }
2836                 break;
2837         case TCP_DEFER_ACCEPT:
2838                 smc->sockopt_defer_accept = val;
2839                 break;
2840         default:
2841                 break;
2842         }
2843 out:
2844         release_sock(sk);
2845
2846         return rc;
2847 }
2848
2849 static int smc_getsockopt(struct socket *sock, int level, int optname,
2850                           char __user *optval, int __user *optlen)
2851 {
2852         struct smc_sock *smc;
2853         int rc;
2854
2855         if (level == SOL_SMC)
2856                 return __smc_getsockopt(sock, level, optname, optval, optlen);
2857
2858         smc = smc_sk(sock->sk);
2859         mutex_lock(&smc->clcsock_release_lock);
2860         if (!smc->clcsock) {
2861                 mutex_unlock(&smc->clcsock_release_lock);
2862                 return -EBADF;
2863         }
2864         /* socket options apply to the CLC socket */
2865         if (unlikely(!smc->clcsock->ops->getsockopt)) {
2866                 mutex_unlock(&smc->clcsock_release_lock);
2867                 return -EOPNOTSUPP;
2868         }
2869         rc = smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
2870                                            optval, optlen);
2871         mutex_unlock(&smc->clcsock_release_lock);
2872         return rc;
2873 }
2874
2875 static int smc_ioctl(struct socket *sock, unsigned int cmd,
2876                      unsigned long arg)
2877 {
2878         union smc_host_cursor cons, urg;
2879         struct smc_connection *conn;
2880         struct smc_sock *smc;
2881         int answ;
2882
2883         smc = smc_sk(sock->sk);
2884         conn = &smc->conn;
2885         lock_sock(&smc->sk);
2886         if (smc->use_fallback) {
2887                 if (!smc->clcsock) {
2888                         release_sock(&smc->sk);
2889                         return -EBADF;
2890                 }
2891                 answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg);
2892                 release_sock(&smc->sk);
2893                 return answ;
2894         }
2895         switch (cmd) {
2896         case SIOCINQ: /* same as FIONREAD */
2897                 if (smc->sk.sk_state == SMC_LISTEN) {
2898                         release_sock(&smc->sk);
2899                         return -EINVAL;
2900                 }
2901                 if (smc->sk.sk_state == SMC_INIT ||
2902                     smc->sk.sk_state == SMC_CLOSED)
2903                         answ = 0;
2904                 else
2905                         answ = atomic_read(&smc->conn.bytes_to_rcv);
2906                 break;
2907         case SIOCOUTQ:
2908                 /* output queue size (not send + not acked) */
2909                 if (smc->sk.sk_state == SMC_LISTEN) {
2910                         release_sock(&smc->sk);
2911                         return -EINVAL;
2912                 }
2913                 if (smc->sk.sk_state == SMC_INIT ||
2914                     smc->sk.sk_state == SMC_CLOSED)
2915                         answ = 0;
2916                 else
2917                         answ = smc->conn.sndbuf_desc->len -
2918                                         atomic_read(&smc->conn.sndbuf_space);
2919                 break;
2920         case SIOCOUTQNSD:
2921                 /* output queue size (not send only) */
2922                 if (smc->sk.sk_state == SMC_LISTEN) {
2923                         release_sock(&smc->sk);
2924                         return -EINVAL;
2925                 }
2926                 if (smc->sk.sk_state == SMC_INIT ||
2927                     smc->sk.sk_state == SMC_CLOSED)
2928                         answ = 0;
2929                 else
2930                         answ = smc_tx_prepared_sends(&smc->conn);
2931                 break;
2932         case SIOCATMARK:
2933                 if (smc->sk.sk_state == SMC_LISTEN) {
2934                         release_sock(&smc->sk);
2935                         return -EINVAL;
2936                 }
2937                 if (smc->sk.sk_state == SMC_INIT ||
2938                     smc->sk.sk_state == SMC_CLOSED) {
2939                         answ = 0;
2940                 } else {
2941                         smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
2942                         smc_curs_copy(&urg, &conn->urg_curs, conn);
2943                         answ = smc_curs_diff(conn->rmb_desc->len,
2944                                              &cons, &urg) == 1;
2945                 }
2946                 break;
2947         default:
2948                 release_sock(&smc->sk);
2949                 return -ENOIOCTLCMD;
2950         }
2951         release_sock(&smc->sk);
2952
2953         return put_user(answ, (int __user *)arg);
2954 }
2955
2956 static ssize_t smc_sendpage(struct socket *sock, struct page *page,
2957                             int offset, size_t size, int flags)
2958 {
2959         struct sock *sk = sock->sk;
2960         struct smc_sock *smc;
2961         int rc = -EPIPE;
2962
2963         smc = smc_sk(sk);
2964         lock_sock(sk);
2965         if (sk->sk_state != SMC_ACTIVE) {
2966                 release_sock(sk);
2967                 goto out;
2968         }
2969         release_sock(sk);
2970         if (smc->use_fallback) {
2971                 rc = kernel_sendpage(smc->clcsock, page, offset,
2972                                      size, flags);
2973         } else {
2974                 lock_sock(sk);
2975                 rc = smc_tx_sendpage(smc, page, offset, size, flags);
2976                 release_sock(sk);
2977                 SMC_STAT_INC(smc, sendpage_cnt);
2978         }
2979
2980 out:
2981         return rc;
2982 }
2983
2984 /* Map the affected portions of the rmbe into an spd, note the number of bytes
2985  * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
2986  * updates till whenever a respective page has been fully processed.
2987  * Note that subsequent recv() calls have to wait till all splice() processing
2988  * completed.
2989  */
2990 static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos,
2991                                struct pipe_inode_info *pipe, size_t len,
2992                                unsigned int flags)
2993 {
2994         struct sock *sk = sock->sk;
2995         struct smc_sock *smc;
2996         int rc = -ENOTCONN;
2997
2998         smc = smc_sk(sk);
2999         lock_sock(sk);
3000         if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
3001                 /* socket was connected before, no more data to read */
3002                 rc = 0;
3003                 goto out;
3004         }
3005         if (sk->sk_state == SMC_INIT ||
3006             sk->sk_state == SMC_LISTEN ||
3007             sk->sk_state == SMC_CLOSED)
3008                 goto out;
3009
3010         if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
3011                 rc = 0;
3012                 goto out;
3013         }
3014
3015         if (smc->use_fallback) {
3016                 rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos,
3017                                                     pipe, len, flags);
3018         } else {
3019                 if (*ppos) {
3020                         rc = -ESPIPE;
3021                         goto out;
3022                 }
3023                 if (flags & SPLICE_F_NONBLOCK)
3024                         flags = MSG_DONTWAIT;
3025                 else
3026                         flags = 0;
3027                 SMC_STAT_INC(smc, splice_cnt);
3028                 rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags);
3029         }
3030 out:
3031         release_sock(sk);
3032
3033         return rc;
3034 }
3035
3036 /* must look like tcp */
3037 static const struct proto_ops smc_sock_ops = {
3038         .family         = PF_SMC,
3039         .owner          = THIS_MODULE,
3040         .release        = smc_release,
3041         .bind           = smc_bind,
3042         .connect        = smc_connect,
3043         .socketpair     = sock_no_socketpair,
3044         .accept         = smc_accept,
3045         .getname        = smc_getname,
3046         .poll           = smc_poll,
3047         .ioctl          = smc_ioctl,
3048         .listen         = smc_listen,
3049         .shutdown       = smc_shutdown,
3050         .setsockopt     = smc_setsockopt,
3051         .getsockopt     = smc_getsockopt,
3052         .sendmsg        = smc_sendmsg,
3053         .recvmsg        = smc_recvmsg,
3054         .mmap           = sock_no_mmap,
3055         .sendpage       = smc_sendpage,
3056         .splice_read    = smc_splice_read,
3057 };
3058
3059 static int __smc_create(struct net *net, struct socket *sock, int protocol,
3060                         int kern, struct socket *clcsock)
3061 {
3062         int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET;
3063         struct smc_sock *smc;
3064         struct sock *sk;
3065         int rc;
3066
3067         rc = -ESOCKTNOSUPPORT;
3068         if (sock->type != SOCK_STREAM)
3069                 goto out;
3070
3071         rc = -EPROTONOSUPPORT;
3072         if (protocol != SMCPROTO_SMC && protocol != SMCPROTO_SMC6)
3073                 goto out;
3074
3075         rc = -ENOBUFS;
3076         sock->ops = &smc_sock_ops;
3077         sk = smc_sock_alloc(net, sock, protocol);
3078         if (!sk)
3079                 goto out;
3080
3081         /* create internal TCP socket for CLC handshake and fallback */
3082         smc = smc_sk(sk);
3083         smc->use_fallback = false; /* assume rdma capability first */
3084         smc->fallback_rsn = 0;
3085
3086         /* default behavior from limit_smc_hs in every net namespace */
3087         smc->limit_smc_hs = net->smc.limit_smc_hs;
3088
3089         rc = 0;
3090         if (!clcsock) {
3091                 rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP,
3092                                       &smc->clcsock);
3093                 if (rc) {
3094                         sk_common_release(sk);
3095                         goto out;
3096                 }
3097         } else {
3098                 smc->clcsock = clcsock;
3099         }
3100
3101         smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE);
3102         smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE);
3103
3104 out:
3105         return rc;
3106 }
3107
3108 static int smc_create(struct net *net, struct socket *sock, int protocol,
3109                       int kern)
3110 {
3111         return __smc_create(net, sock, protocol, kern, NULL);
3112 }
3113
3114 static const struct net_proto_family smc_sock_family_ops = {
3115         .family = PF_SMC,
3116         .owner  = THIS_MODULE,
3117         .create = smc_create,
3118 };
3119
3120 static int smc_ulp_init(struct sock *sk)
3121 {
3122         struct socket *tcp = sk->sk_socket;
3123         struct net *net = sock_net(sk);
3124         struct socket *smcsock;
3125         int protocol, ret;
3126
3127         /* only TCP can be replaced */
3128         if (tcp->type != SOCK_STREAM || sk->sk_protocol != IPPROTO_TCP ||
3129             (sk->sk_family != AF_INET && sk->sk_family != AF_INET6))
3130                 return -ESOCKTNOSUPPORT;
3131         /* don't handle wq now */
3132         if (tcp->state != SS_UNCONNECTED || !tcp->file || tcp->wq.fasync_list)
3133                 return -ENOTCONN;
3134
3135         if (sk->sk_family == AF_INET)
3136                 protocol = SMCPROTO_SMC;
3137         else
3138                 protocol = SMCPROTO_SMC6;
3139
3140         smcsock = sock_alloc();
3141         if (!smcsock)
3142                 return -ENFILE;
3143
3144         smcsock->type = SOCK_STREAM;
3145         __module_get(THIS_MODULE); /* tried in __tcp_ulp_find_autoload */
3146         ret = __smc_create(net, smcsock, protocol, 1, tcp);
3147         if (ret) {
3148                 sock_release(smcsock); /* module_put() which ops won't be NULL */
3149                 return ret;
3150         }
3151
3152         /* replace tcp socket to smc */
3153         smcsock->file = tcp->file;
3154         smcsock->file->private_data = smcsock;
3155         smcsock->file->f_inode = SOCK_INODE(smcsock); /* replace inode when sock_close */
3156         smcsock->file->f_path.dentry->d_inode = SOCK_INODE(smcsock); /* dput() in __fput */
3157         tcp->file = NULL;
3158
3159         return ret;
3160 }
3161
3162 static void smc_ulp_clone(const struct request_sock *req, struct sock *newsk,
3163                           const gfp_t priority)
3164 {
3165         struct inet_connection_sock *icsk = inet_csk(newsk);
3166
3167         /* don't inherit ulp ops to child when listen */
3168         icsk->icsk_ulp_ops = NULL;
3169 }
3170
3171 static struct tcp_ulp_ops smc_ulp_ops __read_mostly = {
3172         .name           = "smc",
3173         .owner          = THIS_MODULE,
3174         .init           = smc_ulp_init,
3175         .clone          = smc_ulp_clone,
3176 };
3177
3178 unsigned int smc_net_id;
3179
3180 static __net_init int smc_net_init(struct net *net)
3181 {
3182         int rc;
3183
3184         rc = smc_sysctl_net_init(net);
3185         if (rc)
3186                 return rc;
3187         return smc_pnet_net_init(net);
3188 }
3189
3190 static void __net_exit smc_net_exit(struct net *net)
3191 {
3192         smc_sysctl_net_exit(net);
3193         smc_pnet_net_exit(net);
3194 }
3195
3196 static __net_init int smc_net_stat_init(struct net *net)
3197 {
3198         return smc_stats_init(net);
3199 }
3200
3201 static void __net_exit smc_net_stat_exit(struct net *net)
3202 {
3203         smc_stats_exit(net);
3204 }
3205
3206 static struct pernet_operations smc_net_ops = {
3207         .init = smc_net_init,
3208         .exit = smc_net_exit,
3209         .id   = &smc_net_id,
3210         .size = sizeof(struct smc_net),
3211 };
3212
3213 static struct pernet_operations smc_net_stat_ops = {
3214         .init = smc_net_stat_init,
3215         .exit = smc_net_stat_exit,
3216 };
3217
3218 static int __init smc_init(void)
3219 {
3220         int rc;
3221
3222         rc = register_pernet_subsys(&smc_net_ops);
3223         if (rc)
3224                 return rc;
3225
3226         rc = register_pernet_subsys(&smc_net_stat_ops);
3227         if (rc)
3228                 return rc;
3229
3230         smc_ism_init();
3231         smc_clc_init();
3232
3233         rc = smc_nl_init();
3234         if (rc)
3235                 goto out_pernet_subsys;
3236
3237         rc = smc_pnet_init();
3238         if (rc)
3239                 goto out_nl;
3240
3241         rc = -ENOMEM;
3242
3243         smc_tcp_ls_wq = alloc_workqueue("smc_tcp_ls_wq", 0, 0);
3244         if (!smc_tcp_ls_wq)
3245                 goto out_pnet;
3246
3247         smc_hs_wq = alloc_workqueue("smc_hs_wq", 0, 0);
3248         if (!smc_hs_wq)
3249                 goto out_alloc_tcp_ls_wq;
3250
3251         smc_close_wq = alloc_workqueue("smc_close_wq", 0, 0);
3252         if (!smc_close_wq)
3253                 goto out_alloc_hs_wq;
3254
3255         rc = smc_core_init();
3256         if (rc) {
3257                 pr_err("%s: smc_core_init fails with %d\n", __func__, rc);
3258                 goto out_alloc_wqs;
3259         }
3260
3261         rc = smc_llc_init();
3262         if (rc) {
3263                 pr_err("%s: smc_llc_init fails with %d\n", __func__, rc);
3264                 goto out_core;
3265         }
3266
3267         rc = smc_cdc_init();
3268         if (rc) {
3269                 pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc);
3270                 goto out_core;
3271         }
3272
3273         rc = proto_register(&smc_proto, 1);
3274         if (rc) {
3275                 pr_err("%s: proto_register(v4) fails with %d\n", __func__, rc);
3276                 goto out_core;
3277         }
3278
3279         rc = proto_register(&smc_proto6, 1);
3280         if (rc) {
3281                 pr_err("%s: proto_register(v6) fails with %d\n", __func__, rc);
3282                 goto out_proto;
3283         }
3284
3285         rc = sock_register(&smc_sock_family_ops);
3286         if (rc) {
3287                 pr_err("%s: sock_register fails with %d\n", __func__, rc);
3288                 goto out_proto6;
3289         }
3290         INIT_HLIST_HEAD(&smc_v4_hashinfo.ht);
3291         INIT_HLIST_HEAD(&smc_v6_hashinfo.ht);
3292
3293         rc = smc_ib_register_client();
3294         if (rc) {
3295                 pr_err("%s: ib_register fails with %d\n", __func__, rc);
3296                 goto out_sock;
3297         }
3298
3299         rc = tcp_register_ulp(&smc_ulp_ops);
3300         if (rc) {
3301                 pr_err("%s: tcp_ulp_register fails with %d\n", __func__, rc);
3302                 goto out_ib;
3303         }
3304
3305         static_branch_enable(&tcp_have_smc);
3306         return 0;
3307
3308 out_ib:
3309         smc_ib_unregister_client();
3310 out_sock:
3311         sock_unregister(PF_SMC);
3312 out_proto6:
3313         proto_unregister(&smc_proto6);
3314 out_proto:
3315         proto_unregister(&smc_proto);
3316 out_core:
3317         smc_core_exit();
3318 out_alloc_wqs:
3319         destroy_workqueue(smc_close_wq);
3320 out_alloc_hs_wq:
3321         destroy_workqueue(smc_hs_wq);
3322 out_alloc_tcp_ls_wq:
3323         destroy_workqueue(smc_tcp_ls_wq);
3324 out_pnet:
3325         smc_pnet_exit();
3326 out_nl:
3327         smc_nl_exit();
3328 out_pernet_subsys:
3329         unregister_pernet_subsys(&smc_net_ops);
3330
3331         return rc;
3332 }
3333
3334 static void __exit smc_exit(void)
3335 {
3336         static_branch_disable(&tcp_have_smc);
3337         tcp_unregister_ulp(&smc_ulp_ops);
3338         sock_unregister(PF_SMC);
3339         smc_core_exit();
3340         smc_ib_unregister_client();
3341         destroy_workqueue(smc_close_wq);
3342         destroy_workqueue(smc_tcp_ls_wq);
3343         destroy_workqueue(smc_hs_wq);
3344         proto_unregister(&smc_proto6);
3345         proto_unregister(&smc_proto);
3346         smc_pnet_exit();
3347         smc_nl_exit();
3348         smc_clc_exit();
3349         unregister_pernet_subsys(&smc_net_stat_ops);
3350         unregister_pernet_subsys(&smc_net_ops);
3351         rcu_barrier();
3352 }
3353
3354 module_init(smc_init);
3355 module_exit(smc_exit);
3356
3357 MODULE_AUTHOR("Ursula Braun <[email protected]>");
3358 MODULE_DESCRIPTION("smc socket address family");
3359 MODULE_LICENSE("GPL");
3360 MODULE_ALIAS_NETPROTO(PF_SMC);
3361 MODULE_ALIAS_TCP_ULP("smc");