net/smc/af_smc.c

   1 /*
   2  *  Shared Memory Communications over RDMA (SMC-R) and RoCE
   3  *
   4  *  AF_SMC protocol family socket handler keeping the AF_INET sock address type
   5  *  applies to SOCK_STREAM sockets only
   6  *  offers an alternative communication option for TCP-protocol sockets
   7  *  applicable with RoCE-cards only
   8  *
   9  *  Initial restrictions:
  10  *    - support for alternate links postponed
  11  *
  12  *  Copyright IBM Corp. 2016, 2018
  13  *
  14  *  Author(s):  Ursula Braun <[email protected]>
  15  *              based on prototype from Frank Blaschka
  16  */
  17
  18 #define KMSG_COMPONENT "smc"
  19 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  20
  21 #include <linux/module.h>
  22 #include <linux/socket.h>
  23 #include <linux/workqueue.h>
  24 #include <linux/in.h>
  25 #include <linux/sched/signal.h>
  26 #include <linux/if_vlan.h>
  27
  28 #include <net/sock.h>
  29 #include <net/tcp.h>
  30 #include <net/smc.h>
  31 #include <asm/ioctls.h>
  32
  33 #include <net/net_namespace.h>
  34 #include <net/netns/generic.h>
  35 #include "smc_netns.h"
  36
  37 #include "smc.h"
  38 #include "smc_clc.h"
  39 #include "smc_llc.h"
  40 #include "smc_cdc.h"
  41 #include "smc_core.h"
  42 #include "smc_ib.h"
  43 #include "smc_ism.h"
  44 #include "smc_pnet.h"
  45 #include "smc_tx.h"
  46 #include "smc_rx.h"
  47 #include "smc_close.h"
  48
  49 static DEFINE_MUTEX(smc_server_lgr_pending);    /* serialize link group
  50                                                  * creation on server
  51                                                  */
  52 static DEFINE_MUTEX(smc_client_lgr_pending);    /* serialize link group
  53                                                  * creation on client
  54                                                  */
  55
  56 static void smc_tcp_listen_work(struct work_struct *);
  57 static void smc_connect_work(struct work_struct *);
  58
  59 static void smc_set_keepalive(struct sock *sk, int val)
  60 {
  61         struct smc_sock *smc = smc_sk(sk);
  62
  63         smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val);
  64 }
  65
  66 static struct smc_hashinfo smc_v4_hashinfo = {
  67         .lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock),
  68 };
  69
  70 static struct smc_hashinfo smc_v6_hashinfo = {
  71         .lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock),
  72 };
  73
  74 int smc_hash_sk(struct sock *sk)
  75 {
  76         struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
  77         struct hlist_head *head;
  78
  79         head = &h->ht;
  80
  81         write_lock_bh(&h->lock);
  82         sk_add_node(sk, head);
  83         sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
  84         write_unlock_bh(&h->lock);
  85
  86         return 0;
  87 }
  88 EXPORT_SYMBOL_GPL(smc_hash_sk);
  89
  90 void smc_unhash_sk(struct sock *sk)
  91 {
  92         struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
  93
  94         write_lock_bh(&h->lock);
  95         if (sk_del_node_init(sk))
  96                 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
  97         write_unlock_bh(&h->lock);
  98 }
  99 EXPORT_SYMBOL_GPL(smc_unhash_sk);
 100
 101 struct proto smc_proto = {
 102         .name           = "SMC",
 103         .owner          = THIS_MODULE,
 104         .keepalive      = smc_set_keepalive,
 105         .hash           = smc_hash_sk,
 106         .unhash         = smc_unhash_sk,
 107         .obj_size       = sizeof(struct smc_sock),
 108         .h.smc_hash     = &smc_v4_hashinfo,
 109         .slab_flags     = SLAB_TYPESAFE_BY_RCU,
 110 };
 111 EXPORT_SYMBOL_GPL(smc_proto);
 112
 113 struct proto smc_proto6 = {
 114         .name           = "SMC6",
 115         .owner          = THIS_MODULE,
 116         .keepalive      = smc_set_keepalive,
 117         .hash           = smc_hash_sk,
 118         .unhash         = smc_unhash_sk,
 119         .obj_size       = sizeof(struct smc_sock),
 120         .h.smc_hash     = &smc_v6_hashinfo,
 121         .slab_flags     = SLAB_TYPESAFE_BY_RCU,
 122 };
 123 EXPORT_SYMBOL_GPL(smc_proto6);
 124
 125 static int smc_release(struct socket *sock)
 126 {
 127         struct sock *sk = sock->sk;
 128         struct smc_sock *smc;
 129         int rc = 0;
 130
 131         if (!sk)
 132                 goto out;
 133
 134         smc = smc_sk(sk);
 135
 136         /* cleanup for a dangling non-blocking connect */
 137         if (smc->connect_nonblock && sk->sk_state == SMC_INIT)
 138                 tcp_abort(smc->clcsock->sk, ECONNABORTED);
 139         flush_work(&smc->connect_work);
 140
 141         if (sk->sk_state == SMC_LISTEN)
 142                 /* smc_close_non_accepted() is called and acquires
 143                  * sock lock for child sockets again
 144                  */
 145                 lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
 146         else
 147                 lock_sock(sk);
 148
 149         if (!smc->use_fallback) {
 150                 rc = smc_close_active(smc);
 151                 sock_set_flag(sk, SOCK_DEAD);
 152                 sk->sk_shutdown |= SHUTDOWN_MASK;
 153         } else {
 154                 if (sk->sk_state != SMC_LISTEN && sk->sk_state != SMC_INIT)
 155                         sock_put(sk); /* passive closing */
 156                 if (sk->sk_state == SMC_LISTEN) {
 157                         /* wake up clcsock accept */
 158                         rc = kernel_sock_shutdown(smc->clcsock, SHUT_RDWR);
 159                 }
 160                 sk->sk_state = SMC_CLOSED;
 161                 sk->sk_state_change(sk);
 162         }
 163
 164         sk->sk_prot->unhash(sk);
 165
 166         if (sk->sk_state == SMC_CLOSED) {
 167                 if (smc->clcsock) {
 168                         release_sock(sk);
 169                         smc_clcsock_release(smc);
 170                         lock_sock(sk);
 171                 }
 172                 if (!smc->use_fallback)
 173                         smc_conn_free(&smc->conn);
 174         }
 175
 176         /* detach socket */
 177         sock_orphan(sk);
 178         sock->sk = NULL;
 179         release_sock(sk);
 180
 181         sock_put(sk); /* final sock_put */
 182 out:
 183         return rc;
 184 }
 185
 186 static void smc_destruct(struct sock *sk)
 187 {
 188         if (sk->sk_state != SMC_CLOSED)
 189                 return;
 190         if (!sock_flag(sk, SOCK_DEAD))
 191                 return;
 192
 193         sk_refcnt_debug_dec(sk);
 194 }
 195
 196 static struct sock *smc_sock_alloc(struct net *net, struct socket *sock,
 197                                    int protocol)
 198 {
 199         struct smc_sock *smc;
 200         struct proto *prot;
 201         struct sock *sk;
 202
 203         prot = (protocol == SMCPROTO_SMC6) ? &smc_proto6 : &smc_proto;
 204         sk = sk_alloc(net, PF_SMC, GFP_KERNEL, prot, 0);
 205         if (!sk)
 206                 return NULL;
 207
 208         sock_init_data(sock, sk); /* sets sk_refcnt to 1 */
 209         sk->sk_state = SMC_INIT;
 210         sk->sk_destruct = smc_destruct;
 211         sk->sk_protocol = protocol;
 212         smc = smc_sk(sk);
 213         INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
 214         INIT_WORK(&smc->connect_work, smc_connect_work);
 215         INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
 216         INIT_LIST_HEAD(&smc->accept_q);
 217         spin_lock_init(&smc->accept_q_lock);
 218         spin_lock_init(&smc->conn.send_lock);
 219         sk->sk_prot->hash(sk);
 220         sk_refcnt_debug_inc(sk);
 221         mutex_init(&smc->clcsock_release_lock);
 222
 223         return sk;
 224 }
 225
 226 static int smc_bind(struct socket *sock, struct sockaddr *uaddr,
 227                     int addr_len)
 228 {
 229         struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
 230         struct sock *sk = sock->sk;
 231         struct smc_sock *smc;
 232         int rc;
 233
 234         smc = smc_sk(sk);
 235
 236         /* replicate tests from inet_bind(), to be safe wrt. future changes */
 237         rc = -EINVAL;
 238         if (addr_len < sizeof(struct sockaddr_in))
 239                 goto out;
 240
 241         rc = -EAFNOSUPPORT;
 242         if (addr->sin_family != AF_INET &&
 243             addr->sin_family != AF_INET6 &&
 244             addr->sin_family != AF_UNSPEC)
 245                 goto out;
 246         /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
 247         if (addr->sin_family == AF_UNSPEC &&
 248             addr->sin_addr.s_addr != htonl(INADDR_ANY))
 249                 goto out;
 250
 251         lock_sock(sk);
 252
 253         /* Check if socket is already active */
 254         rc = -EINVAL;
 255         if (sk->sk_state != SMC_INIT)
 256                 goto out_rel;
 257
 258         smc->clcsock->sk->sk_reuse = sk->sk_reuse;
 259         rc = kernel_bind(smc->clcsock, uaddr, addr_len);
 260
 261 out_rel:
 262         release_sock(sk);
 263 out:
 264         return rc;
 265 }
 266
 267 static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk,
 268                                    unsigned long mask)
 269 {
 270         /* options we don't get control via setsockopt for */
 271         nsk->sk_type = osk->sk_type;
 272         nsk->sk_sndbuf = osk->sk_sndbuf;
 273         nsk->sk_rcvbuf = osk->sk_rcvbuf;
 274         nsk->sk_sndtimeo = osk->sk_sndtimeo;
 275         nsk->sk_rcvtimeo = osk->sk_rcvtimeo;
 276         nsk->sk_mark = osk->sk_mark;
 277         nsk->sk_priority = osk->sk_priority;
 278         nsk->sk_rcvlowat = osk->sk_rcvlowat;
 279         nsk->sk_bound_dev_if = osk->sk_bound_dev_if;
 280         nsk->sk_err = osk->sk_err;
 281
 282         nsk->sk_flags &= ~mask;
 283         nsk->sk_flags |= osk->sk_flags & mask;
 284 }
 285
 286 #define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
 287                              (1UL << SOCK_KEEPOPEN) | \
 288                              (1UL << SOCK_LINGER) | \
 289                              (1UL << SOCK_BROADCAST) | \
 290                              (1UL << SOCK_TIMESTAMP) | \
 291                              (1UL << SOCK_DBG) | \
 292                              (1UL << SOCK_RCVTSTAMP) | \
 293                              (1UL << SOCK_RCVTSTAMPNS) | \
 294                              (1UL << SOCK_LOCALROUTE) | \
 295                              (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
 296                              (1UL << SOCK_RXQ_OVFL) | \
 297                              (1UL << SOCK_WIFI_STATUS) | \
 298                              (1UL << SOCK_NOFCS) | \
 299                              (1UL << SOCK_FILTER_LOCKED) | \
 300                              (1UL << SOCK_TSTAMP_NEW))
 301 /* copy only relevant settings and flags of SOL_SOCKET level from smc to
 302  * clc socket (since smc is not called for these options from net/core)
 303  */
 304 static void smc_copy_sock_settings_to_clc(struct smc_sock *smc)
 305 {
 306         smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC);
 307 }
 308
 309 #define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
 310                              (1UL << SOCK_KEEPOPEN) | \
 311                              (1UL << SOCK_LINGER) | \
 312                              (1UL << SOCK_DBG))
 313 /* copy only settings and flags relevant for smc from clc to smc socket */
 314 static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
 315 {
 316         smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
 317 }
 318
 319 /* register a new rmb, send confirm_rkey msg to register with peer */
 320 static int smc_reg_rmb(struct smc_link *link, struct smc_buf_desc *rmb_desc,
 321                        bool conf_rkey)
 322 {
 323         if (!rmb_desc->wr_reg) {
 324                 /* register memory region for new rmb */
 325                 if (smc_wr_reg_send(link, rmb_desc->mr_rx[SMC_SINGLE_LINK])) {
 326                         rmb_desc->regerr = 1;
 327                         return -EFAULT;
 328                 }
 329                 rmb_desc->wr_reg = 1;
 330         }
 331         if (!conf_rkey)
 332                 return 0;
 333         /* exchange confirm_rkey msg with peer */
 334         if (smc_llc_do_confirm_rkey(link, rmb_desc)) {
 335                 rmb_desc->regerr = 1;
 336                 return -EFAULT;
 337         }
 338         return 0;
 339 }
 340
 341 static int smc_clnt_conf_first_link(struct smc_sock *smc)
 342 {
 343         struct net *net = sock_net(smc->clcsock->sk);
 344         struct smc_link_group *lgr = smc->conn.lgr;
 345         struct smc_link *link;
 346         int rest;
 347         int rc;
 348
 349         link = &lgr->lnk[SMC_SINGLE_LINK];
 350         /* receive CONFIRM LINK request from server over RoCE fabric */
 351         rest = wait_for_completion_interruptible_timeout(
 352                 &link->llc_confirm,
 353                 SMC_LLC_WAIT_FIRST_TIME);
 354         if (rest <= 0) {
 355                 struct smc_clc_msg_decline dclc;
 356
 357                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 358                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
 359                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
 360         }
 361
 362         if (link->llc_confirm_rc)
 363                 return SMC_CLC_DECL_RMBE_EC;
 364
 365         rc = smc_ib_modify_qp_rts(link);
 366         if (rc)
 367                 return SMC_CLC_DECL_ERR_RDYLNK;
 368
 369         smc_wr_remember_qp_attr(link);
 370
 371         if (smc_reg_rmb(link, smc->conn.rmb_desc, false))
 372                 return SMC_CLC_DECL_ERR_REGRMB;
 373
 374         /* send CONFIRM LINK response over RoCE fabric */
 375         rc = smc_llc_send_confirm_link(link, SMC_LLC_RESP);
 376         if (rc < 0)
 377                 return SMC_CLC_DECL_TIMEOUT_CL;
 378
 379         /* receive ADD LINK request from server over RoCE fabric */
 380         rest = wait_for_completion_interruptible_timeout(&link->llc_add,
 381                                                          SMC_LLC_WAIT_TIME);
 382         if (rest <= 0) {
 383                 struct smc_clc_msg_decline dclc;
 384
 385                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 386                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
 387                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_AL : rc;
 388         }
 389
 390         /* send add link reject message, only one link supported for now */
 391         rc = smc_llc_send_add_link(link,
 392                                    link->smcibdev->mac[link->ibport - 1],
 393                                    link->gid, SMC_LLC_RESP);
 394         if (rc < 0)
 395                 return SMC_CLC_DECL_TIMEOUT_AL;
 396
 397         smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);
 398
 399         return 0;
 400 }
 401
 402 static void smcr_conn_save_peer_info(struct smc_sock *smc,
 403                                      struct smc_clc_msg_accept_confirm *clc)
 404 {
 405         int bufsize = smc_uncompress_bufsize(clc->rmbe_size);
 406
 407         smc->conn.peer_rmbe_idx = clc->rmbe_idx;
 408         smc->conn.local_tx_ctrl.token = ntohl(clc->rmbe_alert_token);
 409         smc->conn.peer_rmbe_size = bufsize;
 410         atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 411         smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1);
 412 }
 413
 414 static void smcd_conn_save_peer_info(struct smc_sock *smc,
 415                                      struct smc_clc_msg_accept_confirm *clc)
 416 {
 417         int bufsize = smc_uncompress_bufsize(clc->dmbe_size);
 418
 419         smc->conn.peer_rmbe_idx = clc->dmbe_idx;
 420         smc->conn.peer_token = clc->token;
 421         /* msg header takes up space in the buffer */
 422         smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg);
 423         atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 424         smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx;
 425 }
 426
 427 static void smc_conn_save_peer_info(struct smc_sock *smc,
 428                                     struct smc_clc_msg_accept_confirm *clc)
 429 {
 430         if (smc->conn.lgr->is_smcd)
 431                 smcd_conn_save_peer_info(smc, clc);
 432         else
 433                 smcr_conn_save_peer_info(smc, clc);
 434 }
 435
 436 static void smc_link_save_peer_info(struct smc_link *link,
 437                                     struct smc_clc_msg_accept_confirm *clc)
 438 {
 439         link->peer_qpn = ntoh24(clc->qpn);
 440         memcpy(link->peer_gid, clc->lcl.gid, SMC_GID_SIZE);
 441         memcpy(link->peer_mac, clc->lcl.mac, sizeof(link->peer_mac));
 442         link->peer_psn = ntoh24(clc->psn);
 443         link->peer_mtu = clc->qp_mtu;
 444 }
 445
 446 static void smc_switch_to_fallback(struct smc_sock *smc)
 447 {
 448         smc->use_fallback = true;
 449         if (smc->sk.sk_socket && smc->sk.sk_socket->file) {
 450                 smc->clcsock->file = smc->sk.sk_socket->file;
 451                 smc->clcsock->file->private_data = smc->clcsock;
 452         }
 453 }
 454
 455 /* fall back during connect */
 456 static int smc_connect_fallback(struct smc_sock *smc, int reason_code)
 457 {
 458         smc_switch_to_fallback(smc);
 459         smc->fallback_rsn = reason_code;
 460         smc_copy_sock_settings_to_clc(smc);
 461         smc->connect_nonblock = 0;
 462         if (smc->sk.sk_state == SMC_INIT)
 463                 smc->sk.sk_state = SMC_ACTIVE;
 464         return 0;
 465 }
 466
 467 /* decline and fall back during connect */
 468 static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code)
 469 {
 470         int rc;
 471
 472         if (reason_code < 0) { /* error, fallback is not possible */
 473                 if (smc->sk.sk_state == SMC_INIT)
 474                         sock_put(&smc->sk); /* passive closing */
 475                 return reason_code;
 476         }
 477         if (reason_code != SMC_CLC_DECL_PEERDECL) {
 478                 rc = smc_clc_send_decline(smc, reason_code);
 479                 if (rc < 0) {
 480                         if (smc->sk.sk_state == SMC_INIT)
 481                                 sock_put(&smc->sk); /* passive closing */
 482                         return rc;
 483                 }
 484         }
 485         return smc_connect_fallback(smc, reason_code);
 486 }
 487
 488 /* abort connecting */
 489 static int smc_connect_abort(struct smc_sock *smc, int reason_code,
 490                              int local_contact)
 491 {
 492         if (local_contact == SMC_FIRST_CONTACT)
 493                 smc_lgr_forget(smc->conn.lgr);
 494         if (smc->conn.lgr->is_smcd)
 495                 /* there is only one lgr role for SMC-D; use server lock */
 496                 mutex_unlock(&smc_server_lgr_pending);
 497         else
 498                 mutex_unlock(&smc_client_lgr_pending);
 499
 500         smc_conn_free(&smc->conn);
 501         smc->connect_nonblock = 0;
 502         return reason_code;
 503 }
 504
 505 /* check if there is a rdma device available for this connection. */
 506 /* called for connect and listen */
 507 static int smc_find_rdma_device(struct smc_sock *smc, struct smc_init_info *ini)
 508 {
 509         /* PNET table look up: search active ib_device and port
 510          * within same PNETID that also contains the ethernet device
 511          * used for the internal TCP socket
 512          */
 513         smc_pnet_find_roce_resource(smc->clcsock->sk, ini);
 514         if (!ini->ib_dev)
 515                 return SMC_CLC_DECL_NOSMCRDEV;
 516         return 0;
 517 }
 518
 519 /* check if there is an ISM device available for this connection. */
 520 /* called for connect and listen */
 521 static int smc_find_ism_device(struct smc_sock *smc, struct smc_init_info *ini)
 522 {
 523         /* Find ISM device with same PNETID as connecting interface  */
 524         smc_pnet_find_ism_resource(smc->clcsock->sk, ini);
 525         if (!ini->ism_dev)
 526                 return SMC_CLC_DECL_NOSMCDDEV;
 527         return 0;
 528 }
 529
 530 /* Check for VLAN ID and register it on ISM device just for CLC handshake */
 531 static int smc_connect_ism_vlan_setup(struct smc_sock *smc,
 532                                       struct smc_init_info *ini)
 533 {
 534         if (ini->vlan_id && smc_ism_get_vlan(ini->ism_dev, ini->vlan_id))
 535                 return SMC_CLC_DECL_ISMVLANERR;
 536         return 0;
 537 }
 538
 539 /* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
 540  * used, the VLAN ID will be registered again during the connection setup.
 541  */
 542 static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc, bool is_smcd,
 543                                         struct smc_init_info *ini)
 544 {
 545         if (!is_smcd)
 546                 return 0;
 547         if (ini->vlan_id && smc_ism_put_vlan(ini->ism_dev, ini->vlan_id))
 548                 return SMC_CLC_DECL_CNFERR;
 549         return 0;
 550 }
 551
 552 /* CLC handshake during connect */
 553 static int smc_connect_clc(struct smc_sock *smc, int smc_type,
 554                            struct smc_clc_msg_accept_confirm *aclc,
 555                            struct smc_init_info *ini)
 556 {
 557         int rc = 0;
 558
 559         /* do inband token exchange */
 560         rc = smc_clc_send_proposal(smc, smc_type, ini);
 561         if (rc)
 562                 return rc;
 563         /* receive SMC Accept CLC message */
 564         return smc_clc_wait_msg(smc, aclc, sizeof(*aclc), SMC_CLC_ACCEPT,
 565                                 CLC_WAIT_TIME);
 566 }
 567
 568 /* setup for RDMA connection of client */
 569 static int smc_connect_rdma(struct smc_sock *smc,
 570                             struct smc_clc_msg_accept_confirm *aclc,
 571                             struct smc_init_info *ini)
 572 {
 573         struct smc_link *link;
 574         int reason_code = 0;
 575
 576         ini->is_smcd = false;
 577         ini->ib_lcl = &aclc->lcl;
 578         ini->ib_clcqpn = ntoh24(aclc->qpn);
 579         ini->srv_first_contact = aclc->hdr.flag;
 580
 581         mutex_lock(&smc_client_lgr_pending);
 582         reason_code = smc_conn_create(smc, ini);
 583         if (reason_code) {
 584                 mutex_unlock(&smc_client_lgr_pending);
 585                 return reason_code;
 586         }
 587         link = &smc->conn.lgr->lnk[SMC_SINGLE_LINK];
 588
 589         smc_conn_save_peer_info(smc, aclc);
 590
 591         /* create send buffer and rmb */
 592         if (smc_buf_create(smc, false))
 593                 return smc_connect_abort(smc, SMC_CLC_DECL_MEM,
 594                                          ini->cln_first_contact);
 595
 596         if (ini->cln_first_contact == SMC_FIRST_CONTACT)
 597                 smc_link_save_peer_info(link, aclc);
 598
 599         if (smc_rmb_rtoken_handling(&smc->conn, aclc))
 600                 return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RTOK,
 601                                          ini->cln_first_contact);
 602
 603         smc_close_init(smc);
 604         smc_rx_init(smc);
 605
 606         if (ini->cln_first_contact == SMC_FIRST_CONTACT) {
 607                 if (smc_ib_ready_link(link))
 608                         return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RDYLNK,
 609                                                  ini->cln_first_contact);
 610         } else {
 611                 if (smc_reg_rmb(link, smc->conn.rmb_desc, true))
 612                         return smc_connect_abort(smc, SMC_CLC_DECL_ERR_REGRMB,
 613                                                  ini->cln_first_contact);
 614         }
 615         smc_rmb_sync_sg_for_device(&smc->conn);
 616
 617         reason_code = smc_clc_send_confirm(smc);
 618         if (reason_code)
 619                 return smc_connect_abort(smc, reason_code,
 620                                          ini->cln_first_contact);
 621
 622         smc_tx_init(smc);
 623
 624         if (ini->cln_first_contact == SMC_FIRST_CONTACT) {
 625                 /* QP confirmation over RoCE fabric */
 626                 reason_code = smc_clnt_conf_first_link(smc);
 627                 if (reason_code)
 628                         return smc_connect_abort(smc, reason_code,
 629                                                  ini->cln_first_contact);
 630         }
 631         mutex_unlock(&smc_client_lgr_pending);
 632
 633         smc_copy_sock_settings_to_clc(smc);
 634         smc->connect_nonblock = 0;
 635         if (smc->sk.sk_state == SMC_INIT)
 636                 smc->sk.sk_state = SMC_ACTIVE;
 637
 638         return 0;
 639 }
 640
 641 /* setup for ISM connection of client */
 642 static int smc_connect_ism(struct smc_sock *smc,
 643                            struct smc_clc_msg_accept_confirm *aclc,
 644                            struct smc_init_info *ini)
 645 {
 646         int rc = 0;
 647
 648         ini->is_smcd = true;
 649         ini->ism_gid = aclc->gid;
 650         ini->srv_first_contact = aclc->hdr.flag;
 651
 652         /* there is only one lgr role for SMC-D; use server lock */
 653         mutex_lock(&smc_server_lgr_pending);
 654         rc = smc_conn_create(smc, ini);
 655         if (rc) {
 656                 mutex_unlock(&smc_server_lgr_pending);
 657                 return rc;
 658         }
 659
 660         /* Create send and receive buffers */
 661         if (smc_buf_create(smc, true))
 662                 return smc_connect_abort(smc, SMC_CLC_DECL_MEM,
 663                                          ini->cln_first_contact);
 664
 665         smc_conn_save_peer_info(smc, aclc);
 666         smc_close_init(smc);
 667         smc_rx_init(smc);
 668         smc_tx_init(smc);
 669
 670         rc = smc_clc_send_confirm(smc);
 671         if (rc)
 672                 return smc_connect_abort(smc, rc, ini->cln_first_contact);
 673         mutex_unlock(&smc_server_lgr_pending);
 674
 675         smc_copy_sock_settings_to_clc(smc);
 676         smc->connect_nonblock = 0;
 677         if (smc->sk.sk_state == SMC_INIT)
 678                 smc->sk.sk_state = SMC_ACTIVE;
 679
 680         return 0;
 681 }
 682
 683 /* perform steps before actually connecting */
 684 static int __smc_connect(struct smc_sock *smc)
 685 {
 686         bool ism_supported = false, rdma_supported = false;
 687         struct smc_clc_msg_accept_confirm aclc;
 688         struct smc_init_info ini = {0};
 689         int smc_type;
 690         int rc = 0;
 691
 692         sock_hold(&smc->sk); /* sock put in passive closing */
 693
 694         if (smc->use_fallback)
 695                 return smc_connect_fallback(smc, smc->fallback_rsn);
 696
 697         /* if peer has not signalled SMC-capability, fall back */
 698         if (!tcp_sk(smc->clcsock->sk)->syn_smc)
 699                 return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC);
 700
 701         /* IPSec connections opt out of SMC-R optimizations */
 702         if (using_ipsec(smc))
 703                 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC);
 704
 705         /* get vlan id from IP device */
 706         if (smc_vlan_by_tcpsk(smc->clcsock, &ini))
 707                 return smc_connect_decline_fallback(smc,
 708                                                     SMC_CLC_DECL_GETVLANERR);
 709
 710         /* check if there is an ism device available */
 711         if (!smc_find_ism_device(smc, &ini) &&
 712             !smc_connect_ism_vlan_setup(smc, &ini)) {
 713                 /* ISM is supported for this connection */
 714                 ism_supported = true;
 715                 smc_type = SMC_TYPE_D;
 716         }
 717
 718         /* check if there is a rdma device available */
 719         if (!smc_find_rdma_device(smc, &ini)) {
 720                 /* RDMA is supported for this connection */
 721                 rdma_supported = true;
 722                 if (ism_supported)
 723                         smc_type = SMC_TYPE_B; /* both */
 724                 else
 725                         smc_type = SMC_TYPE_R; /* only RDMA */
 726         }
 727
 728         /* if neither ISM nor RDMA are supported, fallback */
 729         if (!rdma_supported && !ism_supported)
 730                 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_NOSMCDEV);
 731
 732         /* perform CLC handshake */
 733         rc = smc_connect_clc(smc, smc_type, &aclc, &ini);
 734         if (rc) {
 735                 smc_connect_ism_vlan_cleanup(smc, ism_supported, &ini);
 736                 return smc_connect_decline_fallback(smc, rc);
 737         }
 738
 739         /* depending on previous steps, connect using rdma or ism */
 740         if (rdma_supported && aclc.hdr.path == SMC_TYPE_R)
 741                 rc = smc_connect_rdma(smc, &aclc, &ini);
 742         else if (ism_supported && aclc.hdr.path == SMC_TYPE_D)
 743                 rc = smc_connect_ism(smc, &aclc, &ini);
 744         else
 745                 rc = SMC_CLC_DECL_MODEUNSUPP;
 746         if (rc) {
 747                 smc_connect_ism_vlan_cleanup(smc, ism_supported, &ini);
 748                 return smc_connect_decline_fallback(smc, rc);
 749         }
 750
 751         smc_connect_ism_vlan_cleanup(smc, ism_supported, &ini);
 752         return 0;
 753 }
 754
 755 static void smc_connect_work(struct work_struct *work)
 756 {
 757         struct smc_sock *smc = container_of(work, struct smc_sock,
 758                                             connect_work);
 759         long timeo = smc->sk.sk_sndtimeo;
 760         int rc = 0;
 761
 762         if (!timeo)
 763                 timeo = MAX_SCHEDULE_TIMEOUT;
 764         lock_sock(smc->clcsock->sk);
 765         if (smc->clcsock->sk->sk_err) {
 766                 smc->sk.sk_err = smc->clcsock->sk->sk_err;
 767         } else if ((1 << smc->clcsock->sk->sk_state) &
 768                                         (TCPF_SYN_SENT | TCP_SYN_RECV)) {
 769                 rc = sk_stream_wait_connect(smc->clcsock->sk, &timeo);
 770                 if ((rc == -EPIPE) &&
 771                     ((1 << smc->clcsock->sk->sk_state) &
 772                                         (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)))
 773                         rc = 0;
 774         }
 775         release_sock(smc->clcsock->sk);
 776         lock_sock(&smc->sk);
 777         if (rc != 0 || smc->sk.sk_err) {
 778                 smc->sk.sk_state = SMC_CLOSED;
 779                 if (rc == -EPIPE || rc == -EAGAIN)
 780                         smc->sk.sk_err = EPIPE;
 781                 else if (signal_pending(current))
 782                         smc->sk.sk_err = -sock_intr_errno(timeo);
 783                 goto out;
 784         }
 785
 786         rc = __smc_connect(smc);
 787         if (rc < 0)
 788                 smc->sk.sk_err = -rc;
 789
 790 out:
 791         if (!sock_flag(&smc->sk, SOCK_DEAD)) {
 792                 if (smc->sk.sk_err) {
 793                         smc->sk.sk_state_change(&smc->sk);
 794                 } else { /* allow polling before and after fallback decision */
 795                         smc->clcsock->sk->sk_write_space(smc->clcsock->sk);
 796                         smc->sk.sk_write_space(&smc->sk);
 797                 }
 798         }
 799         release_sock(&smc->sk);
 800 }
 801
 802 static int smc_connect(struct socket *sock, struct sockaddr *addr,
 803                        int alen, int flags)
 804 {
 805         struct sock *sk = sock->sk;
 806         struct smc_sock *smc;
 807         int rc = -EINVAL;
 808
 809         smc = smc_sk(sk);
 810
 811         /* separate smc parameter checking to be safe */
 812         if (alen < sizeof(addr->sa_family))
 813                 goto out_err;
 814         if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
 815                 goto out_err;
 816
 817         lock_sock(sk);
 818         switch (sk->sk_state) {
 819         default:
 820                 goto out;
 821         case SMC_ACTIVE:
 822                 rc = -EISCONN;
 823                 goto out;
 824         case SMC_INIT:
 825                 rc = 0;
 826                 break;
 827         }
 828
 829         smc_copy_sock_settings_to_clc(smc);
 830         tcp_sk(smc->clcsock->sk)->syn_smc = 1;
 831         if (smc->connect_nonblock) {
 832                 rc = -EALREADY;
 833                 goto out;
 834         }
 835         rc = kernel_connect(smc->clcsock, addr, alen, flags);
 836         if (rc && rc != -EINPROGRESS)
 837                 goto out;
 838         if (flags & O_NONBLOCK) {
 839                 if (schedule_work(&smc->connect_work))
 840                         smc->connect_nonblock = 1;
 841                 rc = -EINPROGRESS;
 842         } else {
 843                 rc = __smc_connect(smc);
 844                 if (rc < 0)
 845                         goto out;
 846                 else
 847                         rc = 0; /* success cases including fallback */
 848         }
 849
 850 out:
 851         release_sock(sk);
 852 out_err:
 853         return rc;
 854 }
 855
 856 static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc)
 857 {
 858         struct socket *new_clcsock = NULL;
 859         struct sock *lsk = &lsmc->sk;
 860         struct sock *new_sk;
 861         int rc = -EINVAL;
 862
 863         release_sock(lsk);
 864         new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol);
 865         if (!new_sk) {
 866                 rc = -ENOMEM;
 867                 lsk->sk_err = ENOMEM;
 868                 *new_smc = NULL;
 869                 lock_sock(lsk);
 870                 goto out;
 871         }
 872         *new_smc = smc_sk(new_sk);
 873
 874         mutex_lock(&lsmc->clcsock_release_lock);
 875         if (lsmc->clcsock)
 876                 rc = kernel_accept(lsmc->clcsock, &new_clcsock, 0);
 877         mutex_unlock(&lsmc->clcsock_release_lock);
 878         lock_sock(lsk);
 879         if  (rc < 0)
 880                 lsk->sk_err = -rc;
 881         if (rc < 0 || lsk->sk_state == SMC_CLOSED) {
 882                 new_sk->sk_prot->unhash(new_sk);
 883                 if (new_clcsock)
 884                         sock_release(new_clcsock);
 885                 new_sk->sk_state = SMC_CLOSED;
 886                 sock_set_flag(new_sk, SOCK_DEAD);
 887                 sock_put(new_sk); /* final */
 888                 *new_smc = NULL;
 889                 goto out;
 890         }
 891
 892         (*new_smc)->clcsock = new_clcsock;
 893 out:
 894         return rc;
 895 }
 896
 897 /* add a just created sock to the accept queue of the listen sock as
 898  * candidate for a following socket accept call from user space
 899  */
 900 static void smc_accept_enqueue(struct sock *parent, struct sock *sk)
 901 {
 902         struct smc_sock *par = smc_sk(parent);
 903
 904         sock_hold(sk); /* sock_put in smc_accept_unlink () */
 905         spin_lock(&par->accept_q_lock);
 906         list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q);
 907         spin_unlock(&par->accept_q_lock);
 908         sk_acceptq_added(parent);
 909 }
 910
 911 /* remove a socket from the accept queue of its parental listening socket */
 912 static void smc_accept_unlink(struct sock *sk)
 913 {
 914         struct smc_sock *par = smc_sk(sk)->listen_smc;
 915
 916         spin_lock(&par->accept_q_lock);
 917         list_del_init(&smc_sk(sk)->accept_q);
 918         spin_unlock(&par->accept_q_lock);
 919         sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk);
 920         sock_put(sk); /* sock_hold in smc_accept_enqueue */
 921 }
 922
 923 /* remove a sock from the accept queue to bind it to a new socket created
 924  * for a socket accept call from user space
 925  */
 926 struct sock *smc_accept_dequeue(struct sock *parent,
 927                                 struct socket *new_sock)
 928 {
 929         struct smc_sock *isk, *n;
 930         struct sock *new_sk;
 931
 932         list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) {
 933                 new_sk = (struct sock *)isk;
 934
 935                 smc_accept_unlink(new_sk);
 936                 if (new_sk->sk_state == SMC_CLOSED) {
 937                         new_sk->sk_prot->unhash(new_sk);
 938                         if (isk->clcsock) {
 939                                 sock_release(isk->clcsock);
 940                                 isk->clcsock = NULL;
 941                         }
 942                         sock_put(new_sk); /* final */
 943                         continue;
 944                 }
 945                 if (new_sock) {
 946                         sock_graft(new_sk, new_sock);
 947                         if (isk->use_fallback) {
 948                                 smc_sk(new_sk)->clcsock->file = new_sock->file;
 949                                 isk->clcsock->file->private_data = isk->clcsock;
 950                         }
 951                 }
 952                 return new_sk;
 953         }
 954         return NULL;
 955 }
 956
 957 /* clean up for a created but never accepted sock */
 958 void smc_close_non_accepted(struct sock *sk)
 959 {
 960         struct smc_sock *smc = smc_sk(sk);
 961
 962         lock_sock(sk);
 963         if (!sk->sk_lingertime)
 964                 /* wait for peer closing */
 965                 sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT;
 966         if (!smc->use_fallback) {
 967                 smc_close_active(smc);
 968                 sock_set_flag(sk, SOCK_DEAD);
 969                 sk->sk_shutdown |= SHUTDOWN_MASK;
 970         }
 971         sk->sk_prot->unhash(sk);
 972         if (smc->clcsock) {
 973                 struct socket *tcp;
 974
 975                 tcp = smc->clcsock;
 976                 smc->clcsock = NULL;
 977                 sock_release(tcp);
 978         }
 979         if (smc->use_fallback) {
 980                 sock_put(sk); /* passive closing */
 981                 sk->sk_state = SMC_CLOSED;
 982         } else {
 983                 if (sk->sk_state == SMC_CLOSED)
 984                         smc_conn_free(&smc->conn);
 985         }
 986         release_sock(sk);
 987         sock_put(sk); /* final sock_put */
 988 }
 989
 990 static int smc_serv_conf_first_link(struct smc_sock *smc)
 991 {
 992         struct net *net = sock_net(smc->clcsock->sk);
 993         struct smc_link_group *lgr = smc->conn.lgr;
 994         struct smc_link *link;
 995         int rest;
 996         int rc;
 997
 998         link = &lgr->lnk[SMC_SINGLE_LINK];
 999
1000         if (smc_reg_rmb(link, smc->conn.rmb_desc, false))
1001                 return SMC_CLC_DECL_ERR_REGRMB;
1002
1003         /* send CONFIRM LINK request to client over the RoCE fabric */
1004         rc = smc_llc_send_confirm_link(link, SMC_LLC_REQ);
1005         if (rc < 0)
1006                 return SMC_CLC_DECL_TIMEOUT_CL;
1007
1008         /* receive CONFIRM LINK response from client over the RoCE fabric */
1009         rest = wait_for_completion_interruptible_timeout(
1010                 &link->llc_confirm_resp,
1011                 SMC_LLC_WAIT_FIRST_TIME);
1012         if (rest <= 0) {
1013                 struct smc_clc_msg_decline dclc;
1014
1015                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
1016                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
1017                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
1018         }
1019
1020         if (link->llc_confirm_resp_rc)
1021                 return SMC_CLC_DECL_RMBE_EC;
1022
1023         /* send ADD LINK request to client over the RoCE fabric */
1024         rc = smc_llc_send_add_link(link,
1025                                    link->smcibdev->mac[link->ibport - 1],
1026                                    link->gid, SMC_LLC_REQ);
1027         if (rc < 0)
1028                 return SMC_CLC_DECL_TIMEOUT_AL;
1029
1030         /* receive ADD LINK response from client over the RoCE fabric */
1031         rest = wait_for_completion_interruptible_timeout(&link->llc_add_resp,
1032                                                          SMC_LLC_WAIT_TIME);
1033         if (rest <= 0) {
1034                 struct smc_clc_msg_decline dclc;
1035
1036                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
1037                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
1038                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_AL : rc;
1039         }
1040
1041         smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);
1042
1043         return 0;
1044 }
1045
1046 /* listen worker: finish */
1047 static void smc_listen_out(struct smc_sock *new_smc)
1048 {
1049         struct smc_sock *lsmc = new_smc->listen_smc;
1050         struct sock *newsmcsk = &new_smc->sk;
1051
1052         if (lsmc->sk.sk_state == SMC_LISTEN) {
1053                 lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING);
1054                 smc_accept_enqueue(&lsmc->sk, newsmcsk);
1055                 release_sock(&lsmc->sk);
1056         } else { /* no longer listening */
1057                 smc_close_non_accepted(newsmcsk);
1058         }
1059
1060         /* Wake up accept */
1061         lsmc->sk.sk_data_ready(&lsmc->sk);
1062         sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */
1063 }
1064
1065 /* listen worker: finish in state connected */
1066 static void smc_listen_out_connected(struct smc_sock *new_smc)
1067 {
1068         struct sock *newsmcsk = &new_smc->sk;
1069
1070         sk_refcnt_debug_inc(newsmcsk);
1071         if (newsmcsk->sk_state == SMC_INIT)
1072                 newsmcsk->sk_state = SMC_ACTIVE;
1073
1074         smc_listen_out(new_smc);
1075 }
1076
1077 /* listen worker: finish in error state */
1078 static void smc_listen_out_err(struct smc_sock *new_smc)
1079 {
1080         struct sock *newsmcsk = &new_smc->sk;
1081
1082         if (newsmcsk->sk_state == SMC_INIT)
1083                 sock_put(&new_smc->sk); /* passive closing */
1084         newsmcsk->sk_state = SMC_CLOSED;
1085         smc_conn_free(&new_smc->conn);
1086
1087         smc_listen_out(new_smc);
1088 }
1089
1090 /* listen worker: decline and fall back if possible */
1091 static void smc_listen_decline(struct smc_sock *new_smc, int reason_code,
1092                                int local_contact)
1093 {
1094         /* RDMA setup failed, switch back to TCP */
1095         if (local_contact == SMC_FIRST_CONTACT)
1096                 smc_lgr_forget(new_smc->conn.lgr);
1097         if (reason_code < 0) { /* error, no fallback possible */
1098                 smc_listen_out_err(new_smc);
1099                 return;
1100         }
1101         smc_conn_free(&new_smc->conn);
1102         smc_switch_to_fallback(new_smc);
1103         new_smc->fallback_rsn = reason_code;
1104         if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) {
1105                 if (smc_clc_send_decline(new_smc, reason_code) < 0) {
1106                         smc_listen_out_err(new_smc);
1107                         return;
1108                 }
1109         }
1110         smc_listen_out_connected(new_smc);
1111 }
1112
1113 /* listen worker: check prefixes */
1114 static int smc_listen_prfx_check(struct smc_sock *new_smc,
1115                                  struct smc_clc_msg_proposal *pclc)
1116 {
1117         struct smc_clc_msg_proposal_prefix *pclc_prfx;
1118         struct socket *newclcsock = new_smc->clcsock;
1119
1120         pclc_prfx = smc_clc_proposal_get_prefix(pclc);
1121         if (smc_clc_prfx_match(newclcsock, pclc_prfx))
1122                 return SMC_CLC_DECL_DIFFPREFIX;
1123
1124         return 0;
1125 }
1126
1127 /* listen worker: initialize connection and buffers */
1128 static int smc_listen_rdma_init(struct smc_sock *new_smc,
1129                                 struct smc_init_info *ini)
1130 {
1131         int rc;
1132
1133         /* allocate connection / link group */
1134         rc = smc_conn_create(new_smc, ini);
1135         if (rc)
1136                 return rc;
1137
1138         /* create send buffer and rmb */
1139         if (smc_buf_create(new_smc, false))
1140                 return SMC_CLC_DECL_MEM;
1141
1142         return 0;
1143 }
1144
1145 /* listen worker: initialize connection and buffers for SMC-D */
1146 static int smc_listen_ism_init(struct smc_sock *new_smc,
1147                                struct smc_clc_msg_proposal *pclc,
1148                                struct smc_init_info *ini)
1149 {
1150         struct smc_clc_msg_smcd *pclc_smcd;
1151         int rc;
1152
1153         pclc_smcd = smc_get_clc_msg_smcd(pclc);
1154         ini->ism_gid = pclc_smcd->gid;
1155         rc = smc_conn_create(new_smc, ini);
1156         if (rc)
1157                 return rc;
1158
1159         /* Check if peer can be reached via ISM device */
1160         if (smc_ism_cantalk(new_smc->conn.lgr->peer_gid,
1161                             new_smc->conn.lgr->vlan_id,
1162                             new_smc->conn.lgr->smcd)) {
1163                 if (ini->cln_first_contact == SMC_FIRST_CONTACT)
1164                         smc_lgr_forget(new_smc->conn.lgr);
1165                 smc_conn_free(&new_smc->conn);
1166                 return SMC_CLC_DECL_SMCDNOTALK;
1167         }
1168
1169         /* Create send and receive buffers */
1170         if (smc_buf_create(new_smc, true)) {
1171                 if (ini->cln_first_contact == SMC_FIRST_CONTACT)
1172                         smc_lgr_forget(new_smc->conn.lgr);
1173                 smc_conn_free(&new_smc->conn);
1174                 return SMC_CLC_DECL_MEM;
1175         }
1176
1177         return 0;
1178 }
1179
1180 /* listen worker: register buffers */
1181 static int smc_listen_rdma_reg(struct smc_sock *new_smc, int local_contact)
1182 {
1183         struct smc_link *link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK];
1184
1185         if (local_contact != SMC_FIRST_CONTACT) {
1186                 if (smc_reg_rmb(link, new_smc->conn.rmb_desc, true))
1187                         return SMC_CLC_DECL_ERR_REGRMB;
1188         }
1189         smc_rmb_sync_sg_for_device(&new_smc->conn);
1190
1191         return 0;
1192 }
1193
1194 /* listen worker: finish RDMA setup */
1195 static int smc_listen_rdma_finish(struct smc_sock *new_smc,
1196                                   struct smc_clc_msg_accept_confirm *cclc,
1197                                   int local_contact)
1198 {
1199         struct smc_link *link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK];
1200         int reason_code = 0;
1201
1202         if (local_contact == SMC_FIRST_CONTACT)
1203                 smc_link_save_peer_info(link, cclc);
1204
1205         if (smc_rmb_rtoken_handling(&new_smc->conn, cclc)) {
1206                 reason_code = SMC_CLC_DECL_ERR_RTOK;
1207                 goto decline;
1208         }
1209
1210         if (local_contact == SMC_FIRST_CONTACT) {
1211                 if (smc_ib_ready_link(link)) {
1212                         reason_code = SMC_CLC_DECL_ERR_RDYLNK;
1213                         goto decline;
1214                 }
1215                 /* QP confirmation over RoCE fabric */
1216                 reason_code = smc_serv_conf_first_link(new_smc);
1217                 if (reason_code)
1218                         goto decline;
1219         }
1220         return 0;
1221
1222 decline:
1223         smc_listen_decline(new_smc, reason_code, local_contact);
1224         return reason_code;
1225 }
1226
1227 /* setup for RDMA connection of server */
1228 static void smc_listen_work(struct work_struct *work)
1229 {
1230         struct smc_sock *new_smc = container_of(work, struct smc_sock,
1231                                                 smc_listen_work);
1232         struct socket *newclcsock = new_smc->clcsock;
1233         struct smc_clc_msg_accept_confirm cclc;
1234         struct smc_clc_msg_proposal *pclc;
1235         struct smc_init_info ini = {0};
1236         bool ism_supported = false;
1237         u8 buf[SMC_CLC_MAX_LEN];
1238         int rc = 0;
1239
1240         if (new_smc->listen_smc->sk.sk_state != SMC_LISTEN)
1241                 return smc_listen_out_err(new_smc);
1242
1243         if (new_smc->use_fallback) {
1244                 smc_listen_out_connected(new_smc);
1245                 return;
1246         }
1247
1248         /* check if peer is smc capable */
1249         if (!tcp_sk(newclcsock->sk)->syn_smc) {
1250                 smc_switch_to_fallback(new_smc);
1251                 new_smc->fallback_rsn = SMC_CLC_DECL_PEERNOSMC;
1252                 smc_listen_out_connected(new_smc);
1253                 return;
1254         }
1255
1256         /* do inband token exchange -
1257          * wait for and receive SMC Proposal CLC message
1258          */
1259         pclc = (struct smc_clc_msg_proposal *)&buf;
1260         rc = smc_clc_wait_msg(new_smc, pclc, SMC_CLC_MAX_LEN,
1261                               SMC_CLC_PROPOSAL, CLC_WAIT_TIME);
1262         if (rc)
1263                 goto out_decl;
1264
1265         /* IPSec connections opt out of SMC-R optimizations */
1266         if (using_ipsec(new_smc)) {
1267                 rc = SMC_CLC_DECL_IPSEC;
1268                 goto out_decl;
1269         }
1270
1271         /* check for matching IP prefix and subnet length */
1272         rc = smc_listen_prfx_check(new_smc, pclc);
1273         if (rc)
1274                 goto out_decl;
1275
1276         /* get vlan id from IP device */
1277         if (smc_vlan_by_tcpsk(new_smc->clcsock, &ini)) {
1278                 rc = SMC_CLC_DECL_GETVLANERR;
1279                 goto out_decl;
1280         }
1281
1282         mutex_lock(&smc_server_lgr_pending);
1283         smc_close_init(new_smc);
1284         smc_rx_init(new_smc);
1285         smc_tx_init(new_smc);
1286
1287         /* check if ISM is available */
1288         if (pclc->hdr.path == SMC_TYPE_D || pclc->hdr.path == SMC_TYPE_B) {
1289                 ini.is_smcd = true; /* prepare ISM check */
1290                 rc = smc_find_ism_device(new_smc, &ini);
1291                 if (!rc)
1292                         rc = smc_listen_ism_init(new_smc, pclc, &ini);
1293                 if (!rc)
1294                         ism_supported = true;
1295                 else if (pclc->hdr.path == SMC_TYPE_D)
1296                         goto out_unlock; /* skip RDMA and decline */
1297         }
1298
1299         /* check if RDMA is available */
1300         if (!ism_supported) { /* SMC_TYPE_R or SMC_TYPE_B */
1301                 /* prepare RDMA check */
1302                 memset(&ini, 0, sizeof(ini));
1303                 ini.is_smcd = false;
1304                 ini.ib_lcl = &pclc->lcl;
1305                 rc = smc_find_rdma_device(new_smc, &ini);
1306                 if (rc) {
1307                         /* no RDMA device found */
1308                         if (pclc->hdr.path == SMC_TYPE_B)
1309                                 /* neither ISM nor RDMA device found */
1310                                 rc = SMC_CLC_DECL_NOSMCDEV;
1311                         goto out_unlock;
1312                 }
1313                 rc = smc_listen_rdma_init(new_smc, &ini);
1314                 if (rc)
1315                         goto out_unlock;
1316                 rc = smc_listen_rdma_reg(new_smc, ini.cln_first_contact);
1317                 if (rc)
1318                         goto out_unlock;
1319         }
1320
1321         /* send SMC Accept CLC message */
1322         rc = smc_clc_send_accept(new_smc, ini.cln_first_contact);
1323         if (rc)
1324                 goto out_unlock;
1325
1326         /* SMC-D does not need this lock any more */
1327         if (ism_supported)
1328                 mutex_unlock(&smc_server_lgr_pending);
1329
1330         /* receive SMC Confirm CLC message */
1331         rc = smc_clc_wait_msg(new_smc, &cclc, sizeof(cclc),
1332                               SMC_CLC_CONFIRM, CLC_WAIT_TIME);
1333         if (rc) {
1334                 if (!ism_supported)
1335                         goto out_unlock;
1336                 goto out_decl;
1337         }
1338
1339         /* finish worker */
1340         if (!ism_supported) {
1341                 rc = smc_listen_rdma_finish(new_smc, &cclc,
1342                                             ini.cln_first_contact);
1343                 mutex_unlock(&smc_server_lgr_pending);
1344                 if (rc)
1345                         return;
1346         }
1347         smc_conn_save_peer_info(new_smc, &cclc);
1348         smc_listen_out_connected(new_smc);
1349         return;
1350
1351 out_unlock:
1352         mutex_unlock(&smc_server_lgr_pending);
1353 out_decl:
1354         smc_listen_decline(new_smc, rc, ini.cln_first_contact);
1355 }
1356
1357 static void smc_tcp_listen_work(struct work_struct *work)
1358 {
1359         struct smc_sock *lsmc = container_of(work, struct smc_sock,
1360                                              tcp_listen_work);
1361         struct sock *lsk = &lsmc->sk;
1362         struct smc_sock *new_smc;
1363         int rc = 0;
1364
1365         lock_sock(lsk);
1366         while (lsk->sk_state == SMC_LISTEN) {
1367                 rc = smc_clcsock_accept(lsmc, &new_smc);
1368                 if (rc)
1369                         goto out;
1370                 if (!new_smc)
1371                         continue;
1372
1373                 new_smc->listen_smc = lsmc;
1374                 new_smc->use_fallback = lsmc->use_fallback;
1375                 new_smc->fallback_rsn = lsmc->fallback_rsn;
1376                 sock_hold(lsk); /* sock_put in smc_listen_work */
1377                 INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
1378                 smc_copy_sock_settings_to_smc(new_smc);
1379                 new_smc->sk.sk_sndbuf = lsmc->sk.sk_sndbuf;
1380                 new_smc->sk.sk_rcvbuf = lsmc->sk.sk_rcvbuf;
1381                 sock_hold(&new_smc->sk); /* sock_put in passive closing */
1382                 if (!schedule_work(&new_smc->smc_listen_work))
1383                         sock_put(&new_smc->sk);
1384         }
1385
1386 out:
1387         release_sock(lsk);
1388         sock_put(&lsmc->sk); /* sock_hold in smc_listen */
1389 }
1390
1391 static int smc_listen(struct socket *sock, int backlog)
1392 {
1393         struct sock *sk = sock->sk;
1394         struct smc_sock *smc;
1395         int rc;
1396
1397         smc = smc_sk(sk);
1398         lock_sock(sk);
1399
1400         rc = -EINVAL;
1401         if ((sk->sk_state != SMC_INIT) && (sk->sk_state != SMC_LISTEN))
1402                 goto out;
1403
1404         rc = 0;
1405         if (sk->sk_state == SMC_LISTEN) {
1406                 sk->sk_max_ack_backlog = backlog;
1407                 goto out;
1408         }
1409         /* some socket options are handled in core, so we could not apply
1410          * them to the clc socket -- copy smc socket options to clc socket
1411          */
1412         smc_copy_sock_settings_to_clc(smc);
1413         if (!smc->use_fallback)
1414                 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
1415
1416         rc = kernel_listen(smc->clcsock, backlog);
1417         if (rc)
1418                 goto out;
1419         sk->sk_max_ack_backlog = backlog;
1420         sk->sk_ack_backlog = 0;
1421         sk->sk_state = SMC_LISTEN;
1422         sock_hold(sk); /* sock_hold in tcp_listen_worker */
1423         if (!schedule_work(&smc->tcp_listen_work))
1424                 sock_put(sk);
1425
1426 out:
1427         release_sock(sk);
1428         return rc;
1429 }
1430
1431 static int smc_accept(struct socket *sock, struct socket *new_sock,
1432                       int flags, bool kern)
1433 {
1434         struct sock *sk = sock->sk, *nsk;
1435         DECLARE_WAITQUEUE(wait, current);
1436         struct smc_sock *lsmc;
1437         long timeo;
1438         int rc = 0;
1439
1440         lsmc = smc_sk(sk);
1441         sock_hold(sk); /* sock_put below */
1442         lock_sock(sk);
1443
1444         if (lsmc->sk.sk_state != SMC_LISTEN) {
1445                 rc = -EINVAL;
1446                 release_sock(sk);
1447                 goto out;
1448         }
1449
1450         /* Wait for an incoming connection */
1451         timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
1452         add_wait_queue_exclusive(sk_sleep(sk), &wait);
1453         while (!(nsk = smc_accept_dequeue(sk, new_sock))) {
1454                 set_current_state(TASK_INTERRUPTIBLE);
1455                 if (!timeo) {
1456                         rc = -EAGAIN;
1457                         break;
1458                 }
1459                 release_sock(sk);
1460                 timeo = schedule_timeout(timeo);
1461                 /* wakeup by sk_data_ready in smc_listen_work() */
1462                 sched_annotate_sleep();
1463                 lock_sock(sk);
1464                 if (signal_pending(current)) {
1465                         rc = sock_intr_errno(timeo);
1466                         break;
1467                 }
1468         }
1469         set_current_state(TASK_RUNNING);
1470         remove_wait_queue(sk_sleep(sk), &wait);
1471
1472         if (!rc)
1473                 rc = sock_error(nsk);
1474         release_sock(sk);
1475         if (rc)
1476                 goto out;
1477
1478         if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) {
1479                 /* wait till data arrives on the socket */
1480                 timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept *
1481                                                                 MSEC_PER_SEC);
1482                 if (smc_sk(nsk)->use_fallback) {
1483                         struct sock *clcsk = smc_sk(nsk)->clcsock->sk;
1484
1485                         lock_sock(clcsk);
1486                         if (skb_queue_empty(&clcsk->sk_receive_queue))
1487                                 sk_wait_data(clcsk, &timeo, NULL);
1488                         release_sock(clcsk);
1489                 } else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) {
1490                         lock_sock(nsk);
1491                         smc_rx_wait(smc_sk(nsk), &timeo, smc_rx_data_available);
1492                         release_sock(nsk);
1493                 }
1494         }
1495
1496 out:
1497         sock_put(sk); /* sock_hold above */
1498         return rc;
1499 }
1500
1501 static int smc_getname(struct socket *sock, struct sockaddr *addr,
1502                        int peer)
1503 {
1504         struct smc_sock *smc;
1505
1506         if (peer && (sock->sk->sk_state != SMC_ACTIVE) &&
1507             (sock->sk->sk_state != SMC_APPCLOSEWAIT1))
1508                 return -ENOTCONN;
1509
1510         smc = smc_sk(sock->sk);
1511
1512         return smc->clcsock->ops->getname(smc->clcsock, addr, peer);
1513 }
1514
1515 static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
1516 {
1517         struct sock *sk = sock->sk;
1518         struct smc_sock *smc;
1519         int rc = -EPIPE;
1520
1521         smc = smc_sk(sk);
1522         lock_sock(sk);
1523         if ((sk->sk_state != SMC_ACTIVE) &&
1524             (sk->sk_state != SMC_APPCLOSEWAIT1) &&
1525             (sk->sk_state != SMC_INIT))
1526                 goto out;
1527
1528         if (msg->msg_flags & MSG_FASTOPEN) {
1529                 if (sk->sk_state == SMC_INIT) {
1530                         smc_switch_to_fallback(smc);
1531                         smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
1532                 } else {
1533                         rc = -EINVAL;
1534                         goto out;
1535                 }
1536         }
1537
1538         if (smc->use_fallback)
1539                 rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len);
1540         else
1541                 rc = smc_tx_sendmsg(smc, msg, len);
1542 out:
1543         release_sock(sk);
1544         return rc;
1545 }
1546
1547 static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
1548                        int flags)
1549 {
1550         struct sock *sk = sock->sk;
1551         struct smc_sock *smc;
1552         int rc = -ENOTCONN;
1553
1554         smc = smc_sk(sk);
1555         lock_sock(sk);
1556         if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
1557                 /* socket was connected before, no more data to read */
1558                 rc = 0;
1559                 goto out;
1560         }
1561         if ((sk->sk_state == SMC_INIT) ||
1562             (sk->sk_state == SMC_LISTEN) ||
1563             (sk->sk_state == SMC_CLOSED))
1564                 goto out;
1565
1566         if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
1567                 rc = 0;
1568                 goto out;
1569         }
1570
1571         if (smc->use_fallback) {
1572                 rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags);
1573         } else {
1574                 msg->msg_namelen = 0;
1575                 rc = smc_rx_recvmsg(smc, msg, NULL, len, flags);
1576         }
1577
1578 out:
1579         release_sock(sk);
1580         return rc;
1581 }
1582
1583 static __poll_t smc_accept_poll(struct sock *parent)
1584 {
1585         struct smc_sock *isk = smc_sk(parent);
1586         __poll_t mask = 0;
1587
1588         spin_lock(&isk->accept_q_lock);
1589         if (!list_empty(&isk->accept_q))
1590                 mask = EPOLLIN | EPOLLRDNORM;
1591         spin_unlock(&isk->accept_q_lock);
1592
1593         return mask;
1594 }
1595
1596 static __poll_t smc_poll(struct file *file, struct socket *sock,
1597                              poll_table *wait)
1598 {
1599         struct sock *sk = sock->sk;
1600         struct smc_sock *smc;
1601         __poll_t mask = 0;
1602
1603         if (!sk)
1604                 return EPOLLNVAL;
1605
1606         smc = smc_sk(sock->sk);
1607         if (smc->use_fallback) {
1608                 /* delegate to CLC child sock */
1609                 mask = smc->clcsock->ops->poll(file, smc->clcsock, wait);
1610                 sk->sk_err = smc->clcsock->sk->sk_err;
1611         } else {
1612                 if (sk->sk_state != SMC_CLOSED)
1613                         sock_poll_wait(file, sock, wait);
1614                 if (sk->sk_err)
1615                         mask |= EPOLLERR;
1616                 if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
1617                     (sk->sk_state == SMC_CLOSED))
1618                         mask |= EPOLLHUP;
1619                 if (sk->sk_state == SMC_LISTEN) {
1620                         /* woken up by sk_data_ready in smc_listen_work() */
1621                         mask |= smc_accept_poll(sk);
1622                 } else if (smc->use_fallback) { /* as result of connect_work()*/
1623                         mask |= smc->clcsock->ops->poll(file, smc->clcsock,
1624                                                            wait);
1625                         sk->sk_err = smc->clcsock->sk->sk_err;
1626                 } else {
1627                         if ((sk->sk_state != SMC_INIT &&
1628                              atomic_read(&smc->conn.sndbuf_space)) ||
1629                             sk->sk_shutdown & SEND_SHUTDOWN) {
1630                                 mask |= EPOLLOUT | EPOLLWRNORM;
1631                         } else {
1632                                 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1633                                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1634                         }
1635                         if (atomic_read(&smc->conn.bytes_to_rcv))
1636                                 mask |= EPOLLIN | EPOLLRDNORM;
1637                         if (sk->sk_shutdown & RCV_SHUTDOWN)
1638                                 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
1639                         if (sk->sk_state == SMC_APPCLOSEWAIT1)
1640                                 mask |= EPOLLIN;
1641                         if (smc->conn.urg_state == SMC_URG_VALID)
1642                                 mask |= EPOLLPRI;
1643                 }
1644         }
1645
1646         return mask;
1647 }
1648
1649 static int smc_shutdown(struct socket *sock, int how)
1650 {
1651         struct sock *sk = sock->sk;
1652         struct smc_sock *smc;
1653         int rc = -EINVAL;
1654         int rc1 = 0;
1655
1656         smc = smc_sk(sk);
1657
1658         if ((how < SHUT_RD) || (how > SHUT_RDWR))
1659                 return rc;
1660
1661         lock_sock(sk);
1662
1663         rc = -ENOTCONN;
1664         if ((sk->sk_state != SMC_ACTIVE) &&
1665             (sk->sk_state != SMC_PEERCLOSEWAIT1) &&
1666             (sk->sk_state != SMC_PEERCLOSEWAIT2) &&
1667             (sk->sk_state != SMC_APPCLOSEWAIT1) &&
1668             (sk->sk_state != SMC_APPCLOSEWAIT2) &&
1669             (sk->sk_state != SMC_APPFINCLOSEWAIT))
1670                 goto out;
1671         if (smc->use_fallback) {
1672                 rc = kernel_sock_shutdown(smc->clcsock, how);
1673                 sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
1674                 if (sk->sk_shutdown == SHUTDOWN_MASK)
1675                         sk->sk_state = SMC_CLOSED;
1676                 goto out;
1677         }
1678         switch (how) {
1679         case SHUT_RDWR:         /* shutdown in both directions */
1680                 rc = smc_close_active(smc);
1681                 break;
1682         case SHUT_WR:
1683                 rc = smc_close_shutdown_write(smc);
1684                 break;
1685         case SHUT_RD:
1686                 rc = 0;
1687                 /* nothing more to do because peer is not involved */
1688                 break;
1689         }
1690         if (smc->clcsock)
1691                 rc1 = kernel_sock_shutdown(smc->clcsock, how);
1692         /* map sock_shutdown_cmd constants to sk_shutdown value range */
1693         sk->sk_shutdown |= how + 1;
1694
1695 out:
1696         release_sock(sk);
1697         return rc ? rc : rc1;
1698 }
1699
1700 static int smc_setsockopt(struct socket *sock, int level, int optname,
1701                           char __user *optval, unsigned int optlen)
1702 {
1703         struct sock *sk = sock->sk;
1704         struct smc_sock *smc;
1705         int val, rc;
1706
1707         smc = smc_sk(sk);
1708
1709         /* generic setsockopts reaching us here always apply to the
1710          * CLC socket
1711          */
1712         rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname,
1713                                            optval, optlen);
1714         if (smc->clcsock->sk->sk_err) {
1715                 sk->sk_err = smc->clcsock->sk->sk_err;
1716                 sk->sk_error_report(sk);
1717         }
1718         if (rc)
1719                 return rc;
1720
1721         if (optlen < sizeof(int))
1722                 return -EINVAL;
1723         if (get_user(val, (int __user *)optval))
1724                 return -EFAULT;
1725
1726         lock_sock(sk);
1727         switch (optname) {
1728         case TCP_ULP:
1729         case TCP_FASTOPEN:
1730         case TCP_FASTOPEN_CONNECT:
1731         case TCP_FASTOPEN_KEY:
1732         case TCP_FASTOPEN_NO_COOKIE:
1733                 /* option not supported by SMC */
1734                 if (sk->sk_state == SMC_INIT) {
1735                         smc_switch_to_fallback(smc);
1736                         smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
1737                 } else {
1738                         if (!smc->use_fallback)
1739                                 rc = -EINVAL;
1740                 }
1741                 break;
1742         case TCP_NODELAY:
1743                 if (sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) {
1744                         if (val && !smc->use_fallback)
1745                                 mod_delayed_work(system_wq, &smc->conn.tx_work,
1746                                                  0);
1747                 }
1748                 break;
1749         case TCP_CORK:
1750                 if (sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) {
1751                         if (!val && !smc->use_fallback)
1752                                 mod_delayed_work(system_wq, &smc->conn.tx_work,
1753                                                  0);
1754                 }
1755                 break;
1756         case TCP_DEFER_ACCEPT:
1757                 smc->sockopt_defer_accept = val;
1758                 break;
1759         default:
1760                 break;
1761         }
1762         release_sock(sk);
1763
1764         return rc;
1765 }
1766
1767 static int smc_getsockopt(struct socket *sock, int level, int optname,
1768                           char __user *optval, int __user *optlen)
1769 {
1770         struct smc_sock *smc;
1771
1772         smc = smc_sk(sock->sk);
1773         /* socket options apply to the CLC socket */
1774         return smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
1775                                              optval, optlen);
1776 }
1777
1778 static int smc_ioctl(struct socket *sock, unsigned int cmd,
1779                      unsigned long arg)
1780 {
1781         union smc_host_cursor cons, urg;
1782         struct smc_connection *conn;
1783         struct smc_sock *smc;
1784         int answ;
1785
1786         smc = smc_sk(sock->sk);
1787         conn = &smc->conn;
1788         lock_sock(&smc->sk);
1789         if (smc->use_fallback) {
1790                 if (!smc->clcsock) {
1791                         release_sock(&smc->sk);
1792                         return -EBADF;
1793                 }
1794                 answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg);
1795                 release_sock(&smc->sk);
1796                 return answ;
1797         }
1798         switch (cmd) {
1799         case SIOCINQ: /* same as FIONREAD */
1800                 if (smc->sk.sk_state == SMC_LISTEN) {
1801                         release_sock(&smc->sk);
1802                         return -EINVAL;
1803                 }
1804                 if (smc->sk.sk_state == SMC_INIT ||
1805                     smc->sk.sk_state == SMC_CLOSED)
1806                         answ = 0;
1807                 else
1808                         answ = atomic_read(&smc->conn.bytes_to_rcv);
1809                 break;
1810         case SIOCOUTQ:
1811                 /* output queue size (not send + not acked) */
1812                 if (smc->sk.sk_state == SMC_LISTEN) {
1813                         release_sock(&smc->sk);
1814                         return -EINVAL;
1815                 }
1816                 if (smc->sk.sk_state == SMC_INIT ||
1817                     smc->sk.sk_state == SMC_CLOSED)
1818                         answ = 0;
1819                 else
1820                         answ = smc->conn.sndbuf_desc->len -
1821                                         atomic_read(&smc->conn.sndbuf_space);
1822                 break;
1823         case SIOCOUTQNSD:
1824                 /* output queue size (not send only) */
1825                 if (smc->sk.sk_state == SMC_LISTEN) {
1826                         release_sock(&smc->sk);
1827                         return -EINVAL;
1828                 }
1829                 if (smc->sk.sk_state == SMC_INIT ||
1830                     smc->sk.sk_state == SMC_CLOSED)
1831                         answ = 0;
1832                 else
1833                         answ = smc_tx_prepared_sends(&smc->conn);
1834                 break;
1835         case SIOCATMARK:
1836                 if (smc->sk.sk_state == SMC_LISTEN) {
1837                         release_sock(&smc->sk);
1838                         return -EINVAL;
1839                 }
1840                 if (smc->sk.sk_state == SMC_INIT ||
1841                     smc->sk.sk_state == SMC_CLOSED) {
1842                         answ = 0;
1843                 } else {
1844                         smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
1845                         smc_curs_copy(&urg, &conn->urg_curs, conn);
1846                         answ = smc_curs_diff(conn->rmb_desc->len,
1847                                              &cons, &urg) == 1;
1848                 }
1849                 break;
1850         default:
1851                 release_sock(&smc->sk);
1852                 return -ENOIOCTLCMD;
1853         }
1854         release_sock(&smc->sk);
1855
1856         return put_user(answ, (int __user *)arg);
1857 }
1858
1859 static ssize_t smc_sendpage(struct socket *sock, struct page *page,
1860                             int offset, size_t size, int flags)
1861 {
1862         struct sock *sk = sock->sk;
1863         struct smc_sock *smc;
1864         int rc = -EPIPE;
1865
1866         smc = smc_sk(sk);
1867         lock_sock(sk);
1868         if (sk->sk_state != SMC_ACTIVE) {
1869                 release_sock(sk);
1870                 goto out;
1871         }
1872         release_sock(sk);
1873         if (smc->use_fallback)
1874                 rc = kernel_sendpage(smc->clcsock, page, offset,
1875                                      size, flags);
1876         else
1877                 rc = sock_no_sendpage(sock, page, offset, size, flags);
1878
1879 out:
1880         return rc;
1881 }
1882
1883 /* Map the affected portions of the rmbe into an spd, note the number of bytes
1884  * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
1885  * updates till whenever a respective page has been fully processed.
1886  * Note that subsequent recv() calls have to wait till all splice() processing
1887  * completed.
1888  */
1889 static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos,
1890                                struct pipe_inode_info *pipe, size_t len,
1891                                unsigned int flags)
1892 {
1893         struct sock *sk = sock->sk;
1894         struct smc_sock *smc;
1895         int rc = -ENOTCONN;
1896
1897         smc = smc_sk(sk);
1898         lock_sock(sk);
1899         if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
1900                 /* socket was connected before, no more data to read */
1901                 rc = 0;
1902                 goto out;
1903         }
1904         if (sk->sk_state == SMC_INIT ||
1905             sk->sk_state == SMC_LISTEN ||
1906             sk->sk_state == SMC_CLOSED)
1907                 goto out;
1908
1909         if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
1910                 rc = 0;
1911                 goto out;
1912         }
1913
1914         if (smc->use_fallback) {
1915                 rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos,
1916                                                     pipe, len, flags);
1917         } else {
1918                 if (*ppos) {
1919                         rc = -ESPIPE;
1920                         goto out;
1921                 }
1922                 if (flags & SPLICE_F_NONBLOCK)
1923                         flags = MSG_DONTWAIT;
1924                 else
1925                         flags = 0;
1926                 rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags);
1927         }
1928 out:
1929         release_sock(sk);
1930
1931         return rc;
1932 }
1933
1934 /* must look like tcp */
1935 static const struct proto_ops smc_sock_ops = {
1936         .family         = PF_SMC,
1937         .owner          = THIS_MODULE,
1938         .release        = smc_release,
1939         .bind           = smc_bind,
1940         .connect        = smc_connect,
1941         .socketpair     = sock_no_socketpair,
1942         .accept         = smc_accept,
1943         .getname        = smc_getname,
1944         .poll           = smc_poll,
1945         .ioctl          = smc_ioctl,
1946         .listen         = smc_listen,
1947         .shutdown       = smc_shutdown,
1948         .setsockopt     = smc_setsockopt,
1949         .getsockopt     = smc_getsockopt,
1950         .sendmsg        = smc_sendmsg,
1951         .recvmsg        = smc_recvmsg,
1952         .mmap           = sock_no_mmap,
1953         .sendpage       = smc_sendpage,
1954         .splice_read    = smc_splice_read,
1955 };
1956
1957 static int smc_create(struct net *net, struct socket *sock, int protocol,
1958                       int kern)
1959 {
1960         int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET;
1961         struct smc_sock *smc;
1962         struct sock *sk;
1963         int rc;
1964
1965         rc = -ESOCKTNOSUPPORT;
1966         if (sock->type != SOCK_STREAM)
1967                 goto out;
1968
1969         rc = -EPROTONOSUPPORT;
1970         if (protocol != SMCPROTO_SMC && protocol != SMCPROTO_SMC6)
1971                 goto out;
1972
1973         rc = -ENOBUFS;
1974         sock->ops = &smc_sock_ops;
1975         sk = smc_sock_alloc(net, sock, protocol);
1976         if (!sk)
1977                 goto out;
1978
1979         /* create internal TCP socket for CLC handshake and fallback */
1980         smc = smc_sk(sk);
1981         smc->use_fallback = false; /* assume rdma capability first */
1982         smc->fallback_rsn = 0;
1983         rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP,
1984                               &smc->clcsock);
1985         if (rc) {
1986                 sk_common_release(sk);
1987                 goto out;
1988         }
1989         smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE);
1990         smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE);
1991
1992 out:
1993         return rc;
1994 }
1995
1996 static const struct net_proto_family smc_sock_family_ops = {
1997         .family = PF_SMC,
1998         .owner  = THIS_MODULE,
1999         .create = smc_create,
2000 };
2001
2002 unsigned int smc_net_id;
2003
2004 static __net_init int smc_net_init(struct net *net)
2005 {
2006         return smc_pnet_net_init(net);
2007 }
2008
2009 static void __net_exit smc_net_exit(struct net *net)
2010 {
2011         smc_pnet_net_exit(net);
2012 }
2013
2014 static struct pernet_operations smc_net_ops = {
2015         .init = smc_net_init,
2016         .exit = smc_net_exit,
2017         .id   = &smc_net_id,
2018         .size = sizeof(struct smc_net),
2019 };
2020
2021 static int __init smc_init(void)
2022 {
2023         int rc;
2024
2025         rc = register_pernet_subsys(&smc_net_ops);
2026         if (rc)
2027                 return rc;
2028
2029         rc = smc_pnet_init();
2030         if (rc)
2031                 return rc;
2032
2033         rc = smc_llc_init();
2034         if (rc) {
2035                 pr_err("%s: smc_llc_init fails with %d\n", __func__, rc);
2036                 goto out_pnet;
2037         }
2038
2039         rc = smc_cdc_init();
2040         if (rc) {
2041                 pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc);
2042                 goto out_pnet;
2043         }
2044
2045         rc = proto_register(&smc_proto, 1);
2046         if (rc) {
2047                 pr_err("%s: proto_register(v4) fails with %d\n", __func__, rc);
2048                 goto out_pnet;
2049         }
2050
2051         rc = proto_register(&smc_proto6, 1);
2052         if (rc) {
2053                 pr_err("%s: proto_register(v6) fails with %d\n", __func__, rc);
2054                 goto out_proto;
2055         }
2056
2057         rc = sock_register(&smc_sock_family_ops);
2058         if (rc) {
2059                 pr_err("%s: sock_register fails with %d\n", __func__, rc);
2060                 goto out_proto6;
2061         }
2062         INIT_HLIST_HEAD(&smc_v4_hashinfo.ht);
2063         INIT_HLIST_HEAD(&smc_v6_hashinfo.ht);
2064
2065         rc = smc_ib_register_client();
2066         if (rc) {
2067                 pr_err("%s: ib_register fails with %d\n", __func__, rc);
2068                 goto out_sock;
2069         }
2070
2071         static_branch_enable(&tcp_have_smc);
2072         return 0;
2073
2074 out_sock:
2075         sock_unregister(PF_SMC);
2076 out_proto6:
2077         proto_unregister(&smc_proto6);
2078 out_proto:
2079         proto_unregister(&smc_proto);
2080 out_pnet:
2081         smc_pnet_exit();
2082         return rc;
2083 }
2084
2085 static void __exit smc_exit(void)
2086 {
2087         smc_core_exit();
2088         static_branch_disable(&tcp_have_smc);
2089         smc_ib_unregister_client();
2090         sock_unregister(PF_SMC);
2091         proto_unregister(&smc_proto6);
2092         proto_unregister(&smc_proto);
2093         smc_pnet_exit();
2094         unregister_pernet_subsys(&smc_net_ops);
2095 }
2096
2097 module_init(smc_init);
2098 module_exit(smc_exit);
2099
2100 MODULE_AUTHOR("Ursula Braun <[email protected]>");
2101 MODULE_DESCRIPTION("smc socket address family");
2102 MODULE_LICENSE("GPL");
2103 MODULE_ALIAS_NETPROTO(PF_SMC);