net/smc/af_smc.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  Shared Memory Communications over RDMA (SMC-R) and RoCE
   4  *
   5  *  AF_SMC protocol family socket handler keeping the AF_INET sock address type
   6  *  applies to SOCK_STREAM sockets only
   7  *  offers an alternative communication option for TCP-protocol sockets
   8  *  applicable with RoCE-cards only
   9  *
  10  *  Initial restrictions:
  11  *    - support for alternate links postponed
  12  *
  13  *  Copyright IBM Corp. 2016, 2018
  14  *
  15  *  Author(s):  Ursula Braun <[email protected]>
  16  *              based on prototype from Frank Blaschka
  17  */
  18
  19 #define KMSG_COMPONENT "smc"
  20 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  21
  22 #include <linux/module.h>
  23 #include <linux/socket.h>
  24 #include <linux/workqueue.h>
  25 #include <linux/in.h>
  26 #include <linux/sched/signal.h>
  27 #include <linux/if_vlan.h>
  28
  29 #include <net/sock.h>
  30 #include <net/tcp.h>
  31 #include <net/smc.h>
  32 #include <asm/ioctls.h>
  33
  34 #include <net/net_namespace.h>
  35 #include <net/netns/generic.h>
  36 #include "smc_netns.h"
  37
  38 #include "smc.h"
  39 #include "smc_clc.h"
  40 #include "smc_llc.h"
  41 #include "smc_cdc.h"
  42 #include "smc_core.h"
  43 #include "smc_ib.h"
  44 #include "smc_ism.h"
  45 #include "smc_pnet.h"
  46 #include "smc_tx.h"
  47 #include "smc_rx.h"
  48 #include "smc_close.h"
  49
  50 static DEFINE_MUTEX(smc_server_lgr_pending);    /* serialize link group
  51                                                  * creation on server
  52                                                  */
  53 static DEFINE_MUTEX(smc_client_lgr_pending);    /* serialize link group
  54                                                  * creation on client
  55                                                  */
  56
  57 static void smc_tcp_listen_work(struct work_struct *);
  58 static void smc_connect_work(struct work_struct *);
  59
  60 static void smc_set_keepalive(struct sock *sk, int val)
  61 {
  62         struct smc_sock *smc = smc_sk(sk);
  63
  64         smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val);
  65 }
  66
  67 static struct smc_hashinfo smc_v4_hashinfo = {
  68         .lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock),
  69 };
  70
  71 static struct smc_hashinfo smc_v6_hashinfo = {
  72         .lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock),
  73 };
  74
  75 int smc_hash_sk(struct sock *sk)
  76 {
  77         struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
  78         struct hlist_head *head;
  79
  80         head = &h->ht;
  81
  82         write_lock_bh(&h->lock);
  83         sk_add_node(sk, head);
  84         sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
  85         write_unlock_bh(&h->lock);
  86
  87         return 0;
  88 }
  89 EXPORT_SYMBOL_GPL(smc_hash_sk);
  90
  91 void smc_unhash_sk(struct sock *sk)
  92 {
  93         struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
  94
  95         write_lock_bh(&h->lock);
  96         if (sk_del_node_init(sk))
  97                 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
  98         write_unlock_bh(&h->lock);
  99 }
 100 EXPORT_SYMBOL_GPL(smc_unhash_sk);
 101
 102 struct proto smc_proto = {
 103         .name           = "SMC",
 104         .owner          = THIS_MODULE,
 105         .keepalive      = smc_set_keepalive,
 106         .hash           = smc_hash_sk,
 107         .unhash         = smc_unhash_sk,
 108         .obj_size       = sizeof(struct smc_sock),
 109         .h.smc_hash     = &smc_v4_hashinfo,
 110         .slab_flags     = SLAB_TYPESAFE_BY_RCU,
 111 };
 112 EXPORT_SYMBOL_GPL(smc_proto);
 113
 114 struct proto smc_proto6 = {
 115         .name           = "SMC6",
 116         .owner          = THIS_MODULE,
 117         .keepalive      = smc_set_keepalive,
 118         .hash           = smc_hash_sk,
 119         .unhash         = smc_unhash_sk,
 120         .obj_size       = sizeof(struct smc_sock),
 121         .h.smc_hash     = &smc_v6_hashinfo,
 122         .slab_flags     = SLAB_TYPESAFE_BY_RCU,
 123 };
 124 EXPORT_SYMBOL_GPL(smc_proto6);
 125
 126 static void smc_restore_fallback_changes(struct smc_sock *smc)
 127 {
 128         smc->clcsock->file->private_data = smc->sk.sk_socket;
 129         smc->clcsock->file = NULL;
 130 }
 131
 132 static int __smc_release(struct smc_sock *smc)
 133 {
 134         struct sock *sk = &smc->sk;
 135         int rc = 0;
 136
 137         if (!smc->use_fallback) {
 138                 rc = smc_close_active(smc);
 139                 sock_set_flag(sk, SOCK_DEAD);
 140                 sk->sk_shutdown |= SHUTDOWN_MASK;
 141         } else {
 142                 if (sk->sk_state != SMC_LISTEN && sk->sk_state != SMC_INIT)
 143                         sock_put(sk); /* passive closing */
 144                 if (sk->sk_state == SMC_LISTEN) {
 145                         /* wake up clcsock accept */
 146                         rc = kernel_sock_shutdown(smc->clcsock, SHUT_RDWR);
 147                 }
 148                 sk->sk_state = SMC_CLOSED;
 149                 sk->sk_state_change(sk);
 150                 smc_restore_fallback_changes(smc);
 151         }
 152
 153         sk->sk_prot->unhash(sk);
 154
 155         if (sk->sk_state == SMC_CLOSED) {
 156                 if (smc->clcsock) {
 157                         release_sock(sk);
 158                         smc_clcsock_release(smc);
 159                         lock_sock(sk);
 160                 }
 161                 if (!smc->use_fallback)
 162                         smc_conn_free(&smc->conn);
 163         }
 164
 165         return rc;
 166 }
 167
 168 static int smc_release(struct socket *sock)
 169 {
 170         struct sock *sk = sock->sk;
 171         struct smc_sock *smc;
 172         int rc = 0;
 173
 174         if (!sk)
 175                 goto out;
 176
 177         sock_hold(sk); /* sock_put below */
 178         smc = smc_sk(sk);
 179
 180         /* cleanup for a dangling non-blocking connect */
 181         if (smc->connect_nonblock && sk->sk_state == SMC_INIT)
 182                 tcp_abort(smc->clcsock->sk, ECONNABORTED);
 183         flush_work(&smc->connect_work);
 184
 185         if (sk->sk_state == SMC_LISTEN)
 186                 /* smc_close_non_accepted() is called and acquires
 187                  * sock lock for child sockets again
 188                  */
 189                 lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
 190         else
 191                 lock_sock(sk);
 192
 193         rc = __smc_release(smc);
 194
 195         /* detach socket */
 196         sock_orphan(sk);
 197         sock->sk = NULL;
 198         release_sock(sk);
 199
 200         sock_put(sk); /* sock_hold above */
 201         sock_put(sk); /* final sock_put */
 202 out:
 203         return rc;
 204 }
 205
 206 static void smc_destruct(struct sock *sk)
 207 {
 208         if (sk->sk_state != SMC_CLOSED)
 209                 return;
 210         if (!sock_flag(sk, SOCK_DEAD))
 211                 return;
 212
 213         sk_refcnt_debug_dec(sk);
 214 }
 215
 216 static struct sock *smc_sock_alloc(struct net *net, struct socket *sock,
 217                                    int protocol)
 218 {
 219         struct smc_sock *smc;
 220         struct proto *prot;
 221         struct sock *sk;
 222
 223         prot = (protocol == SMCPROTO_SMC6) ? &smc_proto6 : &smc_proto;
 224         sk = sk_alloc(net, PF_SMC, GFP_KERNEL, prot, 0);
 225         if (!sk)
 226                 return NULL;
 227
 228         sock_init_data(sock, sk); /* sets sk_refcnt to 1 */
 229         sk->sk_state = SMC_INIT;
 230         sk->sk_destruct = smc_destruct;
 231         sk->sk_protocol = protocol;
 232         smc = smc_sk(sk);
 233         INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
 234         INIT_WORK(&smc->connect_work, smc_connect_work);
 235         INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
 236         INIT_LIST_HEAD(&smc->accept_q);
 237         spin_lock_init(&smc->accept_q_lock);
 238         spin_lock_init(&smc->conn.send_lock);
 239         sk->sk_prot->hash(sk);
 240         sk_refcnt_debug_inc(sk);
 241         mutex_init(&smc->clcsock_release_lock);
 242
 243         return sk;
 244 }
 245
 246 static int smc_bind(struct socket *sock, struct sockaddr *uaddr,
 247                     int addr_len)
 248 {
 249         struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
 250         struct sock *sk = sock->sk;
 251         struct smc_sock *smc;
 252         int rc;
 253
 254         smc = smc_sk(sk);
 255
 256         /* replicate tests from inet_bind(), to be safe wrt. future changes */
 257         rc = -EINVAL;
 258         if (addr_len < sizeof(struct sockaddr_in))
 259                 goto out;
 260
 261         rc = -EAFNOSUPPORT;
 262         if (addr->sin_family != AF_INET &&
 263             addr->sin_family != AF_INET6 &&
 264             addr->sin_family != AF_UNSPEC)
 265                 goto out;
 266         /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
 267         if (addr->sin_family == AF_UNSPEC &&
 268             addr->sin_addr.s_addr != htonl(INADDR_ANY))
 269                 goto out;
 270
 271         lock_sock(sk);
 272
 273         /* Check if socket is already active */
 274         rc = -EINVAL;
 275         if (sk->sk_state != SMC_INIT || smc->connect_nonblock)
 276                 goto out_rel;
 277
 278         smc->clcsock->sk->sk_reuse = sk->sk_reuse;
 279         rc = kernel_bind(smc->clcsock, uaddr, addr_len);
 280
 281 out_rel:
 282         release_sock(sk);
 283 out:
 284         return rc;
 285 }
 286
 287 static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk,
 288                                    unsigned long mask)
 289 {
 290         /* options we don't get control via setsockopt for */
 291         nsk->sk_type = osk->sk_type;
 292         nsk->sk_sndbuf = osk->sk_sndbuf;
 293         nsk->sk_rcvbuf = osk->sk_rcvbuf;
 294         nsk->sk_sndtimeo = osk->sk_sndtimeo;
 295         nsk->sk_rcvtimeo = osk->sk_rcvtimeo;
 296         nsk->sk_mark = osk->sk_mark;
 297         nsk->sk_priority = osk->sk_priority;
 298         nsk->sk_rcvlowat = osk->sk_rcvlowat;
 299         nsk->sk_bound_dev_if = osk->sk_bound_dev_if;
 300         nsk->sk_err = osk->sk_err;
 301
 302         nsk->sk_flags &= ~mask;
 303         nsk->sk_flags |= osk->sk_flags & mask;
 304 }
 305
 306 #define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
 307                              (1UL << SOCK_KEEPOPEN) | \
 308                              (1UL << SOCK_LINGER) | \
 309                              (1UL << SOCK_BROADCAST) | \
 310                              (1UL << SOCK_TIMESTAMP) | \
 311                              (1UL << SOCK_DBG) | \
 312                              (1UL << SOCK_RCVTSTAMP) | \
 313                              (1UL << SOCK_RCVTSTAMPNS) | \
 314                              (1UL << SOCK_LOCALROUTE) | \
 315                              (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
 316                              (1UL << SOCK_RXQ_OVFL) | \
 317                              (1UL << SOCK_WIFI_STATUS) | \
 318                              (1UL << SOCK_NOFCS) | \
 319                              (1UL << SOCK_FILTER_LOCKED) | \
 320                              (1UL << SOCK_TSTAMP_NEW))
 321 /* copy only relevant settings and flags of SOL_SOCKET level from smc to
 322  * clc socket (since smc is not called for these options from net/core)
 323  */
 324 static void smc_copy_sock_settings_to_clc(struct smc_sock *smc)
 325 {
 326         smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC);
 327 }
 328
 329 #define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
 330                              (1UL << SOCK_KEEPOPEN) | \
 331                              (1UL << SOCK_LINGER) | \
 332                              (1UL << SOCK_DBG))
 333 /* copy only settings and flags relevant for smc from clc to smc socket */
 334 static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
 335 {
 336         smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
 337 }
 338
 339 /* register a new rmb, send confirm_rkey msg to register with peer */
 340 static int smc_reg_rmb(struct smc_link *link, struct smc_buf_desc *rmb_desc,
 341                        bool conf_rkey)
 342 {
 343         if (!rmb_desc->wr_reg) {
 344                 /* register memory region for new rmb */
 345                 if (smc_wr_reg_send(link, rmb_desc->mr_rx[SMC_SINGLE_LINK])) {
 346                         rmb_desc->regerr = 1;
 347                         return -EFAULT;
 348                 }
 349                 rmb_desc->wr_reg = 1;
 350         }
 351         if (!conf_rkey)
 352                 return 0;
 353         /* exchange confirm_rkey msg with peer */
 354         if (smc_llc_do_confirm_rkey(link, rmb_desc)) {
 355                 rmb_desc->regerr = 1;
 356                 return -EFAULT;
 357         }
 358         return 0;
 359 }
 360
 361 static int smc_clnt_conf_first_link(struct smc_sock *smc)
 362 {
 363         struct net *net = sock_net(smc->clcsock->sk);
 364         struct smc_link_group *lgr = smc->conn.lgr;
 365         struct smc_link *link;
 366         int rest;
 367         int rc;
 368
 369         link = &lgr->lnk[SMC_SINGLE_LINK];
 370         /* receive CONFIRM LINK request from server over RoCE fabric */
 371         rest = wait_for_completion_interruptible_timeout(
 372                 &link->llc_confirm,
 373                 SMC_LLC_WAIT_FIRST_TIME);
 374         if (rest <= 0) {
 375                 struct smc_clc_msg_decline dclc;
 376
 377                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 378                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
 379                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
 380         }
 381
 382         if (link->llc_confirm_rc)
 383                 return SMC_CLC_DECL_RMBE_EC;
 384
 385         rc = smc_ib_modify_qp_rts(link);
 386         if (rc)
 387                 return SMC_CLC_DECL_ERR_RDYLNK;
 388
 389         smc_wr_remember_qp_attr(link);
 390
 391         if (smc_reg_rmb(link, smc->conn.rmb_desc, false))
 392                 return SMC_CLC_DECL_ERR_REGRMB;
 393
 394         /* send CONFIRM LINK response over RoCE fabric */
 395         rc = smc_llc_send_confirm_link(link, SMC_LLC_RESP);
 396         if (rc < 0)
 397                 return SMC_CLC_DECL_TIMEOUT_CL;
 398
 399         /* receive ADD LINK request from server over RoCE fabric */
 400         rest = wait_for_completion_interruptible_timeout(&link->llc_add,
 401                                                          SMC_LLC_WAIT_TIME);
 402         if (rest <= 0) {
 403                 struct smc_clc_msg_decline dclc;
 404
 405                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 406                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
 407                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_AL : rc;
 408         }
 409
 410         /* send add link reject message, only one link supported for now */
 411         rc = smc_llc_send_add_link(link,
 412                                    link->smcibdev->mac[link->ibport - 1],
 413                                    link->gid, SMC_LLC_RESP);
 414         if (rc < 0)
 415                 return SMC_CLC_DECL_TIMEOUT_AL;
 416
 417         smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);
 418
 419         return 0;
 420 }
 421
 422 static void smcr_conn_save_peer_info(struct smc_sock *smc,
 423                                      struct smc_clc_msg_accept_confirm *clc)
 424 {
 425         int bufsize = smc_uncompress_bufsize(clc->rmbe_size);
 426
 427         smc->conn.peer_rmbe_idx = clc->rmbe_idx;
 428         smc->conn.local_tx_ctrl.token = ntohl(clc->rmbe_alert_token);
 429         smc->conn.peer_rmbe_size = bufsize;
 430         atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 431         smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1);
 432 }
 433
 434 static void smcd_conn_save_peer_info(struct smc_sock *smc,
 435                                      struct smc_clc_msg_accept_confirm *clc)
 436 {
 437         int bufsize = smc_uncompress_bufsize(clc->dmbe_size);
 438
 439         smc->conn.peer_rmbe_idx = clc->dmbe_idx;
 440         smc->conn.peer_token = clc->token;
 441         /* msg header takes up space in the buffer */
 442         smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg);
 443         atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 444         smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx;
 445 }
 446
 447 static void smc_conn_save_peer_info(struct smc_sock *smc,
 448                                     struct smc_clc_msg_accept_confirm *clc)
 449 {
 450         if (smc->conn.lgr->is_smcd)
 451                 smcd_conn_save_peer_info(smc, clc);
 452         else
 453                 smcr_conn_save_peer_info(smc, clc);
 454 }
 455
 456 static void smc_link_save_peer_info(struct smc_link *link,
 457                                     struct smc_clc_msg_accept_confirm *clc)
 458 {
 459         link->peer_qpn = ntoh24(clc->qpn);
 460         memcpy(link->peer_gid, clc->lcl.gid, SMC_GID_SIZE);
 461         memcpy(link->peer_mac, clc->lcl.mac, sizeof(link->peer_mac));
 462         link->peer_psn = ntoh24(clc->psn);
 463         link->peer_mtu = clc->qp_mtu;
 464 }
 465
 466 static void smc_switch_to_fallback(struct smc_sock *smc)
 467 {
 468         smc->use_fallback = true;
 469         if (smc->sk.sk_socket && smc->sk.sk_socket->file) {
 470                 smc->clcsock->file = smc->sk.sk_socket->file;
 471                 smc->clcsock->file->private_data = smc->clcsock;
 472         }
 473 }
 474
 475 /* fall back during connect */
 476 static int smc_connect_fallback(struct smc_sock *smc, int reason_code)
 477 {
 478         smc_switch_to_fallback(smc);
 479         smc->fallback_rsn = reason_code;
 480         smc_copy_sock_settings_to_clc(smc);
 481         smc->connect_nonblock = 0;
 482         if (smc->sk.sk_state == SMC_INIT)
 483                 smc->sk.sk_state = SMC_ACTIVE;
 484         return 0;
 485 }
 486
 487 /* decline and fall back during connect */
 488 static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code)
 489 {
 490         int rc;
 491
 492         if (reason_code < 0) { /* error, fallback is not possible */
 493                 if (smc->sk.sk_state == SMC_INIT)
 494                         sock_put(&smc->sk); /* passive closing */
 495                 return reason_code;
 496         }
 497         if (reason_code != SMC_CLC_DECL_PEERDECL) {
 498                 rc = smc_clc_send_decline(smc, reason_code);
 499                 if (rc < 0) {
 500                         if (smc->sk.sk_state == SMC_INIT)
 501                                 sock_put(&smc->sk); /* passive closing */
 502                         return rc;
 503                 }
 504         }
 505         return smc_connect_fallback(smc, reason_code);
 506 }
 507
 508 /* abort connecting */
 509 static int smc_connect_abort(struct smc_sock *smc, int reason_code,
 510                              int local_contact)
 511 {
 512         if (local_contact == SMC_FIRST_CONTACT)
 513                 smc_lgr_forget(smc->conn.lgr);
 514         if (smc->conn.lgr->is_smcd)
 515                 /* there is only one lgr role for SMC-D; use server lock */
 516                 mutex_unlock(&smc_server_lgr_pending);
 517         else
 518                 mutex_unlock(&smc_client_lgr_pending);
 519
 520         smc_conn_free(&smc->conn);
 521         smc->connect_nonblock = 0;
 522         return reason_code;
 523 }
 524
 525 /* check if there is a rdma device available for this connection. */
 526 /* called for connect and listen */
 527 static int smc_find_rdma_device(struct smc_sock *smc, struct smc_init_info *ini)
 528 {
 529         /* PNET table look up: search active ib_device and port
 530          * within same PNETID that also contains the ethernet device
 531          * used for the internal TCP socket
 532          */
 533         smc_pnet_find_roce_resource(smc->clcsock->sk, ini);
 534         if (!ini->ib_dev)
 535                 return SMC_CLC_DECL_NOSMCRDEV;
 536         return 0;
 537 }
 538
 539 /* check if there is an ISM device available for this connection. */
 540 /* called for connect and listen */
 541 static int smc_find_ism_device(struct smc_sock *smc, struct smc_init_info *ini)
 542 {
 543         /* Find ISM device with same PNETID as connecting interface  */
 544         smc_pnet_find_ism_resource(smc->clcsock->sk, ini);
 545         if (!ini->ism_dev)
 546                 return SMC_CLC_DECL_NOSMCDDEV;
 547         return 0;
 548 }
 549
 550 /* Check for VLAN ID and register it on ISM device just for CLC handshake */
 551 static int smc_connect_ism_vlan_setup(struct smc_sock *smc,
 552                                       struct smc_init_info *ini)
 553 {
 554         if (ini->vlan_id && smc_ism_get_vlan(ini->ism_dev, ini->vlan_id))
 555                 return SMC_CLC_DECL_ISMVLANERR;
 556         return 0;
 557 }
 558
 559 /* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
 560  * used, the VLAN ID will be registered again during the connection setup.
 561  */
 562 static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc, bool is_smcd,
 563                                         struct smc_init_info *ini)
 564 {
 565         if (!is_smcd)
 566                 return 0;
 567         if (ini->vlan_id && smc_ism_put_vlan(ini->ism_dev, ini->vlan_id))
 568                 return SMC_CLC_DECL_CNFERR;
 569         return 0;
 570 }
 571
 572 /* CLC handshake during connect */
 573 static int smc_connect_clc(struct smc_sock *smc, int smc_type,
 574                            struct smc_clc_msg_accept_confirm *aclc,
 575                            struct smc_init_info *ini)
 576 {
 577         int rc = 0;
 578
 579         /* do inband token exchange */
 580         rc = smc_clc_send_proposal(smc, smc_type, ini);
 581         if (rc)
 582                 return rc;
 583         /* receive SMC Accept CLC message */
 584         return smc_clc_wait_msg(smc, aclc, sizeof(*aclc), SMC_CLC_ACCEPT,
 585                                 CLC_WAIT_TIME);
 586 }
 587
 588 /* setup for RDMA connection of client */
 589 static int smc_connect_rdma(struct smc_sock *smc,
 590                             struct smc_clc_msg_accept_confirm *aclc,
 591                             struct smc_init_info *ini)
 592 {
 593         struct smc_link *link;
 594         int reason_code = 0;
 595
 596         ini->is_smcd = false;
 597         ini->ib_lcl = &aclc->lcl;
 598         ini->ib_clcqpn = ntoh24(aclc->qpn);
 599         ini->srv_first_contact = aclc->hdr.flag;
 600
 601         mutex_lock(&smc_client_lgr_pending);
 602         reason_code = smc_conn_create(smc, ini);
 603         if (reason_code) {
 604                 mutex_unlock(&smc_client_lgr_pending);
 605                 return reason_code;
 606         }
 607         link = &smc->conn.lgr->lnk[SMC_SINGLE_LINK];
 608
 609         smc_conn_save_peer_info(smc, aclc);
 610
 611         /* create send buffer and rmb */
 612         if (smc_buf_create(smc, false))
 613                 return smc_connect_abort(smc, SMC_CLC_DECL_MEM,
 614                                          ini->cln_first_contact);
 615
 616         if (ini->cln_first_contact == SMC_FIRST_CONTACT)
 617                 smc_link_save_peer_info(link, aclc);
 618
 619         if (smc_rmb_rtoken_handling(&smc->conn, aclc))
 620                 return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RTOK,
 621                                          ini->cln_first_contact);
 622
 623         smc_close_init(smc);
 624         smc_rx_init(smc);
 625
 626         if (ini->cln_first_contact == SMC_FIRST_CONTACT) {
 627                 if (smc_ib_ready_link(link))
 628                         return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RDYLNK,
 629                                                  ini->cln_first_contact);
 630         } else {
 631                 if (smc_reg_rmb(link, smc->conn.rmb_desc, true))
 632                         return smc_connect_abort(smc, SMC_CLC_DECL_ERR_REGRMB,
 633                                                  ini->cln_first_contact);
 634         }
 635         smc_rmb_sync_sg_for_device(&smc->conn);
 636
 637         reason_code = smc_clc_send_confirm(smc);
 638         if (reason_code)
 639                 return smc_connect_abort(smc, reason_code,
 640                                          ini->cln_first_contact);
 641
 642         smc_tx_init(smc);
 643
 644         if (ini->cln_first_contact == SMC_FIRST_CONTACT) {
 645                 /* QP confirmation over RoCE fabric */
 646                 reason_code = smc_clnt_conf_first_link(smc);
 647                 if (reason_code)
 648                         return smc_connect_abort(smc, reason_code,
 649                                                  ini->cln_first_contact);
 650         }
 651         mutex_unlock(&smc_client_lgr_pending);
 652
 653         smc_copy_sock_settings_to_clc(smc);
 654         smc->connect_nonblock = 0;
 655         if (smc->sk.sk_state == SMC_INIT)
 656                 smc->sk.sk_state = SMC_ACTIVE;
 657
 658         return 0;
 659 }
 660
 661 /* setup for ISM connection of client */
 662 static int smc_connect_ism(struct smc_sock *smc,
 663                            struct smc_clc_msg_accept_confirm *aclc,
 664                            struct smc_init_info *ini)
 665 {
 666         int rc = 0;
 667
 668         ini->is_smcd = true;
 669         ini->ism_gid = aclc->gid;
 670         ini->srv_first_contact = aclc->hdr.flag;
 671
 672         /* there is only one lgr role for SMC-D; use server lock */
 673         mutex_lock(&smc_server_lgr_pending);
 674         rc = smc_conn_create(smc, ini);
 675         if (rc) {
 676                 mutex_unlock(&smc_server_lgr_pending);
 677                 return rc;
 678         }
 679
 680         /* Create send and receive buffers */
 681         if (smc_buf_create(smc, true))
 682                 return smc_connect_abort(smc, SMC_CLC_DECL_MEM,
 683                                          ini->cln_first_contact);
 684
 685         smc_conn_save_peer_info(smc, aclc);
 686         smc_close_init(smc);
 687         smc_rx_init(smc);
 688         smc_tx_init(smc);
 689
 690         rc = smc_clc_send_confirm(smc);
 691         if (rc)
 692                 return smc_connect_abort(smc, rc, ini->cln_first_contact);
 693         mutex_unlock(&smc_server_lgr_pending);
 694
 695         smc_copy_sock_settings_to_clc(smc);
 696         smc->connect_nonblock = 0;
 697         if (smc->sk.sk_state == SMC_INIT)
 698                 smc->sk.sk_state = SMC_ACTIVE;
 699
 700         return 0;
 701 }
 702
 703 /* perform steps before actually connecting */
 704 static int __smc_connect(struct smc_sock *smc)
 705 {
 706         bool ism_supported = false, rdma_supported = false;
 707         struct smc_clc_msg_accept_confirm aclc;
 708         struct smc_init_info ini = {0};
 709         int smc_type;
 710         int rc = 0;
 711
 712         if (smc->use_fallback)
 713                 return smc_connect_fallback(smc, smc->fallback_rsn);
 714
 715         /* if peer has not signalled SMC-capability, fall back */
 716         if (!tcp_sk(smc->clcsock->sk)->syn_smc)
 717                 return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC);
 718
 719         /* IPSec connections opt out of SMC-R optimizations */
 720         if (using_ipsec(smc))
 721                 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC);
 722
 723         /* get vlan id from IP device */
 724         if (smc_vlan_by_tcpsk(smc->clcsock, &ini))
 725                 return smc_connect_decline_fallback(smc,
 726                                                     SMC_CLC_DECL_GETVLANERR);
 727
 728         /* check if there is an ism device available */
 729         if (!smc_find_ism_device(smc, &ini) &&
 730             !smc_connect_ism_vlan_setup(smc, &ini)) {
 731                 /* ISM is supported for this connection */
 732                 ism_supported = true;
 733                 smc_type = SMC_TYPE_D;
 734         }
 735
 736         /* check if there is a rdma device available */
 737         if (!smc_find_rdma_device(smc, &ini)) {
 738                 /* RDMA is supported for this connection */
 739                 rdma_supported = true;
 740                 if (ism_supported)
 741                         smc_type = SMC_TYPE_B; /* both */
 742                 else
 743                         smc_type = SMC_TYPE_R; /* only RDMA */
 744         }
 745
 746         /* if neither ISM nor RDMA are supported, fallback */
 747         if (!rdma_supported && !ism_supported)
 748                 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_NOSMCDEV);
 749
 750         /* perform CLC handshake */
 751         rc = smc_connect_clc(smc, smc_type, &aclc, &ini);
 752         if (rc) {
 753                 smc_connect_ism_vlan_cleanup(smc, ism_supported, &ini);
 754                 return smc_connect_decline_fallback(smc, rc);
 755         }
 756
 757         /* depending on previous steps, connect using rdma or ism */
 758         if (rdma_supported && aclc.hdr.path == SMC_TYPE_R)
 759                 rc = smc_connect_rdma(smc, &aclc, &ini);
 760         else if (ism_supported && aclc.hdr.path == SMC_TYPE_D)
 761                 rc = smc_connect_ism(smc, &aclc, &ini);
 762         else
 763                 rc = SMC_CLC_DECL_MODEUNSUPP;
 764         if (rc) {
 765                 smc_connect_ism_vlan_cleanup(smc, ism_supported, &ini);
 766                 return smc_connect_decline_fallback(smc, rc);
 767         }
 768
 769         smc_connect_ism_vlan_cleanup(smc, ism_supported, &ini);
 770         return 0;
 771 }
 772
 773 static void smc_connect_work(struct work_struct *work)
 774 {
 775         struct smc_sock *smc = container_of(work, struct smc_sock,
 776                                             connect_work);
 777         long timeo = smc->sk.sk_sndtimeo;
 778         int rc = 0;
 779
 780         if (!timeo)
 781                 timeo = MAX_SCHEDULE_TIMEOUT;
 782         lock_sock(smc->clcsock->sk);
 783         if (smc->clcsock->sk->sk_err) {
 784                 smc->sk.sk_err = smc->clcsock->sk->sk_err;
 785         } else if ((1 << smc->clcsock->sk->sk_state) &
 786                                         (TCPF_SYN_SENT | TCP_SYN_RECV)) {
 787                 rc = sk_stream_wait_connect(smc->clcsock->sk, &timeo);
 788                 if ((rc == -EPIPE) &&
 789                     ((1 << smc->clcsock->sk->sk_state) &
 790                                         (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)))
 791                         rc = 0;
 792         }
 793         release_sock(smc->clcsock->sk);
 794         lock_sock(&smc->sk);
 795         if (rc != 0 || smc->sk.sk_err) {
 796                 smc->sk.sk_state = SMC_CLOSED;
 797                 if (rc == -EPIPE || rc == -EAGAIN)
 798                         smc->sk.sk_err = EPIPE;
 799                 else if (signal_pending(current))
 800                         smc->sk.sk_err = -sock_intr_errno(timeo);
 801                 goto out;
 802         }
 803
 804         rc = __smc_connect(smc);
 805         if (rc < 0)
 806                 smc->sk.sk_err = -rc;
 807
 808 out:
 809         if (!sock_flag(&smc->sk, SOCK_DEAD)) {
 810                 if (smc->sk.sk_err) {
 811                         smc->sk.sk_state_change(&smc->sk);
 812                 } else { /* allow polling before and after fallback decision */
 813                         smc->clcsock->sk->sk_write_space(smc->clcsock->sk);
 814                         smc->sk.sk_write_space(&smc->sk);
 815                 }
 816         }
 817         release_sock(&smc->sk);
 818 }
 819
 820 static int smc_connect(struct socket *sock, struct sockaddr *addr,
 821                        int alen, int flags)
 822 {
 823         struct sock *sk = sock->sk;
 824         struct smc_sock *smc;
 825         int rc = -EINVAL;
 826
 827         smc = smc_sk(sk);
 828
 829         /* separate smc parameter checking to be safe */
 830         if (alen < sizeof(addr->sa_family))
 831                 goto out_err;
 832         if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
 833                 goto out_err;
 834
 835         lock_sock(sk);
 836         switch (sk->sk_state) {
 837         default:
 838                 goto out;
 839         case SMC_ACTIVE:
 840                 rc = -EISCONN;
 841                 goto out;
 842         case SMC_INIT:
 843                 rc = 0;
 844                 break;
 845         }
 846
 847         smc_copy_sock_settings_to_clc(smc);
 848         tcp_sk(smc->clcsock->sk)->syn_smc = 1;
 849         if (smc->connect_nonblock) {
 850                 rc = -EALREADY;
 851                 goto out;
 852         }
 853         rc = kernel_connect(smc->clcsock, addr, alen, flags);
 854         if (rc && rc != -EINPROGRESS)
 855                 goto out;
 856
 857         sock_hold(&smc->sk); /* sock put in passive closing */
 858         if (flags & O_NONBLOCK) {
 859                 if (schedule_work(&smc->connect_work))
 860                         smc->connect_nonblock = 1;
 861                 rc = -EINPROGRESS;
 862         } else {
 863                 rc = __smc_connect(smc);
 864                 if (rc < 0)
 865                         goto out;
 866                 else
 867                         rc = 0; /* success cases including fallback */
 868         }
 869
 870 out:
 871         release_sock(sk);
 872 out_err:
 873         return rc;
 874 }
 875
 876 static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc)
 877 {
 878         struct socket *new_clcsock = NULL;
 879         struct sock *lsk = &lsmc->sk;
 880         struct sock *new_sk;
 881         int rc = -EINVAL;
 882
 883         release_sock(lsk);
 884         new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol);
 885         if (!new_sk) {
 886                 rc = -ENOMEM;
 887                 lsk->sk_err = ENOMEM;
 888                 *new_smc = NULL;
 889                 lock_sock(lsk);
 890                 goto out;
 891         }
 892         *new_smc = smc_sk(new_sk);
 893
 894         mutex_lock(&lsmc->clcsock_release_lock);
 895         if (lsmc->clcsock)
 896                 rc = kernel_accept(lsmc->clcsock, &new_clcsock, 0);
 897         mutex_unlock(&lsmc->clcsock_release_lock);
 898         lock_sock(lsk);
 899         if  (rc < 0)
 900                 lsk->sk_err = -rc;
 901         if (rc < 0 || lsk->sk_state == SMC_CLOSED) {
 902                 new_sk->sk_prot->unhash(new_sk);
 903                 if (new_clcsock)
 904                         sock_release(new_clcsock);
 905                 new_sk->sk_state = SMC_CLOSED;
 906                 sock_set_flag(new_sk, SOCK_DEAD);
 907                 sock_put(new_sk); /* final */
 908                 *new_smc = NULL;
 909                 goto out;
 910         }
 911
 912         (*new_smc)->clcsock = new_clcsock;
 913 out:
 914         return rc;
 915 }
 916
 917 /* add a just created sock to the accept queue of the listen sock as
 918  * candidate for a following socket accept call from user space
 919  */
 920 static void smc_accept_enqueue(struct sock *parent, struct sock *sk)
 921 {
 922         struct smc_sock *par = smc_sk(parent);
 923
 924         sock_hold(sk); /* sock_put in smc_accept_unlink () */
 925         spin_lock(&par->accept_q_lock);
 926         list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q);
 927         spin_unlock(&par->accept_q_lock);
 928         sk_acceptq_added(parent);
 929 }
 930
 931 /* remove a socket from the accept queue of its parental listening socket */
 932 static void smc_accept_unlink(struct sock *sk)
 933 {
 934         struct smc_sock *par = smc_sk(sk)->listen_smc;
 935
 936         spin_lock(&par->accept_q_lock);
 937         list_del_init(&smc_sk(sk)->accept_q);
 938         spin_unlock(&par->accept_q_lock);
 939         sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk);
 940         sock_put(sk); /* sock_hold in smc_accept_enqueue */
 941 }
 942
 943 /* remove a sock from the accept queue to bind it to a new socket created
 944  * for a socket accept call from user space
 945  */
 946 struct sock *smc_accept_dequeue(struct sock *parent,
 947                                 struct socket *new_sock)
 948 {
 949         struct smc_sock *isk, *n;
 950         struct sock *new_sk;
 951
 952         list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) {
 953                 new_sk = (struct sock *)isk;
 954
 955                 smc_accept_unlink(new_sk);
 956                 if (new_sk->sk_state == SMC_CLOSED) {
 957                         new_sk->sk_prot->unhash(new_sk);
 958                         if (isk->clcsock) {
 959                                 sock_release(isk->clcsock);
 960                                 isk->clcsock = NULL;
 961                         }
 962                         sock_put(new_sk); /* final */
 963                         continue;
 964                 }
 965                 if (new_sock) {
 966                         sock_graft(new_sk, new_sock);
 967                         if (isk->use_fallback) {
 968                                 smc_sk(new_sk)->clcsock->file = new_sock->file;
 969                                 isk->clcsock->file->private_data = isk->clcsock;
 970                         }
 971                 }
 972                 return new_sk;
 973         }
 974         return NULL;
 975 }
 976
 977 /* clean up for a created but never accepted sock */
 978 void smc_close_non_accepted(struct sock *sk)
 979 {
 980         struct smc_sock *smc = smc_sk(sk);
 981
 982         sock_hold(sk); /* sock_put below */
 983         lock_sock(sk);
 984         if (!sk->sk_lingertime)
 985                 /* wait for peer closing */
 986                 sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT;
 987         __smc_release(smc);
 988         release_sock(sk);
 989         sock_put(sk); /* sock_hold above */
 990         sock_put(sk); /* final sock_put */
 991 }
 992
 993 static int smc_serv_conf_first_link(struct smc_sock *smc)
 994 {
 995         struct net *net = sock_net(smc->clcsock->sk);
 996         struct smc_link_group *lgr = smc->conn.lgr;
 997         struct smc_link *link;
 998         int rest;
 999         int rc;
1000
1001         link = &lgr->lnk[SMC_SINGLE_LINK];
1002
1003         if (smc_reg_rmb(link, smc->conn.rmb_desc, false))
1004                 return SMC_CLC_DECL_ERR_REGRMB;
1005
1006         /* send CONFIRM LINK request to client over the RoCE fabric */
1007         rc = smc_llc_send_confirm_link(link, SMC_LLC_REQ);
1008         if (rc < 0)
1009                 return SMC_CLC_DECL_TIMEOUT_CL;
1010
1011         /* receive CONFIRM LINK response from client over the RoCE fabric */
1012         rest = wait_for_completion_interruptible_timeout(
1013                 &link->llc_confirm_resp,
1014                 SMC_LLC_WAIT_FIRST_TIME);
1015         if (rest <= 0) {
1016                 struct smc_clc_msg_decline dclc;
1017
1018                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
1019                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
1020                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
1021         }
1022
1023         if (link->llc_confirm_resp_rc)
1024                 return SMC_CLC_DECL_RMBE_EC;
1025
1026         /* send ADD LINK request to client over the RoCE fabric */
1027         rc = smc_llc_send_add_link(link,
1028                                    link->smcibdev->mac[link->ibport - 1],
1029                                    link->gid, SMC_LLC_REQ);
1030         if (rc < 0)
1031                 return SMC_CLC_DECL_TIMEOUT_AL;
1032
1033         /* receive ADD LINK response from client over the RoCE fabric */
1034         rest = wait_for_completion_interruptible_timeout(&link->llc_add_resp,
1035                                                          SMC_LLC_WAIT_TIME);
1036         if (rest <= 0) {
1037                 struct smc_clc_msg_decline dclc;
1038
1039                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
1040                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
1041                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_AL : rc;
1042         }
1043
1044         smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);
1045
1046         return 0;
1047 }
1048
1049 /* listen worker: finish */
1050 static void smc_listen_out(struct smc_sock *new_smc)
1051 {
1052         struct smc_sock *lsmc = new_smc->listen_smc;
1053         struct sock *newsmcsk = &new_smc->sk;
1054
1055         if (lsmc->sk.sk_state == SMC_LISTEN) {
1056                 lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING);
1057                 smc_accept_enqueue(&lsmc->sk, newsmcsk);
1058                 release_sock(&lsmc->sk);
1059         } else { /* no longer listening */
1060                 smc_close_non_accepted(newsmcsk);
1061         }
1062
1063         /* Wake up accept */
1064         lsmc->sk.sk_data_ready(&lsmc->sk);
1065         sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */
1066 }
1067
1068 /* listen worker: finish in state connected */
1069 static void smc_listen_out_connected(struct smc_sock *new_smc)
1070 {
1071         struct sock *newsmcsk = &new_smc->sk;
1072
1073         sk_refcnt_debug_inc(newsmcsk);
1074         if (newsmcsk->sk_state == SMC_INIT)
1075                 newsmcsk->sk_state = SMC_ACTIVE;
1076
1077         smc_listen_out(new_smc);
1078 }
1079
1080 /* listen worker: finish in error state */
1081 static void smc_listen_out_err(struct smc_sock *new_smc)
1082 {
1083         struct sock *newsmcsk = &new_smc->sk;
1084
1085         if (newsmcsk->sk_state == SMC_INIT)
1086                 sock_put(&new_smc->sk); /* passive closing */
1087         newsmcsk->sk_state = SMC_CLOSED;
1088         smc_conn_free(&new_smc->conn);
1089
1090         smc_listen_out(new_smc);
1091 }
1092
1093 /* listen worker: decline and fall back if possible */
1094 static void smc_listen_decline(struct smc_sock *new_smc, int reason_code,
1095                                int local_contact)
1096 {
1097         /* RDMA setup failed, switch back to TCP */
1098         if (local_contact == SMC_FIRST_CONTACT)
1099                 smc_lgr_forget(new_smc->conn.lgr);
1100         if (reason_code < 0) { /* error, no fallback possible */
1101                 smc_listen_out_err(new_smc);
1102                 return;
1103         }
1104         smc_conn_free(&new_smc->conn);
1105         smc_switch_to_fallback(new_smc);
1106         new_smc->fallback_rsn = reason_code;
1107         if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) {
1108                 if (smc_clc_send_decline(new_smc, reason_code) < 0) {
1109                         smc_listen_out_err(new_smc);
1110                         return;
1111                 }
1112         }
1113         smc_listen_out_connected(new_smc);
1114 }
1115
1116 /* listen worker: check prefixes */
1117 static int smc_listen_prfx_check(struct smc_sock *new_smc,
1118                                  struct smc_clc_msg_proposal *pclc)
1119 {
1120         struct smc_clc_msg_proposal_prefix *pclc_prfx;
1121         struct socket *newclcsock = new_smc->clcsock;
1122
1123         pclc_prfx = smc_clc_proposal_get_prefix(pclc);
1124         if (smc_clc_prfx_match(newclcsock, pclc_prfx))
1125                 return SMC_CLC_DECL_DIFFPREFIX;
1126
1127         return 0;
1128 }
1129
1130 /* listen worker: initialize connection and buffers */
1131 static int smc_listen_rdma_init(struct smc_sock *new_smc,
1132                                 struct smc_init_info *ini)
1133 {
1134         int rc;
1135
1136         /* allocate connection / link group */
1137         rc = smc_conn_create(new_smc, ini);
1138         if (rc)
1139                 return rc;
1140
1141         /* create send buffer and rmb */
1142         if (smc_buf_create(new_smc, false))
1143                 return SMC_CLC_DECL_MEM;
1144
1145         return 0;
1146 }
1147
1148 /* listen worker: initialize connection and buffers for SMC-D */
1149 static int smc_listen_ism_init(struct smc_sock *new_smc,
1150                                struct smc_clc_msg_proposal *pclc,
1151                                struct smc_init_info *ini)
1152 {
1153         struct smc_clc_msg_smcd *pclc_smcd;
1154         int rc;
1155
1156         pclc_smcd = smc_get_clc_msg_smcd(pclc);
1157         ini->ism_gid = pclc_smcd->gid;
1158         rc = smc_conn_create(new_smc, ini);
1159         if (rc)
1160                 return rc;
1161
1162         /* Check if peer can be reached via ISM device */
1163         if (smc_ism_cantalk(new_smc->conn.lgr->peer_gid,
1164                             new_smc->conn.lgr->vlan_id,
1165                             new_smc->conn.lgr->smcd)) {
1166                 if (ini->cln_first_contact == SMC_FIRST_CONTACT)
1167                         smc_lgr_forget(new_smc->conn.lgr);
1168                 smc_conn_free(&new_smc->conn);
1169                 return SMC_CLC_DECL_SMCDNOTALK;
1170         }
1171
1172         /* Create send and receive buffers */
1173         if (smc_buf_create(new_smc, true)) {
1174                 if (ini->cln_first_contact == SMC_FIRST_CONTACT)
1175                         smc_lgr_forget(new_smc->conn.lgr);
1176                 smc_conn_free(&new_smc->conn);
1177                 return SMC_CLC_DECL_MEM;
1178         }
1179
1180         return 0;
1181 }
1182
1183 /* listen worker: register buffers */
1184 static int smc_listen_rdma_reg(struct smc_sock *new_smc, int local_contact)
1185 {
1186         struct smc_link *link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK];
1187
1188         if (local_contact != SMC_FIRST_CONTACT) {
1189                 if (smc_reg_rmb(link, new_smc->conn.rmb_desc, true))
1190                         return SMC_CLC_DECL_ERR_REGRMB;
1191         }
1192         smc_rmb_sync_sg_for_device(&new_smc->conn);
1193
1194         return 0;
1195 }
1196
1197 /* listen worker: finish RDMA setup */
1198 static int smc_listen_rdma_finish(struct smc_sock *new_smc,
1199                                   struct smc_clc_msg_accept_confirm *cclc,
1200                                   int local_contact)
1201 {
1202         struct smc_link *link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK];
1203         int reason_code = 0;
1204
1205         if (local_contact == SMC_FIRST_CONTACT)
1206                 smc_link_save_peer_info(link, cclc);
1207
1208         if (smc_rmb_rtoken_handling(&new_smc->conn, cclc)) {
1209                 reason_code = SMC_CLC_DECL_ERR_RTOK;
1210                 goto decline;
1211         }
1212
1213         if (local_contact == SMC_FIRST_CONTACT) {
1214                 if (smc_ib_ready_link(link)) {
1215                         reason_code = SMC_CLC_DECL_ERR_RDYLNK;
1216                         goto decline;
1217                 }
1218                 /* QP confirmation over RoCE fabric */
1219                 reason_code = smc_serv_conf_first_link(new_smc);
1220                 if (reason_code)
1221                         goto decline;
1222         }
1223         return 0;
1224
1225 decline:
1226         smc_listen_decline(new_smc, reason_code, local_contact);
1227         return reason_code;
1228 }
1229
1230 /* setup for RDMA connection of server */
1231 static void smc_listen_work(struct work_struct *work)
1232 {
1233         struct smc_sock *new_smc = container_of(work, struct smc_sock,
1234                                                 smc_listen_work);
1235         struct socket *newclcsock = new_smc->clcsock;
1236         struct smc_clc_msg_accept_confirm cclc;
1237         struct smc_clc_msg_proposal *pclc;
1238         struct smc_init_info ini = {0};
1239         bool ism_supported = false;
1240         u8 buf[SMC_CLC_MAX_LEN];
1241         int rc = 0;
1242
1243         if (new_smc->listen_smc->sk.sk_state != SMC_LISTEN)
1244                 return smc_listen_out_err(new_smc);
1245
1246         if (new_smc->use_fallback) {
1247                 smc_listen_out_connected(new_smc);
1248                 return;
1249         }
1250
1251         /* check if peer is smc capable */
1252         if (!tcp_sk(newclcsock->sk)->syn_smc) {
1253                 smc_switch_to_fallback(new_smc);
1254                 new_smc->fallback_rsn = SMC_CLC_DECL_PEERNOSMC;
1255                 smc_listen_out_connected(new_smc);
1256                 return;
1257         }
1258
1259         /* do inband token exchange -
1260          * wait for and receive SMC Proposal CLC message
1261          */
1262         pclc = (struct smc_clc_msg_proposal *)&buf;
1263         rc = smc_clc_wait_msg(new_smc, pclc, SMC_CLC_MAX_LEN,
1264                               SMC_CLC_PROPOSAL, CLC_WAIT_TIME);
1265         if (rc)
1266                 goto out_decl;
1267
1268         /* IPSec connections opt out of SMC-R optimizations */
1269         if (using_ipsec(new_smc)) {
1270                 rc = SMC_CLC_DECL_IPSEC;
1271                 goto out_decl;
1272         }
1273
1274         /* check for matching IP prefix and subnet length */
1275         rc = smc_listen_prfx_check(new_smc, pclc);
1276         if (rc)
1277                 goto out_decl;
1278
1279         /* get vlan id from IP device */
1280         if (smc_vlan_by_tcpsk(new_smc->clcsock, &ini)) {
1281                 rc = SMC_CLC_DECL_GETVLANERR;
1282                 goto out_decl;
1283         }
1284
1285         mutex_lock(&smc_server_lgr_pending);
1286         smc_close_init(new_smc);
1287         smc_rx_init(new_smc);
1288         smc_tx_init(new_smc);
1289
1290         /* check if ISM is available */
1291         if (pclc->hdr.path == SMC_TYPE_D || pclc->hdr.path == SMC_TYPE_B) {
1292                 ini.is_smcd = true; /* prepare ISM check */
1293                 rc = smc_find_ism_device(new_smc, &ini);
1294                 if (!rc)
1295                         rc = smc_listen_ism_init(new_smc, pclc, &ini);
1296                 if (!rc)
1297                         ism_supported = true;
1298                 else if (pclc->hdr.path == SMC_TYPE_D)
1299                         goto out_unlock; /* skip RDMA and decline */
1300         }
1301
1302         /* check if RDMA is available */
1303         if (!ism_supported) { /* SMC_TYPE_R or SMC_TYPE_B */
1304                 /* prepare RDMA check */
1305                 ini.is_smcd = false;
1306                 ini.ism_dev = NULL;
1307                 ini.ib_lcl = &pclc->lcl;
1308                 rc = smc_find_rdma_device(new_smc, &ini);
1309                 if (rc) {
1310                         /* no RDMA device found */
1311                         if (pclc->hdr.path == SMC_TYPE_B)
1312                                 /* neither ISM nor RDMA device found */
1313                                 rc = SMC_CLC_DECL_NOSMCDEV;
1314                         goto out_unlock;
1315                 }
1316                 rc = smc_listen_rdma_init(new_smc, &ini);
1317                 if (rc)
1318                         goto out_unlock;
1319                 rc = smc_listen_rdma_reg(new_smc, ini.cln_first_contact);
1320                 if (rc)
1321                         goto out_unlock;
1322         }
1323
1324         /* send SMC Accept CLC message */
1325         rc = smc_clc_send_accept(new_smc, ini.cln_first_contact);
1326         if (rc)
1327                 goto out_unlock;
1328
1329         /* SMC-D does not need this lock any more */
1330         if (ism_supported)
1331                 mutex_unlock(&smc_server_lgr_pending);
1332
1333         /* receive SMC Confirm CLC message */
1334         rc = smc_clc_wait_msg(new_smc, &cclc, sizeof(cclc),
1335                               SMC_CLC_CONFIRM, CLC_WAIT_TIME);
1336         if (rc) {
1337                 if (!ism_supported)
1338                         goto out_unlock;
1339                 goto out_decl;
1340         }
1341
1342         /* finish worker */
1343         if (!ism_supported) {
1344                 rc = smc_listen_rdma_finish(new_smc, &cclc,
1345                                             ini.cln_first_contact);
1346                 mutex_unlock(&smc_server_lgr_pending);
1347                 if (rc)
1348                         return;
1349         }
1350         smc_conn_save_peer_info(new_smc, &cclc);
1351         smc_listen_out_connected(new_smc);
1352         return;
1353
1354 out_unlock:
1355         mutex_unlock(&smc_server_lgr_pending);
1356 out_decl:
1357         smc_listen_decline(new_smc, rc, ini.cln_first_contact);
1358 }
1359
1360 static void smc_tcp_listen_work(struct work_struct *work)
1361 {
1362         struct smc_sock *lsmc = container_of(work, struct smc_sock,
1363                                              tcp_listen_work);
1364         struct sock *lsk = &lsmc->sk;
1365         struct smc_sock *new_smc;
1366         int rc = 0;
1367
1368         lock_sock(lsk);
1369         while (lsk->sk_state == SMC_LISTEN) {
1370                 rc = smc_clcsock_accept(lsmc, &new_smc);
1371                 if (rc)
1372                         goto out;
1373                 if (!new_smc)
1374                         continue;
1375
1376                 new_smc->listen_smc = lsmc;
1377                 new_smc->use_fallback = lsmc->use_fallback;
1378                 new_smc->fallback_rsn = lsmc->fallback_rsn;
1379                 sock_hold(lsk); /* sock_put in smc_listen_work */
1380                 INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
1381                 smc_copy_sock_settings_to_smc(new_smc);
1382                 new_smc->sk.sk_sndbuf = lsmc->sk.sk_sndbuf;
1383                 new_smc->sk.sk_rcvbuf = lsmc->sk.sk_rcvbuf;
1384                 sock_hold(&new_smc->sk); /* sock_put in passive closing */
1385                 if (!schedule_work(&new_smc->smc_listen_work))
1386                         sock_put(&new_smc->sk);
1387         }
1388
1389 out:
1390         release_sock(lsk);
1391         sock_put(&lsmc->sk); /* sock_hold in smc_listen */
1392 }
1393
1394 static int smc_listen(struct socket *sock, int backlog)
1395 {
1396         struct sock *sk = sock->sk;
1397         struct smc_sock *smc;
1398         int rc;
1399
1400         smc = smc_sk(sk);
1401         lock_sock(sk);
1402
1403         rc = -EINVAL;
1404         if ((sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) ||
1405             smc->connect_nonblock)
1406                 goto out;
1407
1408         rc = 0;
1409         if (sk->sk_state == SMC_LISTEN) {
1410                 sk->sk_max_ack_backlog = backlog;
1411                 goto out;
1412         }
1413         /* some socket options are handled in core, so we could not apply
1414          * them to the clc socket -- copy smc socket options to clc socket
1415          */
1416         smc_copy_sock_settings_to_clc(smc);
1417         if (!smc->use_fallback)
1418                 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
1419
1420         rc = kernel_listen(smc->clcsock, backlog);
1421         if (rc)
1422                 goto out;
1423         sk->sk_max_ack_backlog = backlog;
1424         sk->sk_ack_backlog = 0;
1425         sk->sk_state = SMC_LISTEN;
1426         sock_hold(sk); /* sock_hold in tcp_listen_worker */
1427         if (!schedule_work(&smc->tcp_listen_work))
1428                 sock_put(sk);
1429
1430 out:
1431         release_sock(sk);
1432         return rc;
1433 }
1434
1435 static int smc_accept(struct socket *sock, struct socket *new_sock,
1436                       int flags, bool kern)
1437 {
1438         struct sock *sk = sock->sk, *nsk;
1439         DECLARE_WAITQUEUE(wait, current);
1440         struct smc_sock *lsmc;
1441         long timeo;
1442         int rc = 0;
1443
1444         lsmc = smc_sk(sk);
1445         sock_hold(sk); /* sock_put below */
1446         lock_sock(sk);
1447
1448         if (lsmc->sk.sk_state != SMC_LISTEN) {
1449                 rc = -EINVAL;
1450                 release_sock(sk);
1451                 goto out;
1452         }
1453
1454         /* Wait for an incoming connection */
1455         timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
1456         add_wait_queue_exclusive(sk_sleep(sk), &wait);
1457         while (!(nsk = smc_accept_dequeue(sk, new_sock))) {
1458                 set_current_state(TASK_INTERRUPTIBLE);
1459                 if (!timeo) {
1460                         rc = -EAGAIN;
1461                         break;
1462                 }
1463                 release_sock(sk);
1464                 timeo = schedule_timeout(timeo);
1465                 /* wakeup by sk_data_ready in smc_listen_work() */
1466                 sched_annotate_sleep();
1467                 lock_sock(sk);
1468                 if (signal_pending(current)) {
1469                         rc = sock_intr_errno(timeo);
1470                         break;
1471                 }
1472         }
1473         set_current_state(TASK_RUNNING);
1474         remove_wait_queue(sk_sleep(sk), &wait);
1475
1476         if (!rc)
1477                 rc = sock_error(nsk);
1478         release_sock(sk);
1479         if (rc)
1480                 goto out;
1481
1482         if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) {
1483                 /* wait till data arrives on the socket */
1484                 timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept *
1485                                                                 MSEC_PER_SEC);
1486                 if (smc_sk(nsk)->use_fallback) {
1487                         struct sock *clcsk = smc_sk(nsk)->clcsock->sk;
1488
1489                         lock_sock(clcsk);
1490                         if (skb_queue_empty(&clcsk->sk_receive_queue))
1491                                 sk_wait_data(clcsk, &timeo, NULL);
1492                         release_sock(clcsk);
1493                 } else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) {
1494                         lock_sock(nsk);
1495                         smc_rx_wait(smc_sk(nsk), &timeo, smc_rx_data_available);
1496                         release_sock(nsk);
1497                 }
1498         }
1499
1500 out:
1501         sock_put(sk); /* sock_hold above */
1502         return rc;
1503 }
1504
1505 static int smc_getname(struct socket *sock, struct sockaddr *addr,
1506                        int peer)
1507 {
1508         struct smc_sock *smc;
1509
1510         if (peer && (sock->sk->sk_state != SMC_ACTIVE) &&
1511             (sock->sk->sk_state != SMC_APPCLOSEWAIT1))
1512                 return -ENOTCONN;
1513
1514         smc = smc_sk(sock->sk);
1515
1516         return smc->clcsock->ops->getname(smc->clcsock, addr, peer);
1517 }
1518
1519 static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
1520 {
1521         struct sock *sk = sock->sk;
1522         struct smc_sock *smc;
1523         int rc = -EPIPE;
1524
1525         smc = smc_sk(sk);
1526         lock_sock(sk);
1527         if ((sk->sk_state != SMC_ACTIVE) &&
1528             (sk->sk_state != SMC_APPCLOSEWAIT1) &&
1529             (sk->sk_state != SMC_INIT))
1530                 goto out;
1531
1532         if (msg->msg_flags & MSG_FASTOPEN) {
1533                 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
1534                         smc_switch_to_fallback(smc);
1535                         smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
1536                 } else {
1537                         rc = -EINVAL;
1538                         goto out;
1539                 }
1540         }
1541
1542         if (smc->use_fallback)
1543                 rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len);
1544         else
1545                 rc = smc_tx_sendmsg(smc, msg, len);
1546 out:
1547         release_sock(sk);
1548         return rc;
1549 }
1550
1551 static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
1552                        int flags)
1553 {
1554         struct sock *sk = sock->sk;
1555         struct smc_sock *smc;
1556         int rc = -ENOTCONN;
1557
1558         smc = smc_sk(sk);
1559         lock_sock(sk);
1560         if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
1561                 /* socket was connected before, no more data to read */
1562                 rc = 0;
1563                 goto out;
1564         }
1565         if ((sk->sk_state == SMC_INIT) ||
1566             (sk->sk_state == SMC_LISTEN) ||
1567             (sk->sk_state == SMC_CLOSED))
1568                 goto out;
1569
1570         if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
1571                 rc = 0;
1572                 goto out;
1573         }
1574
1575         if (smc->use_fallback) {
1576                 rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags);
1577         } else {
1578                 msg->msg_namelen = 0;
1579                 rc = smc_rx_recvmsg(smc, msg, NULL, len, flags);
1580         }
1581
1582 out:
1583         release_sock(sk);
1584         return rc;
1585 }
1586
1587 static __poll_t smc_accept_poll(struct sock *parent)
1588 {
1589         struct smc_sock *isk = smc_sk(parent);
1590         __poll_t mask = 0;
1591
1592         spin_lock(&isk->accept_q_lock);
1593         if (!list_empty(&isk->accept_q))
1594                 mask = EPOLLIN | EPOLLRDNORM;
1595         spin_unlock(&isk->accept_q_lock);
1596
1597         return mask;
1598 }
1599
1600 static __poll_t smc_poll(struct file *file, struct socket *sock,
1601                              poll_table *wait)
1602 {
1603         struct sock *sk = sock->sk;
1604         struct smc_sock *smc;
1605         __poll_t mask = 0;
1606
1607         if (!sk)
1608                 return EPOLLNVAL;
1609
1610         smc = smc_sk(sock->sk);
1611         if (smc->use_fallback) {
1612                 /* delegate to CLC child sock */
1613                 mask = smc->clcsock->ops->poll(file, smc->clcsock, wait);
1614                 sk->sk_err = smc->clcsock->sk->sk_err;
1615         } else {
1616                 if (sk->sk_state != SMC_CLOSED)
1617                         sock_poll_wait(file, sock, wait);
1618                 if (sk->sk_err)
1619                         mask |= EPOLLERR;
1620                 if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
1621                     (sk->sk_state == SMC_CLOSED))
1622                         mask |= EPOLLHUP;
1623                 if (sk->sk_state == SMC_LISTEN) {
1624                         /* woken up by sk_data_ready in smc_listen_work() */
1625                         mask |= smc_accept_poll(sk);
1626                 } else if (smc->use_fallback) { /* as result of connect_work()*/
1627                         mask |= smc->clcsock->ops->poll(file, smc->clcsock,
1628                                                            wait);
1629                         sk->sk_err = smc->clcsock->sk->sk_err;
1630                 } else {
1631                         if ((sk->sk_state != SMC_INIT &&
1632                              atomic_read(&smc->conn.sndbuf_space)) ||
1633                             sk->sk_shutdown & SEND_SHUTDOWN) {
1634                                 mask |= EPOLLOUT | EPOLLWRNORM;
1635                         } else {
1636                                 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1637                                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1638                         }
1639                         if (atomic_read(&smc->conn.bytes_to_rcv))
1640                                 mask |= EPOLLIN | EPOLLRDNORM;
1641                         if (sk->sk_shutdown & RCV_SHUTDOWN)
1642                                 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
1643                         if (sk->sk_state == SMC_APPCLOSEWAIT1)
1644                                 mask |= EPOLLIN;
1645                         if (smc->conn.urg_state == SMC_URG_VALID)
1646                                 mask |= EPOLLPRI;
1647                 }
1648         }
1649
1650         return mask;
1651 }
1652
1653 static int smc_shutdown(struct socket *sock, int how)
1654 {
1655         struct sock *sk = sock->sk;
1656         struct smc_sock *smc;
1657         int rc = -EINVAL;
1658         int rc1 = 0;
1659
1660         smc = smc_sk(sk);
1661
1662         if ((how < SHUT_RD) || (how > SHUT_RDWR))
1663                 return rc;
1664
1665         lock_sock(sk);
1666
1667         rc = -ENOTCONN;
1668         if ((sk->sk_state != SMC_ACTIVE) &&
1669             (sk->sk_state != SMC_PEERCLOSEWAIT1) &&
1670             (sk->sk_state != SMC_PEERCLOSEWAIT2) &&
1671             (sk->sk_state != SMC_APPCLOSEWAIT1) &&
1672             (sk->sk_state != SMC_APPCLOSEWAIT2) &&
1673             (sk->sk_state != SMC_APPFINCLOSEWAIT))
1674                 goto out;
1675         if (smc->use_fallback) {
1676                 rc = kernel_sock_shutdown(smc->clcsock, how);
1677                 sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
1678                 if (sk->sk_shutdown == SHUTDOWN_MASK)
1679                         sk->sk_state = SMC_CLOSED;
1680                 goto out;
1681         }
1682         switch (how) {
1683         case SHUT_RDWR:         /* shutdown in both directions */
1684                 rc = smc_close_active(smc);
1685                 break;
1686         case SHUT_WR:
1687                 rc = smc_close_shutdown_write(smc);
1688                 break;
1689         case SHUT_RD:
1690                 rc = 0;
1691                 /* nothing more to do because peer is not involved */
1692                 break;
1693         }
1694         if (smc->clcsock)
1695                 rc1 = kernel_sock_shutdown(smc->clcsock, how);
1696         /* map sock_shutdown_cmd constants to sk_shutdown value range */
1697         sk->sk_shutdown |= how + 1;
1698
1699 out:
1700         release_sock(sk);
1701         return rc ? rc : rc1;
1702 }
1703
1704 static int smc_setsockopt(struct socket *sock, int level, int optname,
1705                           char __user *optval, unsigned int optlen)
1706 {
1707         struct sock *sk = sock->sk;
1708         struct smc_sock *smc;
1709         int val, rc;
1710
1711         smc = smc_sk(sk);
1712
1713         /* generic setsockopts reaching us here always apply to the
1714          * CLC socket
1715          */
1716         rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname,
1717                                            optval, optlen);
1718         if (smc->clcsock->sk->sk_err) {
1719                 sk->sk_err = smc->clcsock->sk->sk_err;
1720                 sk->sk_error_report(sk);
1721         }
1722         if (rc)
1723                 return rc;
1724
1725         if (optlen < sizeof(int))
1726                 return -EINVAL;
1727         if (get_user(val, (int __user *)optval))
1728                 return -EFAULT;
1729
1730         lock_sock(sk);
1731         switch (optname) {
1732         case TCP_ULP:
1733         case TCP_FASTOPEN:
1734         case TCP_FASTOPEN_CONNECT:
1735         case TCP_FASTOPEN_KEY:
1736         case TCP_FASTOPEN_NO_COOKIE:
1737                 /* option not supported by SMC */
1738                 if (sk->sk_state == SMC_INIT) {
1739                         smc_switch_to_fallback(smc);
1740                         smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
1741                 } else {
1742                         if (!smc->use_fallback)
1743                                 rc = -EINVAL;
1744                 }
1745                 break;
1746         case TCP_NODELAY:
1747                 if (sk->sk_state != SMC_INIT &&
1748                     sk->sk_state != SMC_LISTEN &&
1749                     sk->sk_state != SMC_CLOSED) {
1750                         if (val && !smc->use_fallback)
1751                                 mod_delayed_work(system_wq, &smc->conn.tx_work,
1752                                                  0);
1753                 }
1754                 break;
1755         case TCP_CORK:
1756                 if (sk->sk_state != SMC_INIT &&
1757                     sk->sk_state != SMC_LISTEN &&
1758                     sk->sk_state != SMC_CLOSED) {
1759                         if (!val && !smc->use_fallback)
1760                                 mod_delayed_work(system_wq, &smc->conn.tx_work,
1761                                                  0);
1762                 }
1763                 break;
1764         case TCP_DEFER_ACCEPT:
1765                 smc->sockopt_defer_accept = val;
1766                 break;
1767         default:
1768                 break;
1769         }
1770         release_sock(sk);
1771
1772         return rc;
1773 }
1774
1775 static int smc_getsockopt(struct socket *sock, int level, int optname,
1776                           char __user *optval, int __user *optlen)
1777 {
1778         struct smc_sock *smc;
1779
1780         smc = smc_sk(sock->sk);
1781         /* socket options apply to the CLC socket */
1782         return smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
1783                                              optval, optlen);
1784 }
1785
1786 static int smc_ioctl(struct socket *sock, unsigned int cmd,
1787                      unsigned long arg)
1788 {
1789         union smc_host_cursor cons, urg;
1790         struct smc_connection *conn;
1791         struct smc_sock *smc;
1792         int answ;
1793
1794         smc = smc_sk(sock->sk);
1795         conn = &smc->conn;
1796         lock_sock(&smc->sk);
1797         if (smc->use_fallback) {
1798                 if (!smc->clcsock) {
1799                         release_sock(&smc->sk);
1800                         return -EBADF;
1801                 }
1802                 answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg);
1803                 release_sock(&smc->sk);
1804                 return answ;
1805         }
1806         switch (cmd) {
1807         case SIOCINQ: /* same as FIONREAD */
1808                 if (smc->sk.sk_state == SMC_LISTEN) {
1809                         release_sock(&smc->sk);
1810                         return -EINVAL;
1811                 }
1812                 if (smc->sk.sk_state == SMC_INIT ||
1813                     smc->sk.sk_state == SMC_CLOSED)
1814                         answ = 0;
1815                 else
1816                         answ = atomic_read(&smc->conn.bytes_to_rcv);
1817                 break;
1818         case SIOCOUTQ:
1819                 /* output queue size (not send + not acked) */
1820                 if (smc->sk.sk_state == SMC_LISTEN) {
1821                         release_sock(&smc->sk);
1822                         return -EINVAL;
1823                 }
1824                 if (smc->sk.sk_state == SMC_INIT ||
1825                     smc->sk.sk_state == SMC_CLOSED)
1826                         answ = 0;
1827                 else
1828                         answ = smc->conn.sndbuf_desc->len -
1829                                         atomic_read(&smc->conn.sndbuf_space);
1830                 break;
1831         case SIOCOUTQNSD:
1832                 /* output queue size (not send only) */
1833                 if (smc->sk.sk_state == SMC_LISTEN) {
1834                         release_sock(&smc->sk);
1835                         return -EINVAL;
1836                 }
1837                 if (smc->sk.sk_state == SMC_INIT ||
1838                     smc->sk.sk_state == SMC_CLOSED)
1839                         answ = 0;
1840                 else
1841                         answ = smc_tx_prepared_sends(&smc->conn);
1842                 break;
1843         case SIOCATMARK:
1844                 if (smc->sk.sk_state == SMC_LISTEN) {
1845                         release_sock(&smc->sk);
1846                         return -EINVAL;
1847                 }
1848                 if (smc->sk.sk_state == SMC_INIT ||
1849                     smc->sk.sk_state == SMC_CLOSED) {
1850                         answ = 0;
1851                 } else {
1852                         smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
1853                         smc_curs_copy(&urg, &conn->urg_curs, conn);
1854                         answ = smc_curs_diff(conn->rmb_desc->len,
1855                                              &cons, &urg) == 1;
1856                 }
1857                 break;
1858         default:
1859                 release_sock(&smc->sk);
1860                 return -ENOIOCTLCMD;
1861         }
1862         release_sock(&smc->sk);
1863
1864         return put_user(answ, (int __user *)arg);
1865 }
1866
1867 static ssize_t smc_sendpage(struct socket *sock, struct page *page,
1868                             int offset, size_t size, int flags)
1869 {
1870         struct sock *sk = sock->sk;
1871         struct smc_sock *smc;
1872         int rc = -EPIPE;
1873
1874         smc = smc_sk(sk);
1875         lock_sock(sk);
1876         if (sk->sk_state != SMC_ACTIVE) {
1877                 release_sock(sk);
1878                 goto out;
1879         }
1880         release_sock(sk);
1881         if (smc->use_fallback)
1882                 rc = kernel_sendpage(smc->clcsock, page, offset,
1883                                      size, flags);
1884         else
1885                 rc = sock_no_sendpage(sock, page, offset, size, flags);
1886
1887 out:
1888         return rc;
1889 }
1890
1891 /* Map the affected portions of the rmbe into an spd, note the number of bytes
1892  * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
1893  * updates till whenever a respective page has been fully processed.
1894  * Note that subsequent recv() calls have to wait till all splice() processing
1895  * completed.
1896  */
1897 static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos,
1898                                struct pipe_inode_info *pipe, size_t len,
1899                                unsigned int flags)
1900 {
1901         struct sock *sk = sock->sk;
1902         struct smc_sock *smc;
1903         int rc = -ENOTCONN;
1904
1905         smc = smc_sk(sk);
1906         lock_sock(sk);
1907         if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
1908                 /* socket was connected before, no more data to read */
1909                 rc = 0;
1910                 goto out;
1911         }
1912         if (sk->sk_state == SMC_INIT ||
1913             sk->sk_state == SMC_LISTEN ||
1914             sk->sk_state == SMC_CLOSED)
1915                 goto out;
1916
1917         if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
1918                 rc = 0;
1919                 goto out;
1920         }
1921
1922         if (smc->use_fallback) {
1923                 rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos,
1924                                                     pipe, len, flags);
1925         } else {
1926                 if (*ppos) {
1927                         rc = -ESPIPE;
1928                         goto out;
1929                 }
1930                 if (flags & SPLICE_F_NONBLOCK)
1931                         flags = MSG_DONTWAIT;
1932                 else
1933                         flags = 0;
1934                 rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags);
1935         }
1936 out:
1937         release_sock(sk);
1938
1939         return rc;
1940 }
1941
1942 /* must look like tcp */
1943 static const struct proto_ops smc_sock_ops = {
1944         .family         = PF_SMC,
1945         .owner          = THIS_MODULE,
1946         .release        = smc_release,
1947         .bind           = smc_bind,
1948         .connect        = smc_connect,
1949         .socketpair     = sock_no_socketpair,
1950         .accept         = smc_accept,
1951         .getname        = smc_getname,
1952         .poll           = smc_poll,
1953         .ioctl          = smc_ioctl,
1954         .listen         = smc_listen,
1955         .shutdown       = smc_shutdown,
1956         .setsockopt     = smc_setsockopt,
1957         .getsockopt     = smc_getsockopt,
1958         .sendmsg        = smc_sendmsg,
1959         .recvmsg        = smc_recvmsg,
1960         .mmap           = sock_no_mmap,
1961         .sendpage       = smc_sendpage,
1962         .splice_read    = smc_splice_read,
1963 };
1964
1965 static int smc_create(struct net *net, struct socket *sock, int protocol,
1966                       int kern)
1967 {
1968         int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET;
1969         struct smc_sock *smc;
1970         struct sock *sk;
1971         int rc;
1972
1973         rc = -ESOCKTNOSUPPORT;
1974         if (sock->type != SOCK_STREAM)
1975                 goto out;
1976
1977         rc = -EPROTONOSUPPORT;
1978         if (protocol != SMCPROTO_SMC && protocol != SMCPROTO_SMC6)
1979                 goto out;
1980
1981         rc = -ENOBUFS;
1982         sock->ops = &smc_sock_ops;
1983         sk = smc_sock_alloc(net, sock, protocol);
1984         if (!sk)
1985                 goto out;
1986
1987         /* create internal TCP socket for CLC handshake and fallback */
1988         smc = smc_sk(sk);
1989         smc->use_fallback = false; /* assume rdma capability first */
1990         smc->fallback_rsn = 0;
1991         rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP,
1992                               &smc->clcsock);
1993         if (rc) {
1994                 sk_common_release(sk);
1995                 goto out;
1996         }
1997         smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE);
1998         smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE);
1999
2000 out:
2001         return rc;
2002 }
2003
2004 static const struct net_proto_family smc_sock_family_ops = {
2005         .family = PF_SMC,
2006         .owner  = THIS_MODULE,
2007         .create = smc_create,
2008 };
2009
2010 unsigned int smc_net_id;
2011
2012 static __net_init int smc_net_init(struct net *net)
2013 {
2014         return smc_pnet_net_init(net);
2015 }
2016
2017 static void __net_exit smc_net_exit(struct net *net)
2018 {
2019         smc_pnet_net_exit(net);
2020 }
2021
2022 static struct pernet_operations smc_net_ops = {
2023         .init = smc_net_init,
2024         .exit = smc_net_exit,
2025         .id   = &smc_net_id,
2026         .size = sizeof(struct smc_net),
2027 };
2028
2029 static int __init smc_init(void)
2030 {
2031         int rc;
2032
2033         rc = register_pernet_subsys(&smc_net_ops);
2034         if (rc)
2035                 return rc;
2036
2037         rc = smc_pnet_init();
2038         if (rc)
2039                 goto out_pernet_subsys;
2040
2041         rc = smc_llc_init();
2042         if (rc) {
2043                 pr_err("%s: smc_llc_init fails with %d\n", __func__, rc);
2044                 goto out_pnet;
2045         }
2046
2047         rc = smc_cdc_init();
2048         if (rc) {
2049                 pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc);
2050                 goto out_pnet;
2051         }
2052
2053         rc = proto_register(&smc_proto, 1);
2054         if (rc) {
2055                 pr_err("%s: proto_register(v4) fails with %d\n", __func__, rc);
2056                 goto out_pnet;
2057         }
2058
2059         rc = proto_register(&smc_proto6, 1);
2060         if (rc) {
2061                 pr_err("%s: proto_register(v6) fails with %d\n", __func__, rc);
2062                 goto out_proto;
2063         }
2064
2065         rc = sock_register(&smc_sock_family_ops);
2066         if (rc) {
2067                 pr_err("%s: sock_register fails with %d\n", __func__, rc);
2068                 goto out_proto6;
2069         }
2070         INIT_HLIST_HEAD(&smc_v4_hashinfo.ht);
2071         INIT_HLIST_HEAD(&smc_v6_hashinfo.ht);
2072
2073         rc = smc_ib_register_client();
2074         if (rc) {
2075                 pr_err("%s: ib_register fails with %d\n", __func__, rc);
2076                 goto out_sock;
2077         }
2078
2079         static_branch_enable(&tcp_have_smc);
2080         return 0;
2081
2082 out_sock:
2083         sock_unregister(PF_SMC);
2084 out_proto6:
2085         proto_unregister(&smc_proto6);
2086 out_proto:
2087         proto_unregister(&smc_proto);
2088 out_pnet:
2089         smc_pnet_exit();
2090 out_pernet_subsys:
2091         unregister_pernet_subsys(&smc_net_ops);
2092
2093         return rc;
2094 }
2095
2096 static void __exit smc_exit(void)
2097 {
2098         smc_core_exit();
2099         static_branch_disable(&tcp_have_smc);
2100         smc_ib_unregister_client();
2101         sock_unregister(PF_SMC);
2102         proto_unregister(&smc_proto6);
2103         proto_unregister(&smc_proto);
2104         smc_pnet_exit();
2105         unregister_pernet_subsys(&smc_net_ops);
2106 }
2107
2108 module_init(smc_init);
2109 module_exit(smc_exit);
2110
2111 MODULE_AUTHOR("Ursula Braun <[email protected]>");
2112 MODULE_DESCRIPTION("smc socket address family");
2113 MODULE_LICENSE("GPL");
2114 MODULE_ALIAS_NETPROTO(PF_SMC);