drivers/nvme/target/rdma.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * NVMe over Fabrics RDMA target.
   4  * Copyright (c) 2015-2016 HGST, a Western Digital Company.
   5  */
   6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   7 #include <linux/atomic.h>
   8 #include <linux/blk-integrity.h>
   9 #include <linux/ctype.h>
  10 #include <linux/delay.h>
  11 #include <linux/err.h>
  12 #include <linux/init.h>
  13 #include <linux/module.h>
  14 #include <linux/nvme.h>
  15 #include <linux/slab.h>
  16 #include <linux/string.h>
  17 #include <linux/wait.h>
  18 #include <linux/inet.h>
  19 #include <linux/unaligned.h>
  20
  21 #include <rdma/ib_verbs.h>
  22 #include <rdma/rdma_cm.h>
  23 #include <rdma/rw.h>
  24 #include <rdma/ib_cm.h>
  25
  26 #include <linux/nvme-rdma.h>
  27 #include "nvmet.h"
  28
  29 /*
  30  * We allow at least 1 page, up to 4 SGEs, and up to 16KB of inline data
  31  */
  32 #define NVMET_RDMA_DEFAULT_INLINE_DATA_SIZE     PAGE_SIZE
  33 #define NVMET_RDMA_MAX_INLINE_SGE               4
  34 #define NVMET_RDMA_MAX_INLINE_DATA_SIZE         max_t(int, SZ_16K, PAGE_SIZE)
  35
  36 /* Assume mpsmin == device_page_size == 4KB */
  37 #define NVMET_RDMA_MAX_MDTS                     8
  38 #define NVMET_RDMA_MAX_METADATA_MDTS            5
  39
  40 #define NVMET_RDMA_BACKLOG 128
  41
  42 #define NVMET_RDMA_DISCRETE_RSP_TAG             -1
  43
  44 struct nvmet_rdma_srq;
  45
  46 struct nvmet_rdma_cmd {
  47         struct ib_sge           sge[NVMET_RDMA_MAX_INLINE_SGE + 1];
  48         struct ib_cqe           cqe;
  49         struct ib_recv_wr       wr;
  50         struct scatterlist      inline_sg[NVMET_RDMA_MAX_INLINE_SGE];
  51         struct nvme_command     *nvme_cmd;
  52         struct nvmet_rdma_queue *queue;
  53         struct nvmet_rdma_srq   *nsrq;
  54 };
  55
  56 enum {
  57         NVMET_RDMA_REQ_INLINE_DATA      = (1 << 0),
  58 };
  59
  60 struct nvmet_rdma_rsp {
  61         struct ib_sge           send_sge;
  62         struct ib_cqe           send_cqe;
  63         struct ib_send_wr       send_wr;
  64
  65         struct nvmet_rdma_cmd   *cmd;
  66         struct nvmet_rdma_queue *queue;
  67
  68         struct ib_cqe           read_cqe;
  69         struct ib_cqe           write_cqe;
  70         struct rdma_rw_ctx      rw;
  71
  72         struct nvmet_req        req;
  73
  74         bool                    allocated;
  75         u8                      n_rdma;
  76         u32                     flags;
  77         u32                     invalidate_rkey;
  78
  79         struct list_head        wait_list;
  80         int                     tag;
  81 };
  82
  83 enum nvmet_rdma_queue_state {
  84         NVMET_RDMA_Q_CONNECTING,
  85         NVMET_RDMA_Q_LIVE,
  86         NVMET_RDMA_Q_DISCONNECTING,
  87 };
  88
  89 struct nvmet_rdma_queue {
  90         struct rdma_cm_id       *cm_id;
  91         struct ib_qp            *qp;
  92         struct nvmet_port       *port;
  93         struct ib_cq            *cq;
  94         atomic_t                sq_wr_avail;
  95         struct nvmet_rdma_device *dev;
  96         struct nvmet_rdma_srq   *nsrq;
  97         spinlock_t              state_lock;
  98         enum nvmet_rdma_queue_state state;
  99         struct nvmet_cq         nvme_cq;
 100         struct nvmet_sq         nvme_sq;
 101
 102         struct nvmet_rdma_rsp   *rsps;
 103         struct sbitmap          rsp_tags;
 104         struct nvmet_rdma_cmd   *cmds;
 105
 106         struct work_struct      release_work;
 107         struct list_head        rsp_wait_list;
 108         struct list_head        rsp_wr_wait_list;
 109         spinlock_t              rsp_wr_wait_lock;
 110
 111         int                     idx;
 112         int                     host_qid;
 113         int                     comp_vector;
 114         int                     recv_queue_size;
 115         int                     send_queue_size;
 116
 117         struct list_head        queue_list;
 118 };
 119
 120 struct nvmet_rdma_port {
 121         struct nvmet_port       *nport;
 122         struct sockaddr_storage addr;
 123         struct rdma_cm_id       *cm_id;
 124         struct delayed_work     repair_work;
 125 };
 126
 127 struct nvmet_rdma_srq {
 128         struct ib_srq            *srq;
 129         struct nvmet_rdma_cmd    *cmds;
 130         struct nvmet_rdma_device *ndev;
 131 };
 132
 133 struct nvmet_rdma_device {
 134         struct ib_device        *device;
 135         struct ib_pd            *pd;
 136         struct nvmet_rdma_srq   **srqs;
 137         int                     srq_count;
 138         size_t                  srq_size;
 139         struct kref             ref;
 140         struct list_head        entry;
 141         int                     inline_data_size;
 142         int                     inline_page_count;
 143 };
 144
 145 static bool nvmet_rdma_use_srq;
 146 module_param_named(use_srq, nvmet_rdma_use_srq, bool, 0444);
 147 MODULE_PARM_DESC(use_srq, "Use shared receive queue.");
 148
 149 static int srq_size_set(const char *val, const struct kernel_param *kp);
 150 static const struct kernel_param_ops srq_size_ops = {
 151         .set = srq_size_set,
 152         .get = param_get_int,
 153 };
 154
 155 static int nvmet_rdma_srq_size = 1024;
 156 module_param_cb(srq_size, &srq_size_ops, &nvmet_rdma_srq_size, 0644);
 157 MODULE_PARM_DESC(srq_size, "set Shared Receive Queue (SRQ) size, should >= 256 (default: 1024)");
 158
 159 static DEFINE_IDA(nvmet_rdma_queue_ida);
 160 static LIST_HEAD(nvmet_rdma_queue_list);
 161 static DEFINE_MUTEX(nvmet_rdma_queue_mutex);
 162
 163 static LIST_HEAD(device_list);
 164 static DEFINE_MUTEX(device_list_mutex);
 165
 166 static bool nvmet_rdma_execute_command(struct nvmet_rdma_rsp *rsp);
 167 static void nvmet_rdma_send_done(struct ib_cq *cq, struct ib_wc *wc);
 168 static void nvmet_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc);
 169 static void nvmet_rdma_read_data_done(struct ib_cq *cq, struct ib_wc *wc);
 170 static void nvmet_rdma_write_data_done(struct ib_cq *cq, struct ib_wc *wc);
 171 static void nvmet_rdma_qp_event(struct ib_event *event, void *priv);
 172 static void nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue);
 173 static void nvmet_rdma_free_rsp(struct nvmet_rdma_device *ndev,
 174                                 struct nvmet_rdma_rsp *r);
 175 static int nvmet_rdma_alloc_rsp(struct nvmet_rdma_device *ndev,
 176                                 struct nvmet_rdma_rsp *r,
 177                                 int tag);
 178
 179 static const struct nvmet_fabrics_ops nvmet_rdma_ops;
 180
 181 static int srq_size_set(const char *val, const struct kernel_param *kp)
 182 {
 183         int n = 0, ret;
 184
 185         ret = kstrtoint(val, 10, &n);
 186         if (ret != 0 || n < 256)
 187                 return -EINVAL;
 188
 189         return param_set_int(val, kp);
 190 }
 191
 192 static int num_pages(int len)
 193 {
 194         return 1 + (((len - 1) & PAGE_MASK) >> PAGE_SHIFT);
 195 }
 196
 197 static inline bool nvmet_rdma_need_data_in(struct nvmet_rdma_rsp *rsp)
 198 {
 199         return nvme_is_write(rsp->req.cmd) &&
 200                 rsp->req.transfer_len &&
 201                 !(rsp->flags & NVMET_RDMA_REQ_INLINE_DATA);
 202 }
 203
 204 static inline bool nvmet_rdma_need_data_out(struct nvmet_rdma_rsp *rsp)
 205 {
 206         return !nvme_is_write(rsp->req.cmd) &&
 207                 rsp->req.transfer_len &&
 208                 !rsp->req.cqe->status &&
 209                 !(rsp->flags & NVMET_RDMA_REQ_INLINE_DATA);
 210 }
 211
 212 static inline struct nvmet_rdma_rsp *
 213 nvmet_rdma_get_rsp(struct nvmet_rdma_queue *queue)
 214 {
 215         struct nvmet_rdma_rsp *rsp = NULL;
 216         int tag;
 217
 218         tag = sbitmap_get(&queue->rsp_tags);
 219         if (tag >= 0)
 220                 rsp = &queue->rsps[tag];
 221
 222         if (unlikely(!rsp)) {
 223                 int ret;
 224
 225                 rsp = kzalloc(sizeof(*rsp), GFP_KERNEL);
 226                 if (unlikely(!rsp))
 227                         return NULL;
 228                 ret = nvmet_rdma_alloc_rsp(queue->dev, rsp,
 229                                 NVMET_RDMA_DISCRETE_RSP_TAG);
 230                 if (unlikely(ret)) {
 231                         kfree(rsp);
 232                         return NULL;
 233                 }
 234         }
 235
 236         return rsp;
 237 }
 238
 239 static inline void
 240 nvmet_rdma_put_rsp(struct nvmet_rdma_rsp *rsp)
 241 {
 242         if (unlikely(rsp->tag == NVMET_RDMA_DISCRETE_RSP_TAG)) {
 243                 nvmet_rdma_free_rsp(rsp->queue->dev, rsp);
 244                 kfree(rsp);
 245                 return;
 246         }
 247
 248         sbitmap_clear_bit(&rsp->queue->rsp_tags, rsp->tag);
 249 }
 250
 251 static void nvmet_rdma_free_inline_pages(struct nvmet_rdma_device *ndev,
 252                                 struct nvmet_rdma_cmd *c)
 253 {
 254         struct scatterlist *sg;
 255         struct ib_sge *sge;
 256         int i;
 257
 258         if (!ndev->inline_data_size)
 259                 return;
 260
 261         sg = c->inline_sg;
 262         sge = &c->sge[1];
 263
 264         for (i = 0; i < ndev->inline_page_count; i++, sg++, sge++) {
 265                 if (sge->length)
 266                         ib_dma_unmap_page(ndev->device, sge->addr,
 267                                         sge->length, DMA_FROM_DEVICE);
 268                 if (sg_page(sg))
 269                         __free_page(sg_page(sg));
 270         }
 271 }
 272
 273 static int nvmet_rdma_alloc_inline_pages(struct nvmet_rdma_device *ndev,
 274                                 struct nvmet_rdma_cmd *c)
 275 {
 276         struct scatterlist *sg;
 277         struct ib_sge *sge;
 278         struct page *pg;
 279         int len;
 280         int i;
 281
 282         if (!ndev->inline_data_size)
 283                 return 0;
 284
 285         sg = c->inline_sg;
 286         sg_init_table(sg, ndev->inline_page_count);
 287         sge = &c->sge[1];
 288         len = ndev->inline_data_size;
 289
 290         for (i = 0; i < ndev->inline_page_count; i++, sg++, sge++) {
 291                 pg = alloc_page(GFP_KERNEL);
 292                 if (!pg)
 293                         goto out_err;
 294                 sg_assign_page(sg, pg);
 295                 sge->addr = ib_dma_map_page(ndev->device,
 296                         pg, 0, PAGE_SIZE, DMA_FROM_DEVICE);
 297                 if (ib_dma_mapping_error(ndev->device, sge->addr))
 298                         goto out_err;
 299                 sge->length = min_t(int, len, PAGE_SIZE);
 300                 sge->lkey = ndev->pd->local_dma_lkey;
 301                 len -= sge->length;
 302         }
 303
 304         return 0;
 305 out_err:
 306         for (; i >= 0; i--, sg--, sge--) {
 307                 if (sge->length)
 308                         ib_dma_unmap_page(ndev->device, sge->addr,
 309                                         sge->length, DMA_FROM_DEVICE);
 310                 if (sg_page(sg))
 311                         __free_page(sg_page(sg));
 312         }
 313         return -ENOMEM;
 314 }
 315
 316 static int nvmet_rdma_alloc_cmd(struct nvmet_rdma_device *ndev,
 317                         struct nvmet_rdma_cmd *c, bool admin)
 318 {
 319         /* NVMe command / RDMA RECV */
 320         c->nvme_cmd = kmalloc(sizeof(*c->nvme_cmd), GFP_KERNEL);
 321         if (!c->nvme_cmd)
 322                 goto out;
 323
 324         c->sge[0].addr = ib_dma_map_single(ndev->device, c->nvme_cmd,
 325                         sizeof(*c->nvme_cmd), DMA_FROM_DEVICE);
 326         if (ib_dma_mapping_error(ndev->device, c->sge[0].addr))
 327                 goto out_free_cmd;
 328
 329         c->sge[0].length = sizeof(*c->nvme_cmd);
 330         c->sge[0].lkey = ndev->pd->local_dma_lkey;
 331
 332         if (!admin && nvmet_rdma_alloc_inline_pages(ndev, c))
 333                 goto out_unmap_cmd;
 334
 335         c->cqe.done = nvmet_rdma_recv_done;
 336
 337         c->wr.wr_cqe = &c->cqe;
 338         c->wr.sg_list = c->sge;
 339         c->wr.num_sge = admin ? 1 : ndev->inline_page_count + 1;
 340
 341         return 0;
 342
 343 out_unmap_cmd:
 344         ib_dma_unmap_single(ndev->device, c->sge[0].addr,
 345                         sizeof(*c->nvme_cmd), DMA_FROM_DEVICE);
 346 out_free_cmd:
 347         kfree(c->nvme_cmd);
 348
 349 out:
 350         return -ENOMEM;
 351 }
 352
 353 static void nvmet_rdma_free_cmd(struct nvmet_rdma_device *ndev,
 354                 struct nvmet_rdma_cmd *c, bool admin)
 355 {
 356         if (!admin)
 357                 nvmet_rdma_free_inline_pages(ndev, c);
 358         ib_dma_unmap_single(ndev->device, c->sge[0].addr,
 359                                 sizeof(*c->nvme_cmd), DMA_FROM_DEVICE);
 360         kfree(c->nvme_cmd);
 361 }
 362
 363 static struct nvmet_rdma_cmd *
 364 nvmet_rdma_alloc_cmds(struct nvmet_rdma_device *ndev,
 365                 int nr_cmds, bool admin)
 366 {
 367         struct nvmet_rdma_cmd *cmds;
 368         int ret = -EINVAL, i;
 369
 370         cmds = kcalloc(nr_cmds, sizeof(struct nvmet_rdma_cmd), GFP_KERNEL);
 371         if (!cmds)
 372                 goto out;
 373
 374         for (i = 0; i < nr_cmds; i++) {
 375                 ret = nvmet_rdma_alloc_cmd(ndev, cmds + i, admin);
 376                 if (ret)
 377                         goto out_free;
 378         }
 379
 380         return cmds;
 381
 382 out_free:
 383         while (--i >= 0)
 384                 nvmet_rdma_free_cmd(ndev, cmds + i, admin);
 385         kfree(cmds);
 386 out:
 387         return ERR_PTR(ret);
 388 }
 389
 390 static void nvmet_rdma_free_cmds(struct nvmet_rdma_device *ndev,
 391                 struct nvmet_rdma_cmd *cmds, int nr_cmds, bool admin)
 392 {
 393         int i;
 394
 395         for (i = 0; i < nr_cmds; i++)
 396                 nvmet_rdma_free_cmd(ndev, cmds + i, admin);
 397         kfree(cmds);
 398 }
 399
 400 static int nvmet_rdma_alloc_rsp(struct nvmet_rdma_device *ndev,
 401                 struct nvmet_rdma_rsp *r, int tag)
 402 {
 403         /* NVMe CQE / RDMA SEND */
 404         r->req.cqe = kmalloc(sizeof(*r->req.cqe), GFP_KERNEL);
 405         if (!r->req.cqe)
 406                 goto out;
 407
 408         r->send_sge.addr = ib_dma_map_single(ndev->device, r->req.cqe,
 409                         sizeof(*r->req.cqe), DMA_TO_DEVICE);
 410         if (ib_dma_mapping_error(ndev->device, r->send_sge.addr))
 411                 goto out_free_rsp;
 412
 413         if (ib_dma_pci_p2p_dma_supported(ndev->device))
 414                 r->req.p2p_client = &ndev->device->dev;
 415         r->send_sge.length = sizeof(*r->req.cqe);
 416         r->send_sge.lkey = ndev->pd->local_dma_lkey;
 417
 418         r->send_cqe.done = nvmet_rdma_send_done;
 419
 420         r->send_wr.wr_cqe = &r->send_cqe;
 421         r->send_wr.sg_list = &r->send_sge;
 422         r->send_wr.num_sge = 1;
 423         r->send_wr.send_flags = IB_SEND_SIGNALED;
 424
 425         /* Data In / RDMA READ */
 426         r->read_cqe.done = nvmet_rdma_read_data_done;
 427         /* Data Out / RDMA WRITE */
 428         r->write_cqe.done = nvmet_rdma_write_data_done;
 429         r->tag = tag;
 430
 431         return 0;
 432
 433 out_free_rsp:
 434         kfree(r->req.cqe);
 435 out:
 436         return -ENOMEM;
 437 }
 438
 439 static void nvmet_rdma_free_rsp(struct nvmet_rdma_device *ndev,
 440                 struct nvmet_rdma_rsp *r)
 441 {
 442         ib_dma_unmap_single(ndev->device, r->send_sge.addr,
 443                                 sizeof(*r->req.cqe), DMA_TO_DEVICE);
 444         kfree(r->req.cqe);
 445 }
 446
 447 static int
 448 nvmet_rdma_alloc_rsps(struct nvmet_rdma_queue *queue)
 449 {
 450         struct nvmet_rdma_device *ndev = queue->dev;
 451         int nr_rsps = queue->recv_queue_size * 2;
 452         int ret = -ENOMEM, i;
 453
 454         if (sbitmap_init_node(&queue->rsp_tags, nr_rsps, -1, GFP_KERNEL,
 455                         NUMA_NO_NODE, false, true))
 456                 goto out;
 457
 458         queue->rsps = kcalloc(nr_rsps, sizeof(struct nvmet_rdma_rsp),
 459                         GFP_KERNEL);
 460         if (!queue->rsps)
 461                 goto out_free_sbitmap;
 462
 463         for (i = 0; i < nr_rsps; i++) {
 464                 struct nvmet_rdma_rsp *rsp = &queue->rsps[i];
 465
 466                 ret = nvmet_rdma_alloc_rsp(ndev, rsp, i);
 467                 if (ret)
 468                         goto out_free;
 469         }
 470
 471         return 0;
 472
 473 out_free:
 474         while (--i >= 0)
 475                 nvmet_rdma_free_rsp(ndev, &queue->rsps[i]);
 476         kfree(queue->rsps);
 477 out_free_sbitmap:
 478         sbitmap_free(&queue->rsp_tags);
 479 out:
 480         return ret;
 481 }
 482
 483 static void nvmet_rdma_free_rsps(struct nvmet_rdma_queue *queue)
 484 {
 485         struct nvmet_rdma_device *ndev = queue->dev;
 486         int i, nr_rsps = queue->recv_queue_size * 2;
 487
 488         for (i = 0; i < nr_rsps; i++)
 489                 nvmet_rdma_free_rsp(ndev, &queue->rsps[i]);
 490         kfree(queue->rsps);
 491         sbitmap_free(&queue->rsp_tags);
 492 }
 493
 494 static int nvmet_rdma_post_recv(struct nvmet_rdma_device *ndev,
 495                 struct nvmet_rdma_cmd *cmd)
 496 {
 497         int ret;
 498
 499         ib_dma_sync_single_for_device(ndev->device,
 500                 cmd->sge[0].addr, cmd->sge[0].length,
 501                 DMA_FROM_DEVICE);
 502
 503         if (cmd->nsrq)
 504                 ret = ib_post_srq_recv(cmd->nsrq->srq, &cmd->wr, NULL);
 505         else
 506                 ret = ib_post_recv(cmd->queue->qp, &cmd->wr, NULL);
 507
 508         if (unlikely(ret))
 509                 pr_err("post_recv cmd failed\n");
 510
 511         return ret;
 512 }
 513
 514 static void nvmet_rdma_process_wr_wait_list(struct nvmet_rdma_queue *queue)
 515 {
 516         spin_lock(&queue->rsp_wr_wait_lock);
 517         while (!list_empty(&queue->rsp_wr_wait_list)) {
 518                 struct nvmet_rdma_rsp *rsp;
 519                 bool ret;
 520
 521                 rsp = list_entry(queue->rsp_wr_wait_list.next,
 522                                 struct nvmet_rdma_rsp, wait_list);
 523                 list_del(&rsp->wait_list);
 524
 525                 spin_unlock(&queue->rsp_wr_wait_lock);
 526                 ret = nvmet_rdma_execute_command(rsp);
 527                 spin_lock(&queue->rsp_wr_wait_lock);
 528
 529                 if (!ret) {
 530                         list_add(&rsp->wait_list, &queue->rsp_wr_wait_list);
 531                         break;
 532                 }
 533         }
 534         spin_unlock(&queue->rsp_wr_wait_lock);
 535 }
 536
 537 static u16 nvmet_rdma_check_pi_status(struct ib_mr *sig_mr)
 538 {
 539         struct ib_mr_status mr_status;
 540         int ret;
 541         u16 status = 0;
 542
 543         ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status);
 544         if (ret) {
 545                 pr_err("ib_check_mr_status failed, ret %d\n", ret);
 546                 return NVME_SC_INVALID_PI;
 547         }
 548
 549         if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) {
 550                 switch (mr_status.sig_err.err_type) {
 551                 case IB_SIG_BAD_GUARD:
 552                         status = NVME_SC_GUARD_CHECK;
 553                         break;
 554                 case IB_SIG_BAD_REFTAG:
 555                         status = NVME_SC_REFTAG_CHECK;
 556                         break;
 557                 case IB_SIG_BAD_APPTAG:
 558                         status = NVME_SC_APPTAG_CHECK;
 559                         break;
 560                 }
 561                 pr_err("PI error found type %d expected 0x%x vs actual 0x%x\n",
 562                        mr_status.sig_err.err_type,
 563                        mr_status.sig_err.expected,
 564                        mr_status.sig_err.actual);
 565         }
 566
 567         return status;
 568 }
 569
 570 static void nvmet_rdma_set_sig_domain(struct blk_integrity *bi,
 571                 struct nvme_command *cmd, struct ib_sig_domain *domain,
 572                 u16 control, u8 pi_type)
 573 {
 574         domain->sig_type = IB_SIG_TYPE_T10_DIF;
 575         domain->sig.dif.bg_type = IB_T10DIF_CRC;
 576         domain->sig.dif.pi_interval = 1 << bi->interval_exp;
 577         domain->sig.dif.ref_tag = le32_to_cpu(cmd->rw.reftag);
 578         if (control & NVME_RW_PRINFO_PRCHK_REF)
 579                 domain->sig.dif.ref_remap = true;
 580
 581         domain->sig.dif.app_tag = le16_to_cpu(cmd->rw.lbat);
 582         domain->sig.dif.apptag_check_mask = le16_to_cpu(cmd->rw.lbatm);
 583         domain->sig.dif.app_escape = true;
 584         if (pi_type == NVME_NS_DPS_PI_TYPE3)
 585                 domain->sig.dif.ref_escape = true;
 586 }
 587
 588 static void nvmet_rdma_set_sig_attrs(struct nvmet_req *req,
 589                                      struct ib_sig_attrs *sig_attrs)
 590 {
 591         struct nvme_command *cmd = req->cmd;
 592         u16 control = le16_to_cpu(cmd->rw.control);
 593         u8 pi_type = req->ns->pi_type;
 594         struct blk_integrity *bi;
 595
 596         bi = bdev_get_integrity(req->ns->bdev);
 597
 598         memset(sig_attrs, 0, sizeof(*sig_attrs));
 599
 600         if (control & NVME_RW_PRINFO_PRACT) {
 601                 /* for WRITE_INSERT/READ_STRIP no wire domain */
 602                 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE;
 603                 nvmet_rdma_set_sig_domain(bi, cmd, &sig_attrs->mem, control,
 604                                           pi_type);
 605                 /* Clear the PRACT bit since HCA will generate/verify the PI */
 606                 control &= ~NVME_RW_PRINFO_PRACT;
 607                 cmd->rw.control = cpu_to_le16(control);
 608                 /* PI is added by the HW */
 609                 req->transfer_len += req->metadata_len;
 610         } else {
 611                 /* for WRITE_PASS/READ_PASS both wire/memory domains exist */
 612                 nvmet_rdma_set_sig_domain(bi, cmd, &sig_attrs->wire, control,
 613                                           pi_type);
 614                 nvmet_rdma_set_sig_domain(bi, cmd, &sig_attrs->mem, control,
 615                                           pi_type);
 616         }
 617
 618         if (control & NVME_RW_PRINFO_PRCHK_REF)
 619                 sig_attrs->check_mask |= IB_SIG_CHECK_REFTAG;
 620         if (control & NVME_RW_PRINFO_PRCHK_GUARD)
 621                 sig_attrs->check_mask |= IB_SIG_CHECK_GUARD;
 622         if (control & NVME_RW_PRINFO_PRCHK_APP)
 623                 sig_attrs->check_mask |= IB_SIG_CHECK_APPTAG;
 624 }
 625
 626 static int nvmet_rdma_rw_ctx_init(struct nvmet_rdma_rsp *rsp, u64 addr, u32 key,
 627                                   struct ib_sig_attrs *sig_attrs)
 628 {
 629         struct rdma_cm_id *cm_id = rsp->queue->cm_id;
 630         struct nvmet_req *req = &rsp->req;
 631         int ret;
 632
 633         if (req->metadata_len)
 634                 ret = rdma_rw_ctx_signature_init(&rsp->rw, cm_id->qp,
 635                         cm_id->port_num, req->sg, req->sg_cnt,
 636                         req->metadata_sg, req->metadata_sg_cnt, sig_attrs,
 637                         addr, key, nvmet_data_dir(req));
 638         else
 639                 ret = rdma_rw_ctx_init(&rsp->rw, cm_id->qp, cm_id->port_num,
 640                                        req->sg, req->sg_cnt, 0, addr, key,
 641                                        nvmet_data_dir(req));
 642
 643         return ret;
 644 }
 645
 646 static void nvmet_rdma_rw_ctx_destroy(struct nvmet_rdma_rsp *rsp)
 647 {
 648         struct rdma_cm_id *cm_id = rsp->queue->cm_id;
 649         struct nvmet_req *req = &rsp->req;
 650
 651         if (req->metadata_len)
 652                 rdma_rw_ctx_destroy_signature(&rsp->rw, cm_id->qp,
 653                         cm_id->port_num, req->sg, req->sg_cnt,
 654                         req->metadata_sg, req->metadata_sg_cnt,
 655                         nvmet_data_dir(req));
 656         else
 657                 rdma_rw_ctx_destroy(&rsp->rw, cm_id->qp, cm_id->port_num,
 658                                     req->sg, req->sg_cnt, nvmet_data_dir(req));
 659 }
 660
 661 static void nvmet_rdma_release_rsp(struct nvmet_rdma_rsp *rsp)
 662 {
 663         struct nvmet_rdma_queue *queue = rsp->queue;
 664
 665         atomic_add(1 + rsp->n_rdma, &queue->sq_wr_avail);
 666
 667         if (rsp->n_rdma)
 668                 nvmet_rdma_rw_ctx_destroy(rsp);
 669
 670         if (rsp->req.sg != rsp->cmd->inline_sg)
 671                 nvmet_req_free_sgls(&rsp->req);
 672
 673         if (unlikely(!list_empty_careful(&queue->rsp_wr_wait_list)))
 674                 nvmet_rdma_process_wr_wait_list(queue);
 675
 676         nvmet_rdma_put_rsp(rsp);
 677 }
 678
 679 static void nvmet_rdma_error_comp(struct nvmet_rdma_queue *queue)
 680 {
 681         if (queue->nvme_sq.ctrl) {
 682                 nvmet_ctrl_fatal_error(queue->nvme_sq.ctrl);
 683         } else {
 684                 /*
 685                  * we didn't setup the controller yet in case
 686                  * of admin connect error, just disconnect and
 687                  * cleanup the queue
 688                  */
 689                 nvmet_rdma_queue_disconnect(queue);
 690         }
 691 }
 692
 693 static void nvmet_rdma_send_done(struct ib_cq *cq, struct ib_wc *wc)
 694 {
 695         struct nvmet_rdma_rsp *rsp =
 696                 container_of(wc->wr_cqe, struct nvmet_rdma_rsp, send_cqe);
 697         struct nvmet_rdma_queue *queue = wc->qp->qp_context;
 698
 699         nvmet_rdma_release_rsp(rsp);
 700
 701         if (unlikely(wc->status != IB_WC_SUCCESS &&
 702                      wc->status != IB_WC_WR_FLUSH_ERR)) {
 703                 pr_err("SEND for CQE 0x%p failed with status %s (%d).\n",
 704                         wc->wr_cqe, ib_wc_status_msg(wc->status), wc->status);
 705                 nvmet_rdma_error_comp(queue);
 706         }
 707 }
 708
 709 static void nvmet_rdma_queue_response(struct nvmet_req *req)
 710 {
 711         struct nvmet_rdma_rsp *rsp =
 712                 container_of(req, struct nvmet_rdma_rsp, req);
 713         struct rdma_cm_id *cm_id = rsp->queue->cm_id;
 714         struct ib_send_wr *first_wr;
 715
 716         if (rsp->invalidate_rkey) {
 717                 rsp->send_wr.opcode = IB_WR_SEND_WITH_INV;
 718                 rsp->send_wr.ex.invalidate_rkey = rsp->invalidate_rkey;
 719         } else {
 720                 rsp->send_wr.opcode = IB_WR_SEND;
 721         }
 722
 723         if (nvmet_rdma_need_data_out(rsp)) {
 724                 if (rsp->req.metadata_len)
 725                         first_wr = rdma_rw_ctx_wrs(&rsp->rw, cm_id->qp,
 726                                         cm_id->port_num, &rsp->write_cqe, NULL);
 727                 else
 728                         first_wr = rdma_rw_ctx_wrs(&rsp->rw, cm_id->qp,
 729                                         cm_id->port_num, NULL, &rsp->send_wr);
 730         } else {
 731                 first_wr = &rsp->send_wr;
 732         }
 733
 734         nvmet_rdma_post_recv(rsp->queue->dev, rsp->cmd);
 735
 736         ib_dma_sync_single_for_device(rsp->queue->dev->device,
 737                 rsp->send_sge.addr, rsp->send_sge.length,
 738                 DMA_TO_DEVICE);
 739
 740         if (unlikely(ib_post_send(cm_id->qp, first_wr, NULL))) {
 741                 pr_err("sending cmd response failed\n");
 742                 nvmet_rdma_release_rsp(rsp);
 743         }
 744 }
 745
 746 static void nvmet_rdma_read_data_done(struct ib_cq *cq, struct ib_wc *wc)
 747 {
 748         struct nvmet_rdma_rsp *rsp =
 749                 container_of(wc->wr_cqe, struct nvmet_rdma_rsp, read_cqe);
 750         struct nvmet_rdma_queue *queue = wc->qp->qp_context;
 751         u16 status = 0;
 752
 753         WARN_ON(rsp->n_rdma <= 0);
 754         atomic_add(rsp->n_rdma, &queue->sq_wr_avail);
 755         rsp->n_rdma = 0;
 756
 757         if (unlikely(wc->status != IB_WC_SUCCESS)) {
 758                 nvmet_rdma_rw_ctx_destroy(rsp);
 759                 nvmet_req_uninit(&rsp->req);
 760                 nvmet_rdma_release_rsp(rsp);
 761                 if (wc->status != IB_WC_WR_FLUSH_ERR) {
 762                         pr_info("RDMA READ for CQE 0x%p failed with status %s (%d).\n",
 763                                 wc->wr_cqe, ib_wc_status_msg(wc->status), wc->status);
 764                         nvmet_rdma_error_comp(queue);
 765                 }
 766                 return;
 767         }
 768
 769         if (rsp->req.metadata_len)
 770                 status = nvmet_rdma_check_pi_status(rsp->rw.reg->mr);
 771         nvmet_rdma_rw_ctx_destroy(rsp);
 772
 773         if (unlikely(status))
 774                 nvmet_req_complete(&rsp->req, status);
 775         else
 776                 rsp->req.execute(&rsp->req);
 777 }
 778
 779 static void nvmet_rdma_write_data_done(struct ib_cq *cq, struct ib_wc *wc)
 780 {
 781         struct nvmet_rdma_rsp *rsp =
 782                 container_of(wc->wr_cqe, struct nvmet_rdma_rsp, write_cqe);
 783         struct nvmet_rdma_queue *queue = wc->qp->qp_context;
 784         struct rdma_cm_id *cm_id = rsp->queue->cm_id;
 785         u16 status;
 786
 787         if (!IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY))
 788                 return;
 789
 790         WARN_ON(rsp->n_rdma <= 0);
 791         atomic_add(rsp->n_rdma, &queue->sq_wr_avail);
 792         rsp->n_rdma = 0;
 793
 794         if (unlikely(wc->status != IB_WC_SUCCESS)) {
 795                 nvmet_rdma_rw_ctx_destroy(rsp);
 796                 nvmet_req_uninit(&rsp->req);
 797                 nvmet_rdma_release_rsp(rsp);
 798                 if (wc->status != IB_WC_WR_FLUSH_ERR) {
 799                         pr_info("RDMA WRITE for CQE failed with status %s (%d).\n",
 800                                 ib_wc_status_msg(wc->status), wc->status);
 801                         nvmet_rdma_error_comp(queue);
 802                 }
 803                 return;
 804         }
 805
 806         /*
 807          * Upon RDMA completion check the signature status
 808          * - if succeeded send good NVMe response
 809          * - if failed send bad NVMe response with appropriate error
 810          */
 811         status = nvmet_rdma_check_pi_status(rsp->rw.reg->mr);
 812         if (unlikely(status))
 813                 rsp->req.cqe->status = cpu_to_le16(status << 1);
 814         nvmet_rdma_rw_ctx_destroy(rsp);
 815
 816         if (unlikely(ib_post_send(cm_id->qp, &rsp->send_wr, NULL))) {
 817                 pr_err("sending cmd response failed\n");
 818                 nvmet_rdma_release_rsp(rsp);
 819         }
 820 }
 821
 822 static void nvmet_rdma_use_inline_sg(struct nvmet_rdma_rsp *rsp, u32 len,
 823                 u64 off)
 824 {
 825         int sg_count = num_pages(len);
 826         struct scatterlist *sg;
 827         int i;
 828
 829         sg = rsp->cmd->inline_sg;
 830         for (i = 0; i < sg_count; i++, sg++) {
 831                 if (i < sg_count - 1)
 832                         sg_unmark_end(sg);
 833                 else
 834                         sg_mark_end(sg);
 835                 sg->offset = off;
 836                 sg->length = min_t(int, len, PAGE_SIZE - off);
 837                 len -= sg->length;
 838                 if (!i)
 839                         off = 0;
 840         }
 841
 842         rsp->req.sg = rsp->cmd->inline_sg;
 843         rsp->req.sg_cnt = sg_count;
 844 }
 845
 846 static u16 nvmet_rdma_map_sgl_inline(struct nvmet_rdma_rsp *rsp)
 847 {
 848         struct nvme_sgl_desc *sgl = &rsp->req.cmd->common.dptr.sgl;
 849         u64 off = le64_to_cpu(sgl->addr);
 850         u32 len = le32_to_cpu(sgl->length);
 851
 852         if (!nvme_is_write(rsp->req.cmd)) {
 853                 rsp->req.error_loc =
 854                         offsetof(struct nvme_common_command, opcode);
 855                 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
 856         }
 857
 858         if (off + len > rsp->queue->dev->inline_data_size) {
 859                 pr_err("invalid inline data offset!\n");
 860                 return NVME_SC_SGL_INVALID_OFFSET | NVME_STATUS_DNR;
 861         }
 862
 863         /* no data command? */
 864         if (!len)
 865                 return 0;
 866
 867         nvmet_rdma_use_inline_sg(rsp, len, off);
 868         rsp->flags |= NVMET_RDMA_REQ_INLINE_DATA;
 869         rsp->req.transfer_len += len;
 870         return 0;
 871 }
 872
 873 static u16 nvmet_rdma_map_sgl_keyed(struct nvmet_rdma_rsp *rsp,
 874                 struct nvme_keyed_sgl_desc *sgl, bool invalidate)
 875 {
 876         u64 addr = le64_to_cpu(sgl->addr);
 877         u32 key = get_unaligned_le32(sgl->key);
 878         struct ib_sig_attrs sig_attrs;
 879         int ret;
 880
 881         rsp->req.transfer_len = get_unaligned_le24(sgl->length);
 882
 883         /* no data command? */
 884         if (!rsp->req.transfer_len)
 885                 return 0;
 886
 887         if (rsp->req.metadata_len)
 888                 nvmet_rdma_set_sig_attrs(&rsp->req, &sig_attrs);
 889
 890         ret = nvmet_req_alloc_sgls(&rsp->req);
 891         if (unlikely(ret < 0))
 892                 goto error_out;
 893
 894         ret = nvmet_rdma_rw_ctx_init(rsp, addr, key, &sig_attrs);
 895         if (unlikely(ret < 0))
 896                 goto error_out;
 897         rsp->n_rdma += ret;
 898
 899         if (invalidate)
 900                 rsp->invalidate_rkey = key;
 901
 902         return 0;
 903
 904 error_out:
 905         rsp->req.transfer_len = 0;
 906         return NVME_SC_INTERNAL;
 907 }
 908
 909 static u16 nvmet_rdma_map_sgl(struct nvmet_rdma_rsp *rsp)
 910 {
 911         struct nvme_keyed_sgl_desc *sgl = &rsp->req.cmd->common.dptr.ksgl;
 912
 913         switch (sgl->type >> 4) {
 914         case NVME_SGL_FMT_DATA_DESC:
 915                 switch (sgl->type & 0xf) {
 916                 case NVME_SGL_FMT_OFFSET:
 917                         return nvmet_rdma_map_sgl_inline(rsp);
 918                 default:
 919                         pr_err("invalid SGL subtype: %#x\n", sgl->type);
 920                         rsp->req.error_loc =
 921                                 offsetof(struct nvme_common_command, dptr);
 922                         return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
 923                 }
 924         case NVME_KEY_SGL_FMT_DATA_DESC:
 925                 switch (sgl->type & 0xf) {
 926                 case NVME_SGL_FMT_ADDRESS | NVME_SGL_FMT_INVALIDATE:
 927                         return nvmet_rdma_map_sgl_keyed(rsp, sgl, true);
 928                 case NVME_SGL_FMT_ADDRESS:
 929                         return nvmet_rdma_map_sgl_keyed(rsp, sgl, false);
 930                 default:
 931                         pr_err("invalid SGL subtype: %#x\n", sgl->type);
 932                         rsp->req.error_loc =
 933                                 offsetof(struct nvme_common_command, dptr);
 934                         return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
 935                 }
 936         default:
 937                 pr_err("invalid SGL type: %#x\n", sgl->type);
 938                 rsp->req.error_loc = offsetof(struct nvme_common_command, dptr);
 939                 return NVME_SC_SGL_INVALID_TYPE | NVME_STATUS_DNR;
 940         }
 941 }
 942
 943 static bool nvmet_rdma_execute_command(struct nvmet_rdma_rsp *rsp)
 944 {
 945         struct nvmet_rdma_queue *queue = rsp->queue;
 946
 947         if (unlikely(atomic_sub_return(1 + rsp->n_rdma,
 948                         &queue->sq_wr_avail) < 0)) {
 949                 pr_debug("IB send queue full (needed %d): queue %u cntlid %u\n",
 950                                 1 + rsp->n_rdma, queue->idx,
 951                                 queue->nvme_sq.ctrl->cntlid);
 952                 atomic_add(1 + rsp->n_rdma, &queue->sq_wr_avail);
 953                 return false;
 954         }
 955
 956         if (nvmet_rdma_need_data_in(rsp)) {
 957                 if (rdma_rw_ctx_post(&rsp->rw, queue->qp,
 958                                 queue->cm_id->port_num, &rsp->read_cqe, NULL))
 959                         nvmet_req_complete(&rsp->req, NVME_SC_DATA_XFER_ERROR);
 960         } else {
 961                 rsp->req.execute(&rsp->req);
 962         }
 963
 964         return true;
 965 }
 966
 967 static void nvmet_rdma_handle_command(struct nvmet_rdma_queue *queue,
 968                 struct nvmet_rdma_rsp *cmd)
 969 {
 970         u16 status;
 971
 972         ib_dma_sync_single_for_cpu(queue->dev->device,
 973                 cmd->cmd->sge[0].addr, cmd->cmd->sge[0].length,
 974                 DMA_FROM_DEVICE);
 975         ib_dma_sync_single_for_cpu(queue->dev->device,
 976                 cmd->send_sge.addr, cmd->send_sge.length,
 977                 DMA_TO_DEVICE);
 978
 979         if (!nvmet_req_init(&cmd->req, &queue->nvme_cq,
 980                         &queue->nvme_sq, &nvmet_rdma_ops))
 981                 return;
 982
 983         status = nvmet_rdma_map_sgl(cmd);
 984         if (status)
 985                 goto out_err;
 986
 987         if (unlikely(!nvmet_rdma_execute_command(cmd))) {
 988                 spin_lock(&queue->rsp_wr_wait_lock);
 989                 list_add_tail(&cmd->wait_list, &queue->rsp_wr_wait_list);
 990                 spin_unlock(&queue->rsp_wr_wait_lock);
 991         }
 992
 993         return;
 994
 995 out_err:
 996         nvmet_req_complete(&cmd->req, status);
 997 }
 998
 999 static void nvmet_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc)
1000 {
1001         struct nvmet_rdma_cmd *cmd =
1002                 container_of(wc->wr_cqe, struct nvmet_rdma_cmd, cqe);
1003         struct nvmet_rdma_queue *queue = wc->qp->qp_context;
1004         struct nvmet_rdma_rsp *rsp;
1005
1006         if (unlikely(wc->status != IB_WC_SUCCESS)) {
1007                 if (wc->status != IB_WC_WR_FLUSH_ERR) {
1008                         pr_err("RECV for CQE 0x%p failed with status %s (%d)\n",
1009                                 wc->wr_cqe, ib_wc_status_msg(wc->status),
1010                                 wc->status);
1011                         nvmet_rdma_error_comp(queue);
1012                 }
1013                 return;
1014         }
1015
1016         if (unlikely(wc->byte_len < sizeof(struct nvme_command))) {
1017                 pr_err("Ctrl Fatal Error: capsule size less than 64 bytes\n");
1018                 nvmet_rdma_error_comp(queue);
1019                 return;
1020         }
1021
1022         cmd->queue = queue;
1023         rsp = nvmet_rdma_get_rsp(queue);
1024         if (unlikely(!rsp)) {
1025                 /*
1026                  * we get here only under memory pressure,
1027                  * silently drop and have the host retry
1028                  * as we can't even fail it.
1029                  */
1030                 nvmet_rdma_post_recv(queue->dev, cmd);
1031                 return;
1032         }
1033         rsp->queue = queue;
1034         rsp->cmd = cmd;
1035         rsp->flags = 0;
1036         rsp->req.cmd = cmd->nvme_cmd;
1037         rsp->req.port = queue->port;
1038         rsp->n_rdma = 0;
1039         rsp->invalidate_rkey = 0;
1040
1041         if (unlikely(queue->state != NVMET_RDMA_Q_LIVE)) {
1042                 unsigned long flags;
1043
1044                 spin_lock_irqsave(&queue->state_lock, flags);
1045                 if (queue->state == NVMET_RDMA_Q_CONNECTING)
1046                         list_add_tail(&rsp->wait_list, &queue->rsp_wait_list);
1047                 else
1048                         nvmet_rdma_put_rsp(rsp);
1049                 spin_unlock_irqrestore(&queue->state_lock, flags);
1050                 return;
1051         }
1052
1053         nvmet_rdma_handle_command(queue, rsp);
1054 }
1055
1056 static void nvmet_rdma_destroy_srq(struct nvmet_rdma_srq *nsrq)
1057 {
1058         nvmet_rdma_free_cmds(nsrq->ndev, nsrq->cmds, nsrq->ndev->srq_size,
1059                              false);
1060         ib_destroy_srq(nsrq->srq);
1061
1062         kfree(nsrq);
1063 }
1064
1065 static void nvmet_rdma_destroy_srqs(struct nvmet_rdma_device *ndev)
1066 {
1067         int i;
1068
1069         if (!ndev->srqs)
1070                 return;
1071
1072         for (i = 0; i < ndev->srq_count; i++)
1073                 nvmet_rdma_destroy_srq(ndev->srqs[i]);
1074
1075         kfree(ndev->srqs);
1076 }
1077
1078 static struct nvmet_rdma_srq *
1079 nvmet_rdma_init_srq(struct nvmet_rdma_device *ndev)
1080 {
1081         struct ib_srq_init_attr srq_attr = { NULL, };
1082         size_t srq_size = ndev->srq_size;
1083         struct nvmet_rdma_srq *nsrq;
1084         struct ib_srq *srq;
1085         int ret, i;
1086
1087         nsrq = kzalloc(sizeof(*nsrq), GFP_KERNEL);
1088         if (!nsrq)
1089                 return ERR_PTR(-ENOMEM);
1090
1091         srq_attr.attr.max_wr = srq_size;
1092         srq_attr.attr.max_sge = 1 + ndev->inline_page_count;
1093         srq_attr.attr.srq_limit = 0;
1094         srq_attr.srq_type = IB_SRQT_BASIC;
1095         srq = ib_create_srq(ndev->pd, &srq_attr);
1096         if (IS_ERR(srq)) {
1097                 ret = PTR_ERR(srq);
1098                 goto out_free;
1099         }
1100
1101         nsrq->cmds = nvmet_rdma_alloc_cmds(ndev, srq_size, false);
1102         if (IS_ERR(nsrq->cmds)) {
1103                 ret = PTR_ERR(nsrq->cmds);
1104                 goto out_destroy_srq;
1105         }
1106
1107         nsrq->srq = srq;
1108         nsrq->ndev = ndev;
1109
1110         for (i = 0; i < srq_size; i++) {
1111                 nsrq->cmds[i].nsrq = nsrq;
1112                 ret = nvmet_rdma_post_recv(ndev, &nsrq->cmds[i]);
1113                 if (ret)
1114                         goto out_free_cmds;
1115         }
1116
1117         return nsrq;
1118
1119 out_free_cmds:
1120         nvmet_rdma_free_cmds(ndev, nsrq->cmds, srq_size, false);
1121 out_destroy_srq:
1122         ib_destroy_srq(srq);
1123 out_free:
1124         kfree(nsrq);
1125         return ERR_PTR(ret);
1126 }
1127
1128 static int nvmet_rdma_init_srqs(struct nvmet_rdma_device *ndev)
1129 {
1130         int i, ret;
1131
1132         if (!ndev->device->attrs.max_srq_wr || !ndev->device->attrs.max_srq) {
1133                 /*
1134                  * If SRQs aren't supported we just go ahead and use normal
1135                  * non-shared receive queues.
1136                  */
1137                 pr_info("SRQ requested but not supported.\n");
1138                 return 0;
1139         }
1140
1141         ndev->srq_size = min(ndev->device->attrs.max_srq_wr,
1142                              nvmet_rdma_srq_size);
1143         ndev->srq_count = min(ndev->device->num_comp_vectors,
1144                               ndev->device->attrs.max_srq);
1145
1146         ndev->srqs = kcalloc(ndev->srq_count, sizeof(*ndev->srqs), GFP_KERNEL);
1147         if (!ndev->srqs)
1148                 return -ENOMEM;
1149
1150         for (i = 0; i < ndev->srq_count; i++) {
1151                 ndev->srqs[i] = nvmet_rdma_init_srq(ndev);
1152                 if (IS_ERR(ndev->srqs[i])) {
1153                         ret = PTR_ERR(ndev->srqs[i]);
1154                         goto err_srq;
1155                 }
1156         }
1157
1158         return 0;
1159
1160 err_srq:
1161         while (--i >= 0)
1162                 nvmet_rdma_destroy_srq(ndev->srqs[i]);
1163         kfree(ndev->srqs);
1164         return ret;
1165 }
1166
1167 static void nvmet_rdma_free_dev(struct kref *ref)
1168 {
1169         struct nvmet_rdma_device *ndev =
1170                 container_of(ref, struct nvmet_rdma_device, ref);
1171
1172         mutex_lock(&device_list_mutex);
1173         list_del(&ndev->entry);
1174         mutex_unlock(&device_list_mutex);
1175
1176         nvmet_rdma_destroy_srqs(ndev);
1177         ib_dealloc_pd(ndev->pd);
1178
1179         kfree(ndev);
1180 }
1181
1182 static struct nvmet_rdma_device *
1183 nvmet_rdma_find_get_device(struct rdma_cm_id *cm_id)
1184 {
1185         struct nvmet_rdma_port *port = cm_id->context;
1186         struct nvmet_port *nport = port->nport;
1187         struct nvmet_rdma_device *ndev;
1188         int inline_page_count;
1189         int inline_sge_count;
1190         int ret;
1191
1192         mutex_lock(&device_list_mutex);
1193         list_for_each_entry(ndev, &device_list, entry) {
1194                 if (ndev->device->node_guid == cm_id->device->node_guid &&
1195                     kref_get_unless_zero(&ndev->ref))
1196                         goto out_unlock;
1197         }
1198
1199         ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
1200         if (!ndev)
1201                 goto out_err;
1202
1203         inline_page_count = num_pages(nport->inline_data_size);
1204         inline_sge_count = max(cm_id->device->attrs.max_sge_rd,
1205                                 cm_id->device->attrs.max_recv_sge) - 1;
1206         if (inline_page_count > inline_sge_count) {
1207                 pr_warn("inline_data_size %d cannot be supported by device %s. Reducing to %lu.\n",
1208                         nport->inline_data_size, cm_id->device->name,
1209                         inline_sge_count * PAGE_SIZE);
1210                 nport->inline_data_size = inline_sge_count * PAGE_SIZE;
1211                 inline_page_count = inline_sge_count;
1212         }
1213         ndev->inline_data_size = nport->inline_data_size;
1214         ndev->inline_page_count = inline_page_count;
1215
1216         if (nport->pi_enable && !(cm_id->device->attrs.kernel_cap_flags &
1217                                   IBK_INTEGRITY_HANDOVER)) {
1218                 pr_warn("T10-PI is not supported by device %s. Disabling it\n",
1219                         cm_id->device->name);
1220                 nport->pi_enable = false;
1221         }
1222
1223         ndev->device = cm_id->device;
1224         kref_init(&ndev->ref);
1225
1226         ndev->pd = ib_alloc_pd(ndev->device, 0);
1227         if (IS_ERR(ndev->pd))
1228                 goto out_free_dev;
1229
1230         if (nvmet_rdma_use_srq) {
1231                 ret = nvmet_rdma_init_srqs(ndev);
1232                 if (ret)
1233                         goto out_free_pd;
1234         }
1235
1236         list_add(&ndev->entry, &device_list);
1237 out_unlock:
1238         mutex_unlock(&device_list_mutex);
1239         pr_debug("added %s.\n", ndev->device->name);
1240         return ndev;
1241
1242 out_free_pd:
1243         ib_dealloc_pd(ndev->pd);
1244 out_free_dev:
1245         kfree(ndev);
1246 out_err:
1247         mutex_unlock(&device_list_mutex);
1248         return NULL;
1249 }
1250
1251 static int nvmet_rdma_create_queue_ib(struct nvmet_rdma_queue *queue)
1252 {
1253         struct ib_qp_init_attr qp_attr = { };
1254         struct nvmet_rdma_device *ndev = queue->dev;
1255         int nr_cqe, ret, i, factor;
1256
1257         /*
1258          * Reserve CQ slots for RECV + RDMA_READ/RDMA_WRITE + RDMA_SEND.
1259          */
1260         nr_cqe = queue->recv_queue_size + 2 * queue->send_queue_size;
1261
1262         queue->cq = ib_cq_pool_get(ndev->device, nr_cqe + 1,
1263                                    queue->comp_vector, IB_POLL_WORKQUEUE);
1264         if (IS_ERR(queue->cq)) {
1265                 ret = PTR_ERR(queue->cq);
1266                 pr_err("failed to create CQ cqe= %d ret= %d\n",
1267                        nr_cqe + 1, ret);
1268                 goto out;
1269         }
1270
1271         qp_attr.qp_context = queue;
1272         qp_attr.event_handler = nvmet_rdma_qp_event;
1273         qp_attr.send_cq = queue->cq;
1274         qp_attr.recv_cq = queue->cq;
1275         qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
1276         qp_attr.qp_type = IB_QPT_RC;
1277         /* +1 for drain */
1278         qp_attr.cap.max_send_wr = queue->send_queue_size + 1;
1279         factor = rdma_rw_mr_factor(ndev->device, queue->cm_id->port_num,
1280                                    1 << NVMET_RDMA_MAX_MDTS);
1281         qp_attr.cap.max_rdma_ctxs = queue->send_queue_size * factor;
1282         qp_attr.cap.max_send_sge = max(ndev->device->attrs.max_sge_rd,
1283                                         ndev->device->attrs.max_send_sge);
1284
1285         if (queue->nsrq) {
1286                 qp_attr.srq = queue->nsrq->srq;
1287         } else {
1288                 /* +1 for drain */
1289                 qp_attr.cap.max_recv_wr = 1 + queue->recv_queue_size;
1290                 qp_attr.cap.max_recv_sge = 1 + ndev->inline_page_count;
1291         }
1292
1293         if (queue->port->pi_enable && queue->host_qid)
1294                 qp_attr.create_flags |= IB_QP_CREATE_INTEGRITY_EN;
1295
1296         ret = rdma_create_qp(queue->cm_id, ndev->pd, &qp_attr);
1297         if (ret) {
1298                 pr_err("failed to create_qp ret= %d\n", ret);
1299                 goto err_destroy_cq;
1300         }
1301         queue->qp = queue->cm_id->qp;
1302
1303         atomic_set(&queue->sq_wr_avail, qp_attr.cap.max_send_wr);
1304
1305         pr_debug("%s: max_cqe= %d max_sge= %d sq_size = %d cm_id= %p\n",
1306                  __func__, queue->cq->cqe, qp_attr.cap.max_send_sge,
1307                  qp_attr.cap.max_send_wr, queue->cm_id);
1308
1309         if (!queue->nsrq) {
1310                 for (i = 0; i < queue->recv_queue_size; i++) {
1311                         queue->cmds[i].queue = queue;
1312                         ret = nvmet_rdma_post_recv(ndev, &queue->cmds[i]);
1313                         if (ret)
1314                                 goto err_destroy_qp;
1315                 }
1316         }
1317
1318 out:
1319         return ret;
1320
1321 err_destroy_qp:
1322         rdma_destroy_qp(queue->cm_id);
1323 err_destroy_cq:
1324         ib_cq_pool_put(queue->cq, nr_cqe + 1);
1325         goto out;
1326 }
1327
1328 static void nvmet_rdma_destroy_queue_ib(struct nvmet_rdma_queue *queue)
1329 {
1330         ib_drain_qp(queue->qp);
1331         if (queue->cm_id)
1332                 rdma_destroy_id(queue->cm_id);
1333         ib_destroy_qp(queue->qp);
1334         ib_cq_pool_put(queue->cq, queue->recv_queue_size + 2 *
1335                        queue->send_queue_size + 1);
1336 }
1337
1338 static void nvmet_rdma_free_queue(struct nvmet_rdma_queue *queue)
1339 {
1340         pr_debug("freeing queue %d\n", queue->idx);
1341
1342         nvmet_sq_destroy(&queue->nvme_sq);
1343
1344         nvmet_rdma_destroy_queue_ib(queue);
1345         if (!queue->nsrq) {
1346                 nvmet_rdma_free_cmds(queue->dev, queue->cmds,
1347                                 queue->recv_queue_size,
1348                                 !queue->host_qid);
1349         }
1350         nvmet_rdma_free_rsps(queue);
1351         ida_free(&nvmet_rdma_queue_ida, queue->idx);
1352         kfree(queue);
1353 }
1354
1355 static void nvmet_rdma_release_queue_work(struct work_struct *w)
1356 {
1357         struct nvmet_rdma_queue *queue =
1358                 container_of(w, struct nvmet_rdma_queue, release_work);
1359         struct nvmet_rdma_device *dev = queue->dev;
1360
1361         nvmet_rdma_free_queue(queue);
1362
1363         kref_put(&dev->ref, nvmet_rdma_free_dev);
1364 }
1365
1366 static int
1367 nvmet_rdma_parse_cm_connect_req(struct rdma_conn_param *conn,
1368                                 struct nvmet_rdma_queue *queue)
1369 {
1370         struct nvme_rdma_cm_req *req;
1371
1372         req = (struct nvme_rdma_cm_req *)conn->private_data;
1373         if (!req || conn->private_data_len == 0)
1374                 return NVME_RDMA_CM_INVALID_LEN;
1375
1376         if (le16_to_cpu(req->recfmt) != NVME_RDMA_CM_FMT_1_0)
1377                 return NVME_RDMA_CM_INVALID_RECFMT;
1378
1379         queue->host_qid = le16_to_cpu(req->qid);
1380
1381         /*
1382          * req->hsqsize corresponds to our recv queue size plus 1
1383          * req->hrqsize corresponds to our send queue size
1384          */
1385         queue->recv_queue_size = le16_to_cpu(req->hsqsize) + 1;
1386         queue->send_queue_size = le16_to_cpu(req->hrqsize);
1387
1388         if (!queue->host_qid && queue->recv_queue_size > NVME_AQ_DEPTH)
1389                 return NVME_RDMA_CM_INVALID_HSQSIZE;
1390
1391         /* XXX: Should we enforce some kind of max for IO queues? */
1392
1393         return 0;
1394 }
1395
1396 static int nvmet_rdma_cm_reject(struct rdma_cm_id *cm_id,
1397                                 enum nvme_rdma_cm_status status)
1398 {
1399         struct nvme_rdma_cm_rej rej;
1400
1401         pr_debug("rejecting connect request: status %d (%s)\n",
1402                  status, nvme_rdma_cm_msg(status));
1403
1404         rej.recfmt = cpu_to_le16(NVME_RDMA_CM_FMT_1_0);
1405         rej.sts = cpu_to_le16(status);
1406
1407         return rdma_reject(cm_id, (void *)&rej, sizeof(rej),
1408                            IB_CM_REJ_CONSUMER_DEFINED);
1409 }
1410
1411 static struct nvmet_rdma_queue *
1412 nvmet_rdma_alloc_queue(struct nvmet_rdma_device *ndev,
1413                 struct rdma_cm_id *cm_id,
1414                 struct rdma_cm_event *event)
1415 {
1416         struct nvmet_rdma_port *port = cm_id->context;
1417         struct nvmet_rdma_queue *queue;
1418         int ret;
1419
1420         queue = kzalloc(sizeof(*queue), GFP_KERNEL);
1421         if (!queue) {
1422                 ret = NVME_RDMA_CM_NO_RSC;
1423                 goto out_reject;
1424         }
1425
1426         ret = nvmet_sq_init(&queue->nvme_sq);
1427         if (ret) {
1428                 ret = NVME_RDMA_CM_NO_RSC;
1429                 goto out_free_queue;
1430         }
1431
1432         ret = nvmet_rdma_parse_cm_connect_req(&event->param.conn, queue);
1433         if (ret)
1434                 goto out_destroy_sq;
1435
1436         /*
1437          * Schedules the actual release because calling rdma_destroy_id from
1438          * inside a CM callback would trigger a deadlock. (great API design..)
1439          */
1440         INIT_WORK(&queue->release_work, nvmet_rdma_release_queue_work);
1441         queue->dev = ndev;
1442         queue->cm_id = cm_id;
1443         queue->port = port->nport;
1444
1445         spin_lock_init(&queue->state_lock);
1446         queue->state = NVMET_RDMA_Q_CONNECTING;
1447         INIT_LIST_HEAD(&queue->rsp_wait_list);
1448         INIT_LIST_HEAD(&queue->rsp_wr_wait_list);
1449         spin_lock_init(&queue->rsp_wr_wait_lock);
1450         INIT_LIST_HEAD(&queue->queue_list);
1451
1452         queue->idx = ida_alloc(&nvmet_rdma_queue_ida, GFP_KERNEL);
1453         if (queue->idx < 0) {
1454                 ret = NVME_RDMA_CM_NO_RSC;
1455                 goto out_destroy_sq;
1456         }
1457
1458         /*
1459          * Spread the io queues across completion vectors,
1460          * but still keep all admin queues on vector 0.
1461          */
1462         queue->comp_vector = !queue->host_qid ? 0 :
1463                 queue->idx % ndev->device->num_comp_vectors;
1464
1465
1466         ret = nvmet_rdma_alloc_rsps(queue);
1467         if (ret) {
1468                 ret = NVME_RDMA_CM_NO_RSC;
1469                 goto out_ida_remove;
1470         }
1471
1472         if (ndev->srqs) {
1473                 queue->nsrq = ndev->srqs[queue->comp_vector % ndev->srq_count];
1474         } else {
1475                 queue->cmds = nvmet_rdma_alloc_cmds(ndev,
1476                                 queue->recv_queue_size,
1477                                 !queue->host_qid);
1478                 if (IS_ERR(queue->cmds)) {
1479                         ret = NVME_RDMA_CM_NO_RSC;
1480                         goto out_free_responses;
1481                 }
1482         }
1483
1484         ret = nvmet_rdma_create_queue_ib(queue);
1485         if (ret) {
1486                 pr_err("%s: creating RDMA queue failed (%d).\n",
1487                         __func__, ret);
1488                 ret = NVME_RDMA_CM_NO_RSC;
1489                 goto out_free_cmds;
1490         }
1491
1492         return queue;
1493
1494 out_free_cmds:
1495         if (!queue->nsrq) {
1496                 nvmet_rdma_free_cmds(queue->dev, queue->cmds,
1497                                 queue->recv_queue_size,
1498                                 !queue->host_qid);
1499         }
1500 out_free_responses:
1501         nvmet_rdma_free_rsps(queue);
1502 out_ida_remove:
1503         ida_free(&nvmet_rdma_queue_ida, queue->idx);
1504 out_destroy_sq:
1505         nvmet_sq_destroy(&queue->nvme_sq);
1506 out_free_queue:
1507         kfree(queue);
1508 out_reject:
1509         nvmet_rdma_cm_reject(cm_id, ret);
1510         return NULL;
1511 }
1512
1513 static void nvmet_rdma_qp_event(struct ib_event *event, void *priv)
1514 {
1515         struct nvmet_rdma_queue *queue = priv;
1516
1517         switch (event->event) {
1518         case IB_EVENT_COMM_EST:
1519                 rdma_notify(queue->cm_id, event->event);
1520                 break;
1521         case IB_EVENT_QP_LAST_WQE_REACHED:
1522                 pr_debug("received last WQE reached event for queue=0x%p\n",
1523                          queue);
1524                 break;
1525         default:
1526                 pr_err("received IB QP event: %s (%d)\n",
1527                        ib_event_msg(event->event), event->event);
1528                 break;
1529         }
1530 }
1531
1532 static int nvmet_rdma_cm_accept(struct rdma_cm_id *cm_id,
1533                 struct nvmet_rdma_queue *queue,
1534                 struct rdma_conn_param *p)
1535 {
1536         struct rdma_conn_param  param = { };
1537         struct nvme_rdma_cm_rep priv = { };
1538         int ret = -ENOMEM;
1539
1540         param.rnr_retry_count = 7;
1541         param.flow_control = 1;
1542         param.initiator_depth = min_t(u8, p->initiator_depth,
1543                 queue->dev->device->attrs.max_qp_init_rd_atom);
1544         param.private_data = &priv;
1545         param.private_data_len = sizeof(priv);
1546         priv.recfmt = cpu_to_le16(NVME_RDMA_CM_FMT_1_0);
1547         priv.crqsize = cpu_to_le16(queue->recv_queue_size);
1548
1549         ret = rdma_accept(cm_id, &param);
1550         if (ret)
1551                 pr_err("rdma_accept failed (error code = %d)\n", ret);
1552
1553         return ret;
1554 }
1555
1556 static int nvmet_rdma_queue_connect(struct rdma_cm_id *cm_id,
1557                 struct rdma_cm_event *event)
1558 {
1559         struct nvmet_rdma_device *ndev;
1560         struct nvmet_rdma_queue *queue;
1561         int ret = -EINVAL;
1562
1563         ndev = nvmet_rdma_find_get_device(cm_id);
1564         if (!ndev) {
1565                 nvmet_rdma_cm_reject(cm_id, NVME_RDMA_CM_NO_RSC);
1566                 return -ECONNREFUSED;
1567         }
1568
1569         queue = nvmet_rdma_alloc_queue(ndev, cm_id, event);
1570         if (!queue) {
1571                 ret = -ENOMEM;
1572                 goto put_device;
1573         }
1574
1575         if (queue->host_qid == 0) {
1576                 struct nvmet_rdma_queue *q;
1577                 int pending = 0;
1578
1579                 /* Check for pending controller teardown */
1580                 mutex_lock(&nvmet_rdma_queue_mutex);
1581                 list_for_each_entry(q, &nvmet_rdma_queue_list, queue_list) {
1582                         if (q->nvme_sq.ctrl == queue->nvme_sq.ctrl &&
1583                             q->state == NVMET_RDMA_Q_DISCONNECTING)
1584                                 pending++;
1585                 }
1586                 mutex_unlock(&nvmet_rdma_queue_mutex);
1587                 if (pending > NVMET_RDMA_BACKLOG)
1588                         return NVME_SC_CONNECT_CTRL_BUSY;
1589         }
1590
1591         ret = nvmet_rdma_cm_accept(cm_id, queue, &event->param.conn);
1592         if (ret) {
1593                 /*
1594                  * Don't destroy the cm_id in free path, as we implicitly
1595                  * destroy the cm_id here with non-zero ret code.
1596                  */
1597                 queue->cm_id = NULL;
1598                 goto free_queue;
1599         }
1600
1601         mutex_lock(&nvmet_rdma_queue_mutex);
1602         list_add_tail(&queue->queue_list, &nvmet_rdma_queue_list);
1603         mutex_unlock(&nvmet_rdma_queue_mutex);
1604
1605         return 0;
1606
1607 free_queue:
1608         nvmet_rdma_free_queue(queue);
1609 put_device:
1610         kref_put(&ndev->ref, nvmet_rdma_free_dev);
1611
1612         return ret;
1613 }
1614
1615 static void nvmet_rdma_queue_established(struct nvmet_rdma_queue *queue)
1616 {
1617         unsigned long flags;
1618
1619         spin_lock_irqsave(&queue->state_lock, flags);
1620         if (queue->state != NVMET_RDMA_Q_CONNECTING) {
1621                 pr_warn("trying to establish a connected queue\n");
1622                 goto out_unlock;
1623         }
1624         queue->state = NVMET_RDMA_Q_LIVE;
1625
1626         while (!list_empty(&queue->rsp_wait_list)) {
1627                 struct nvmet_rdma_rsp *cmd;
1628
1629                 cmd = list_first_entry(&queue->rsp_wait_list,
1630                                         struct nvmet_rdma_rsp, wait_list);
1631                 list_del(&cmd->wait_list);
1632
1633                 spin_unlock_irqrestore(&queue->state_lock, flags);
1634                 nvmet_rdma_handle_command(queue, cmd);
1635                 spin_lock_irqsave(&queue->state_lock, flags);
1636         }
1637
1638 out_unlock:
1639         spin_unlock_irqrestore(&queue->state_lock, flags);
1640 }
1641
1642 static void __nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue)
1643 {
1644         bool disconnect = false;
1645         unsigned long flags;
1646
1647         pr_debug("cm_id= %p queue->state= %d\n", queue->cm_id, queue->state);
1648
1649         spin_lock_irqsave(&queue->state_lock, flags);
1650         switch (queue->state) {
1651         case NVMET_RDMA_Q_CONNECTING:
1652                 while (!list_empty(&queue->rsp_wait_list)) {
1653                         struct nvmet_rdma_rsp *rsp;
1654
1655                         rsp = list_first_entry(&queue->rsp_wait_list,
1656                                                struct nvmet_rdma_rsp,
1657                                                wait_list);
1658                         list_del(&rsp->wait_list);
1659                         nvmet_rdma_put_rsp(rsp);
1660                 }
1661                 fallthrough;
1662         case NVMET_RDMA_Q_LIVE:
1663                 queue->state = NVMET_RDMA_Q_DISCONNECTING;
1664                 disconnect = true;
1665                 break;
1666         case NVMET_RDMA_Q_DISCONNECTING:
1667                 break;
1668         }
1669         spin_unlock_irqrestore(&queue->state_lock, flags);
1670
1671         if (disconnect) {
1672                 rdma_disconnect(queue->cm_id);
1673                 queue_work(nvmet_wq, &queue->release_work);
1674         }
1675 }
1676
1677 static void nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue)
1678 {
1679         bool disconnect = false;
1680
1681         mutex_lock(&nvmet_rdma_queue_mutex);
1682         if (!list_empty(&queue->queue_list)) {
1683                 list_del_init(&queue->queue_list);
1684                 disconnect = true;
1685         }
1686         mutex_unlock(&nvmet_rdma_queue_mutex);
1687
1688         if (disconnect)
1689                 __nvmet_rdma_queue_disconnect(queue);
1690 }
1691
1692 static void nvmet_rdma_queue_connect_fail(struct rdma_cm_id *cm_id,
1693                 struct nvmet_rdma_queue *queue)
1694 {
1695         WARN_ON_ONCE(queue->state != NVMET_RDMA_Q_CONNECTING);
1696
1697         mutex_lock(&nvmet_rdma_queue_mutex);
1698         if (!list_empty(&queue->queue_list))
1699                 list_del_init(&queue->queue_list);
1700         mutex_unlock(&nvmet_rdma_queue_mutex);
1701
1702         pr_err("failed to connect queue %d\n", queue->idx);
1703         queue_work(nvmet_wq, &queue->release_work);
1704 }
1705
1706 /**
1707  * nvmet_rdma_device_removal() - Handle RDMA device removal
1708  * @cm_id:      rdma_cm id, used for nvmet port
1709  * @queue:      nvmet rdma queue (cm id qp_context)
1710  *
1711  * DEVICE_REMOVAL event notifies us that the RDMA device is about
1712  * to unplug. Note that this event can be generated on a normal
1713  * queue cm_id and/or a device bound listener cm_id (where in this
1714  * case queue will be null).
1715  *
1716  * We registered an ib_client to handle device removal for queues,
1717  * so we only need to handle the listening port cm_ids. In this case
1718  * we nullify the priv to prevent double cm_id destruction and destroying
1719  * the cm_id implicitely by returning a non-zero rc to the callout.
1720  */
1721 static int nvmet_rdma_device_removal(struct rdma_cm_id *cm_id,
1722                 struct nvmet_rdma_queue *queue)
1723 {
1724         struct nvmet_rdma_port *port;
1725
1726         if (queue) {
1727                 /*
1728                  * This is a queue cm_id. we have registered
1729                  * an ib_client to handle queues removal
1730                  * so don't interfear and just return.
1731                  */
1732                 return 0;
1733         }
1734
1735         port = cm_id->context;
1736
1737         /*
1738          * This is a listener cm_id. Make sure that
1739          * future remove_port won't invoke a double
1740          * cm_id destroy. use atomic xchg to make sure
1741          * we don't compete with remove_port.
1742          */
1743         if (xchg(&port->cm_id, NULL) != cm_id)
1744                 return 0;
1745
1746         /*
1747          * We need to return 1 so that the core will destroy
1748          * it's own ID.  What a great API design..
1749          */
1750         return 1;
1751 }
1752
1753 static int nvmet_rdma_cm_handler(struct rdma_cm_id *cm_id,
1754                 struct rdma_cm_event *event)
1755 {
1756         struct nvmet_rdma_queue *queue = NULL;
1757         int ret = 0;
1758
1759         if (cm_id->qp)
1760                 queue = cm_id->qp->qp_context;
1761
1762         pr_debug("%s (%d): status %d id %p\n",
1763                 rdma_event_msg(event->event), event->event,
1764                 event->status, cm_id);
1765
1766         switch (event->event) {
1767         case RDMA_CM_EVENT_CONNECT_REQUEST:
1768                 ret = nvmet_rdma_queue_connect(cm_id, event);
1769                 break;
1770         case RDMA_CM_EVENT_ESTABLISHED:
1771                 nvmet_rdma_queue_established(queue);
1772                 break;
1773         case RDMA_CM_EVENT_ADDR_CHANGE:
1774                 if (!queue) {
1775                         struct nvmet_rdma_port *port = cm_id->context;
1776
1777                         queue_delayed_work(nvmet_wq, &port->repair_work, 0);
1778                         break;
1779                 }
1780                 fallthrough;
1781         case RDMA_CM_EVENT_DISCONNECTED:
1782         case RDMA_CM_EVENT_TIMEWAIT_EXIT:
1783                 nvmet_rdma_queue_disconnect(queue);
1784                 break;
1785         case RDMA_CM_EVENT_DEVICE_REMOVAL:
1786                 ret = nvmet_rdma_device_removal(cm_id, queue);
1787                 break;
1788         case RDMA_CM_EVENT_REJECTED:
1789                 pr_debug("Connection rejected: %s\n",
1790                          rdma_reject_msg(cm_id, event->status));
1791                 fallthrough;
1792         case RDMA_CM_EVENT_UNREACHABLE:
1793         case RDMA_CM_EVENT_CONNECT_ERROR:
1794                 nvmet_rdma_queue_connect_fail(cm_id, queue);
1795                 break;
1796         default:
1797                 pr_err("received unrecognized RDMA CM event %d\n",
1798                         event->event);
1799                 break;
1800         }
1801
1802         return ret;
1803 }
1804
1805 static void nvmet_rdma_delete_ctrl(struct nvmet_ctrl *ctrl)
1806 {
1807         struct nvmet_rdma_queue *queue, *n;
1808
1809         mutex_lock(&nvmet_rdma_queue_mutex);
1810         list_for_each_entry_safe(queue, n, &nvmet_rdma_queue_list, queue_list) {
1811                 if (queue->nvme_sq.ctrl != ctrl)
1812                         continue;
1813                 list_del_init(&queue->queue_list);
1814                 __nvmet_rdma_queue_disconnect(queue);
1815         }
1816         mutex_unlock(&nvmet_rdma_queue_mutex);
1817 }
1818
1819 static void nvmet_rdma_destroy_port_queues(struct nvmet_rdma_port *port)
1820 {
1821         struct nvmet_rdma_queue *queue, *tmp;
1822         struct nvmet_port *nport = port->nport;
1823
1824         mutex_lock(&nvmet_rdma_queue_mutex);
1825         list_for_each_entry_safe(queue, tmp, &nvmet_rdma_queue_list,
1826                                  queue_list) {
1827                 if (queue->port != nport)
1828                         continue;
1829
1830                 list_del_init(&queue->queue_list);
1831                 __nvmet_rdma_queue_disconnect(queue);
1832         }
1833         mutex_unlock(&nvmet_rdma_queue_mutex);
1834 }
1835
1836 static void nvmet_rdma_disable_port(struct nvmet_rdma_port *port)
1837 {
1838         struct rdma_cm_id *cm_id = xchg(&port->cm_id, NULL);
1839
1840         if (cm_id)
1841                 rdma_destroy_id(cm_id);
1842
1843         /*
1844          * Destroy the remaining queues, which are not belong to any
1845          * controller yet. Do it here after the RDMA-CM was destroyed
1846          * guarantees that no new queue will be created.
1847          */
1848         nvmet_rdma_destroy_port_queues(port);
1849 }
1850
1851 static int nvmet_rdma_enable_port(struct nvmet_rdma_port *port)
1852 {
1853         struct sockaddr *addr = (struct sockaddr *)&port->addr;
1854         struct rdma_cm_id *cm_id;
1855         int ret;
1856
1857         cm_id = rdma_create_id(&init_net, nvmet_rdma_cm_handler, port,
1858                         RDMA_PS_TCP, IB_QPT_RC);
1859         if (IS_ERR(cm_id)) {
1860                 pr_err("CM ID creation failed\n");
1861                 return PTR_ERR(cm_id);
1862         }
1863
1864         /*
1865          * Allow both IPv4 and IPv6 sockets to bind a single port
1866          * at the same time.
1867          */
1868         ret = rdma_set_afonly(cm_id, 1);
1869         if (ret) {
1870                 pr_err("rdma_set_afonly failed (%d)\n", ret);
1871                 goto out_destroy_id;
1872         }
1873
1874         ret = rdma_bind_addr(cm_id, addr);
1875         if (ret) {
1876                 pr_err("binding CM ID to %pISpcs failed (%d)\n", addr, ret);
1877                 goto out_destroy_id;
1878         }
1879
1880         ret = rdma_listen(cm_id, NVMET_RDMA_BACKLOG);
1881         if (ret) {
1882                 pr_err("listening to %pISpcs failed (%d)\n", addr, ret);
1883                 goto out_destroy_id;
1884         }
1885
1886         port->cm_id = cm_id;
1887         return 0;
1888
1889 out_destroy_id:
1890         rdma_destroy_id(cm_id);
1891         return ret;
1892 }
1893
1894 static void nvmet_rdma_repair_port_work(struct work_struct *w)
1895 {
1896         struct nvmet_rdma_port *port = container_of(to_delayed_work(w),
1897                         struct nvmet_rdma_port, repair_work);
1898         int ret;
1899
1900         nvmet_rdma_disable_port(port);
1901         ret = nvmet_rdma_enable_port(port);
1902         if (ret)
1903                 queue_delayed_work(nvmet_wq, &port->repair_work, 5 * HZ);
1904 }
1905
1906 static int nvmet_rdma_add_port(struct nvmet_port *nport)
1907 {
1908         struct nvmet_rdma_port *port;
1909         __kernel_sa_family_t af;
1910         int ret;
1911
1912         port = kzalloc(sizeof(*port), GFP_KERNEL);
1913         if (!port)
1914                 return -ENOMEM;
1915
1916         nport->priv = port;
1917         port->nport = nport;
1918         INIT_DELAYED_WORK(&port->repair_work, nvmet_rdma_repair_port_work);
1919
1920         switch (nport->disc_addr.adrfam) {
1921         case NVMF_ADDR_FAMILY_IP4:
1922                 af = AF_INET;
1923                 break;
1924         case NVMF_ADDR_FAMILY_IP6:
1925                 af = AF_INET6;
1926                 break;
1927         default:
1928                 pr_err("address family %d not supported\n",
1929                         nport->disc_addr.adrfam);
1930                 ret = -EINVAL;
1931                 goto out_free_port;
1932         }
1933
1934         if (nport->inline_data_size < 0) {
1935                 nport->inline_data_size = NVMET_RDMA_DEFAULT_INLINE_DATA_SIZE;
1936         } else if (nport->inline_data_size > NVMET_RDMA_MAX_INLINE_DATA_SIZE) {
1937                 pr_warn("inline_data_size %u is too large, reducing to %u\n",
1938                         nport->inline_data_size,
1939                         NVMET_RDMA_MAX_INLINE_DATA_SIZE);
1940                 nport->inline_data_size = NVMET_RDMA_MAX_INLINE_DATA_SIZE;
1941         }
1942
1943         if (nport->max_queue_size < 0) {
1944                 nport->max_queue_size = NVME_RDMA_DEFAULT_QUEUE_SIZE;
1945         } else if (nport->max_queue_size > NVME_RDMA_MAX_QUEUE_SIZE) {
1946                 pr_warn("max_queue_size %u is too large, reducing to %u\n",
1947                         nport->max_queue_size, NVME_RDMA_MAX_QUEUE_SIZE);
1948                 nport->max_queue_size = NVME_RDMA_MAX_QUEUE_SIZE;
1949         }
1950
1951         ret = inet_pton_with_scope(&init_net, af, nport->disc_addr.traddr,
1952                         nport->disc_addr.trsvcid, &port->addr);
1953         if (ret) {
1954                 pr_err("malformed ip/port passed: %s:%s\n",
1955                         nport->disc_addr.traddr, nport->disc_addr.trsvcid);
1956                 goto out_free_port;
1957         }
1958
1959         ret = nvmet_rdma_enable_port(port);
1960         if (ret)
1961                 goto out_free_port;
1962
1963         pr_info("enabling port %d (%pISpcs)\n",
1964                 le16_to_cpu(nport->disc_addr.portid),
1965                 (struct sockaddr *)&port->addr);
1966
1967         return 0;
1968
1969 out_free_port:
1970         kfree(port);
1971         return ret;
1972 }
1973
1974 static void nvmet_rdma_remove_port(struct nvmet_port *nport)
1975 {
1976         struct nvmet_rdma_port *port = nport->priv;
1977
1978         cancel_delayed_work_sync(&port->repair_work);
1979         nvmet_rdma_disable_port(port);
1980         kfree(port);
1981 }
1982
1983 static void nvmet_rdma_disc_port_addr(struct nvmet_req *req,
1984                 struct nvmet_port *nport, char *traddr)
1985 {
1986         struct nvmet_rdma_port *port = nport->priv;
1987         struct rdma_cm_id *cm_id = port->cm_id;
1988
1989         if (inet_addr_is_any((struct sockaddr *)&cm_id->route.addr.src_addr)) {
1990                 struct nvmet_rdma_rsp *rsp =
1991                         container_of(req, struct nvmet_rdma_rsp, req);
1992                 struct rdma_cm_id *req_cm_id = rsp->queue->cm_id;
1993                 struct sockaddr *addr = (void *)&req_cm_id->route.addr.src_addr;
1994
1995                 sprintf(traddr, "%pISc", addr);
1996         } else {
1997                 memcpy(traddr, nport->disc_addr.traddr, NVMF_TRADDR_SIZE);
1998         }
1999 }
2000
2001 static ssize_t nvmet_rdma_host_port_addr(struct nvmet_ctrl *ctrl,
2002                 char *traddr, size_t traddr_len)
2003 {
2004         struct nvmet_sq *nvme_sq = ctrl->sqs[0];
2005         struct nvmet_rdma_queue *queue =
2006                 container_of(nvme_sq, struct nvmet_rdma_queue, nvme_sq);
2007
2008         return snprintf(traddr, traddr_len, "%pISc",
2009                         (struct sockaddr *)&queue->cm_id->route.addr.dst_addr);
2010 }
2011
2012 static u8 nvmet_rdma_get_mdts(const struct nvmet_ctrl *ctrl)
2013 {
2014         if (ctrl->pi_support)
2015                 return NVMET_RDMA_MAX_METADATA_MDTS;
2016         return NVMET_RDMA_MAX_MDTS;
2017 }
2018
2019 static u16 nvmet_rdma_get_max_queue_size(const struct nvmet_ctrl *ctrl)
2020 {
2021         if (ctrl->pi_support)
2022                 return NVME_RDMA_MAX_METADATA_QUEUE_SIZE;
2023         return NVME_RDMA_MAX_QUEUE_SIZE;
2024 }
2025
2026 static const struct nvmet_fabrics_ops nvmet_rdma_ops = {
2027         .owner                  = THIS_MODULE,
2028         .type                   = NVMF_TRTYPE_RDMA,
2029         .msdbd                  = 1,
2030         .flags                  = NVMF_KEYED_SGLS | NVMF_METADATA_SUPPORTED,
2031         .add_port               = nvmet_rdma_add_port,
2032         .remove_port            = nvmet_rdma_remove_port,
2033         .queue_response         = nvmet_rdma_queue_response,
2034         .delete_ctrl            = nvmet_rdma_delete_ctrl,
2035         .disc_traddr            = nvmet_rdma_disc_port_addr,
2036         .host_traddr            = nvmet_rdma_host_port_addr,
2037         .get_mdts               = nvmet_rdma_get_mdts,
2038         .get_max_queue_size     = nvmet_rdma_get_max_queue_size,
2039 };
2040
2041 static void nvmet_rdma_remove_one(struct ib_device *ib_device, void *client_data)
2042 {
2043         struct nvmet_rdma_queue *queue, *tmp;
2044         struct nvmet_rdma_device *ndev;
2045         bool found = false;
2046
2047         mutex_lock(&device_list_mutex);
2048         list_for_each_entry(ndev, &device_list, entry) {
2049                 if (ndev->device == ib_device) {
2050                         found = true;
2051                         break;
2052                 }
2053         }
2054         mutex_unlock(&device_list_mutex);
2055
2056         if (!found)
2057                 return;
2058
2059         /*
2060          * IB Device that is used by nvmet controllers is being removed,
2061          * delete all queues using this device.
2062          */
2063         mutex_lock(&nvmet_rdma_queue_mutex);
2064         list_for_each_entry_safe(queue, tmp, &nvmet_rdma_queue_list,
2065                                  queue_list) {
2066                 if (queue->dev->device != ib_device)
2067                         continue;
2068
2069                 pr_info("Removing queue %d\n", queue->idx);
2070                 list_del_init(&queue->queue_list);
2071                 __nvmet_rdma_queue_disconnect(queue);
2072         }
2073         mutex_unlock(&nvmet_rdma_queue_mutex);
2074
2075         flush_workqueue(nvmet_wq);
2076 }
2077
2078 static struct ib_client nvmet_rdma_ib_client = {
2079         .name   = "nvmet_rdma",
2080         .remove = nvmet_rdma_remove_one
2081 };
2082
2083 static int __init nvmet_rdma_init(void)
2084 {
2085         int ret;
2086
2087         ret = ib_register_client(&nvmet_rdma_ib_client);
2088         if (ret)
2089                 return ret;
2090
2091         ret = nvmet_register_transport(&nvmet_rdma_ops);
2092         if (ret)
2093                 goto err_ib_client;
2094
2095         return 0;
2096
2097 err_ib_client:
2098         ib_unregister_client(&nvmet_rdma_ib_client);
2099         return ret;
2100 }
2101
2102 static void __exit nvmet_rdma_exit(void)
2103 {
2104         nvmet_unregister_transport(&nvmet_rdma_ops);
2105         ib_unregister_client(&nvmet_rdma_ib_client);
2106         WARN_ON_ONCE(!list_empty(&nvmet_rdma_queue_list));
2107         ida_destroy(&nvmet_rdma_queue_ida);
2108 }
2109
2110 module_init(nvmet_rdma_init);
2111 module_exit(nvmet_rdma_exit);
2112
2113 MODULE_DESCRIPTION("NVMe target RDMA transport driver");
2114 MODULE_LICENSE("GPL v2");
2115 MODULE_ALIAS("nvmet-transport-1"); /* 1 == NVMF_TRTYPE_RDMA */