drivers/infiniband/hw/erdma/erdma_eq.c

   1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
   2
   3 /* Authors: Cheng Xu <[email protected]> */
   4 /*          Kai Shen <[email protected]> */
   5 /* Copyright (c) 2020-2022, Alibaba Group. */
   6
   7 #include "erdma_verbs.h"
   8
   9 #define MAX_POLL_CHUNK_SIZE 16
  10
  11 void notify_eq(struct erdma_eq *eq)
  12 {
  13         u64 db_data = FIELD_PREP(ERDMA_EQDB_CI_MASK, eq->ci) |
  14                       FIELD_PREP(ERDMA_EQDB_ARM_MASK, 1);
  15
  16         *eq->db_record = db_data;
  17         writeq(db_data, eq->db);
  18
  19         atomic64_inc(&eq->notify_num);
  20 }
  21
  22 void *get_next_valid_eqe(struct erdma_eq *eq)
  23 {
  24         u64 *eqe = get_queue_entry(eq->qbuf, eq->ci, eq->depth, EQE_SHIFT);
  25         u32 owner = FIELD_GET(ERDMA_CEQE_HDR_O_MASK, READ_ONCE(*eqe));
  26
  27         return owner ^ !!(eq->ci & eq->depth) ? eqe : NULL;
  28 }
  29
  30 void erdma_aeq_event_handler(struct erdma_dev *dev)
  31 {
  32         struct erdma_aeqe *aeqe;
  33         u32 cqn, qpn;
  34         struct erdma_qp *qp;
  35         struct erdma_cq *cq;
  36         struct ib_event event;
  37         u32 poll_cnt = 0;
  38
  39         memset(&event, 0, sizeof(event));
  40
  41         while (poll_cnt < MAX_POLL_CHUNK_SIZE) {
  42                 aeqe = get_next_valid_eqe(&dev->aeq);
  43                 if (!aeqe)
  44                         break;
  45
  46                 dma_rmb();
  47
  48                 dev->aeq.ci++;
  49                 atomic64_inc(&dev->aeq.event_num);
  50                 poll_cnt++;
  51
  52                 if (FIELD_GET(ERDMA_AEQE_HDR_TYPE_MASK,
  53                               le32_to_cpu(aeqe->hdr)) == ERDMA_AE_TYPE_CQ_ERR) {
  54                         cqn = le32_to_cpu(aeqe->event_data0);
  55                         cq = find_cq_by_cqn(dev, cqn);
  56                         if (!cq)
  57                                 continue;
  58
  59                         event.device = cq->ibcq.device;
  60                         event.element.cq = &cq->ibcq;
  61                         event.event = IB_EVENT_CQ_ERR;
  62                         if (cq->ibcq.event_handler)
  63                                 cq->ibcq.event_handler(&event,
  64                                                        cq->ibcq.cq_context);
  65                 } else {
  66                         qpn = le32_to_cpu(aeqe->event_data0);
  67                         qp = find_qp_by_qpn(dev, qpn);
  68                         if (!qp)
  69                                 continue;
  70
  71                         event.device = qp->ibqp.device;
  72                         event.element.qp = &qp->ibqp;
  73                         event.event = IB_EVENT_QP_FATAL;
  74                         if (qp->ibqp.event_handler)
  75                                 qp->ibqp.event_handler(&event,
  76                                                        qp->ibqp.qp_context);
  77                 }
  78         }
  79
  80         notify_eq(&dev->aeq);
  81 }
  82
  83 int erdma_aeq_init(struct erdma_dev *dev)
  84 {
  85         struct erdma_eq *eq = &dev->aeq;
  86         u32 buf_size;
  87
  88         eq->depth = ERDMA_DEFAULT_EQ_DEPTH;
  89         buf_size = eq->depth << EQE_SHIFT;
  90
  91         eq->qbuf =
  92                 dma_alloc_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size),
  93                                    &eq->qbuf_dma_addr, GFP_KERNEL | __GFP_ZERO);
  94         if (!eq->qbuf)
  95                 return -ENOMEM;
  96
  97         spin_lock_init(&eq->lock);
  98         atomic64_set(&eq->event_num, 0);
  99         atomic64_set(&eq->notify_num, 0);
 100
 101         eq->db = dev->func_bar + ERDMA_REGS_AEQ_DB_REG;
 102         eq->db_record = (u64 *)(eq->qbuf + buf_size);
 103
 104         erdma_reg_write32(dev, ERDMA_REGS_AEQ_ADDR_H_REG,
 105                           upper_32_bits(eq->qbuf_dma_addr));
 106         erdma_reg_write32(dev, ERDMA_REGS_AEQ_ADDR_L_REG,
 107                           lower_32_bits(eq->qbuf_dma_addr));
 108         erdma_reg_write32(dev, ERDMA_REGS_AEQ_DEPTH_REG, eq->depth);
 109         erdma_reg_write64(dev, ERDMA_AEQ_DB_HOST_ADDR_REG,
 110                           eq->qbuf_dma_addr + buf_size);
 111
 112         return 0;
 113 }
 114
 115 void erdma_aeq_destroy(struct erdma_dev *dev)
 116 {
 117         struct erdma_eq *eq = &dev->aeq;
 118
 119         dma_free_coherent(&dev->pdev->dev,
 120                           WARPPED_BUFSIZE(eq->depth << EQE_SHIFT), eq->qbuf,
 121                           eq->qbuf_dma_addr);
 122 }
 123
 124 void erdma_ceq_completion_handler(struct erdma_eq_cb *ceq_cb)
 125 {
 126         struct erdma_dev *dev = ceq_cb->dev;
 127         struct erdma_cq *cq;
 128         u32 poll_cnt = 0;
 129         u64 *ceqe;
 130         int cqn;
 131
 132         if (!ceq_cb->ready)
 133                 return;
 134
 135         while (poll_cnt < MAX_POLL_CHUNK_SIZE) {
 136                 ceqe = get_next_valid_eqe(&ceq_cb->eq);
 137                 if (!ceqe)
 138                         break;
 139
 140                 dma_rmb();
 141                 ceq_cb->eq.ci++;
 142                 poll_cnt++;
 143                 cqn = FIELD_GET(ERDMA_CEQE_HDR_CQN_MASK, READ_ONCE(*ceqe));
 144
 145                 cq = find_cq_by_cqn(dev, cqn);
 146                 if (!cq)
 147                         continue;
 148
 149                 if (rdma_is_kernel_res(&cq->ibcq.res))
 150                         cq->kern_cq.cmdsn++;
 151
 152                 if (cq->ibcq.comp_handler)
 153                         cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
 154         }
 155
 156         notify_eq(&ceq_cb->eq);
 157 }
 158
 159 static irqreturn_t erdma_intr_ceq_handler(int irq, void *data)
 160 {
 161         struct erdma_eq_cb *ceq_cb = data;
 162
 163         tasklet_schedule(&ceq_cb->tasklet);
 164
 165         return IRQ_HANDLED;
 166 }
 167
 168 static void erdma_intr_ceq_task(unsigned long data)
 169 {
 170         erdma_ceq_completion_handler((struct erdma_eq_cb *)data);
 171 }
 172
 173 static int erdma_set_ceq_irq(struct erdma_dev *dev, u16 ceqn)
 174 {
 175         struct erdma_eq_cb *eqc = &dev->ceqs[ceqn];
 176         int err;
 177
 178         snprintf(eqc->irq.name, ERDMA_IRQNAME_SIZE, "erdma-ceq%u@pci:%s", ceqn,
 179                  pci_name(dev->pdev));
 180         eqc->irq.msix_vector = pci_irq_vector(dev->pdev, ceqn + 1);
 181
 182         tasklet_init(&dev->ceqs[ceqn].tasklet, erdma_intr_ceq_task,
 183                      (unsigned long)&dev->ceqs[ceqn]);
 184
 185         cpumask_set_cpu(cpumask_local_spread(ceqn + 1, dev->attrs.numa_node),
 186                         &eqc->irq.affinity_hint_mask);
 187
 188         err = request_irq(eqc->irq.msix_vector, erdma_intr_ceq_handler, 0,
 189                           eqc->irq.name, eqc);
 190         if (err) {
 191                 dev_err(&dev->pdev->dev, "failed to request_irq(%d)\n", err);
 192                 return err;
 193         }
 194
 195         irq_set_affinity_hint(eqc->irq.msix_vector,
 196                               &eqc->irq.affinity_hint_mask);
 197
 198         return 0;
 199 }
 200
 201 static void erdma_free_ceq_irq(struct erdma_dev *dev, u16 ceqn)
 202 {
 203         struct erdma_eq_cb *eqc = &dev->ceqs[ceqn];
 204
 205         irq_set_affinity_hint(eqc->irq.msix_vector, NULL);
 206         free_irq(eqc->irq.msix_vector, eqc);
 207 }
 208
 209 static int create_eq_cmd(struct erdma_dev *dev, u32 eqn, struct erdma_eq *eq)
 210 {
 211         struct erdma_cmdq_create_eq_req req;
 212         dma_addr_t db_info_dma_addr;
 213
 214         erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
 215                                 CMDQ_OPCODE_CREATE_EQ);
 216         req.eqn = eqn;
 217         req.depth = ilog2(eq->depth);
 218         req.qbuf_addr = eq->qbuf_dma_addr;
 219         req.qtype = ERDMA_EQ_TYPE_CEQ;
 220         /* Vector index is the same as EQN. */
 221         req.vector_idx = eqn;
 222         db_info_dma_addr = eq->qbuf_dma_addr + (eq->depth << EQE_SHIFT);
 223         req.db_dma_addr_l = lower_32_bits(db_info_dma_addr);
 224         req.db_dma_addr_h = upper_32_bits(db_info_dma_addr);
 225
 226         return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
 227 }
 228
 229 static int erdma_ceq_init_one(struct erdma_dev *dev, u16 ceqn)
 230 {
 231         struct erdma_eq *eq = &dev->ceqs[ceqn].eq;
 232         u32 buf_size = ERDMA_DEFAULT_EQ_DEPTH << EQE_SHIFT;
 233         int ret;
 234
 235         eq->qbuf =
 236                 dma_alloc_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size),
 237                                    &eq->qbuf_dma_addr, GFP_KERNEL | __GFP_ZERO);
 238         if (!eq->qbuf)
 239                 return -ENOMEM;
 240
 241         spin_lock_init(&eq->lock);
 242         atomic64_set(&eq->event_num, 0);
 243         atomic64_set(&eq->notify_num, 0);
 244
 245         eq->depth = ERDMA_DEFAULT_EQ_DEPTH;
 246         eq->db = dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG +
 247                  (ceqn + 1) * ERDMA_DB_SIZE;
 248         eq->db_record = (u64 *)(eq->qbuf + buf_size);
 249         eq->ci = 0;
 250         dev->ceqs[ceqn].dev = dev;
 251
 252         /* CEQ indexed from 1, 0 rsvd for CMDQ-EQ. */
 253         ret = create_eq_cmd(dev, ceqn + 1, eq);
 254         dev->ceqs[ceqn].ready = ret ? false : true;
 255
 256         return ret;
 257 }
 258
 259 static void erdma_ceq_uninit_one(struct erdma_dev *dev, u16 ceqn)
 260 {
 261         struct erdma_eq *eq = &dev->ceqs[ceqn].eq;
 262         u32 buf_size = ERDMA_DEFAULT_EQ_DEPTH << EQE_SHIFT;
 263         struct erdma_cmdq_destroy_eq_req req;
 264         int err;
 265
 266         dev->ceqs[ceqn].ready = 0;
 267
 268         erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
 269                                 CMDQ_OPCODE_DESTROY_EQ);
 270         /* CEQ indexed from 1, 0 rsvd for CMDQ-EQ. */
 271         req.eqn = ceqn + 1;
 272         req.qtype = ERDMA_EQ_TYPE_CEQ;
 273         req.vector_idx = ceqn + 1;
 274
 275         err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
 276         if (err)
 277                 return;
 278
 279         dma_free_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size), eq->qbuf,
 280                           eq->qbuf_dma_addr);
 281 }
 282
 283 int erdma_ceqs_init(struct erdma_dev *dev)
 284 {
 285         u32 i, j;
 286         int err;
 287
 288         for (i = 0; i < dev->attrs.irq_num - 1; i++) {
 289                 err = erdma_ceq_init_one(dev, i);
 290                 if (err)
 291                         goto out_err;
 292
 293                 err = erdma_set_ceq_irq(dev, i);
 294                 if (err) {
 295                         erdma_ceq_uninit_one(dev, i);
 296                         goto out_err;
 297                 }
 298         }
 299
 300         return 0;
 301
 302 out_err:
 303         for (j = 0; j < i; j++) {
 304                 erdma_free_ceq_irq(dev, j);
 305                 erdma_ceq_uninit_one(dev, j);
 306         }
 307
 308         return err;
 309 }
 310
 311 void erdma_ceqs_uninit(struct erdma_dev *dev)
 312 {
 313         u32 i;
 314
 315         for (i = 0; i < dev->attrs.irq_num - 1; i++) {
 316                 erdma_free_ceq_irq(dev, i);
 317                 erdma_ceq_uninit_one(dev, i);
 318         }
 319 }