]> Git Repo - linux.git/blob - drivers/infiniband/hw/erdma/erdma_cmdq.c
Linux 6.14-rc3
[linux.git] / drivers / infiniband / hw / erdma / erdma_cmdq.c
1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2
3 /* Authors: Cheng Xu <[email protected]> */
4 /*          Kai Shen <[email protected]> */
5 /* Copyright (c) 2020-2022, Alibaba Group. */
6
7 #include "erdma.h"
8
9 static void arm_cmdq_cq(struct erdma_cmdq *cmdq)
10 {
11         struct erdma_dev *dev = container_of(cmdq, struct erdma_dev, cmdq);
12         u64 db_data = FIELD_PREP(ERDMA_CQDB_CI_MASK, cmdq->cq.ci) |
13                       FIELD_PREP(ERDMA_CQDB_ARM_MASK, 1) |
14                       FIELD_PREP(ERDMA_CQDB_CMDSN_MASK, cmdq->cq.cmdsn) |
15                       FIELD_PREP(ERDMA_CQDB_IDX_MASK, cmdq->cq.cmdsn);
16
17         *cmdq->cq.dbrec = db_data;
18         writeq(db_data, dev->func_bar + ERDMA_CMDQ_CQDB_REG);
19
20         atomic64_inc(&cmdq->cq.armed_num);
21 }
22
23 static void kick_cmdq_db(struct erdma_cmdq *cmdq)
24 {
25         struct erdma_dev *dev = container_of(cmdq, struct erdma_dev, cmdq);
26         u64 db_data = FIELD_PREP(ERDMA_CMD_HDR_WQEBB_INDEX_MASK, cmdq->sq.pi);
27
28         *cmdq->sq.dbrec = db_data;
29         writeq(db_data, dev->func_bar + ERDMA_CMDQ_SQDB_REG);
30 }
31
32 static struct erdma_comp_wait *get_comp_wait(struct erdma_cmdq *cmdq)
33 {
34         int comp_idx;
35
36         spin_lock(&cmdq->lock);
37         comp_idx = find_first_zero_bit(cmdq->comp_wait_bitmap,
38                                        cmdq->max_outstandings);
39         if (comp_idx == cmdq->max_outstandings) {
40                 spin_unlock(&cmdq->lock);
41                 return ERR_PTR(-ENOMEM);
42         }
43
44         __set_bit(comp_idx, cmdq->comp_wait_bitmap);
45         spin_unlock(&cmdq->lock);
46
47         return &cmdq->wait_pool[comp_idx];
48 }
49
50 static void put_comp_wait(struct erdma_cmdq *cmdq,
51                           struct erdma_comp_wait *comp_wait)
52 {
53         int used;
54
55         cmdq->wait_pool[comp_wait->ctx_id].cmd_status = ERDMA_CMD_STATUS_INIT;
56         spin_lock(&cmdq->lock);
57         used = __test_and_clear_bit(comp_wait->ctx_id, cmdq->comp_wait_bitmap);
58         spin_unlock(&cmdq->lock);
59
60         WARN_ON(!used);
61 }
62
63 static int erdma_cmdq_wait_res_init(struct erdma_dev *dev,
64                                     struct erdma_cmdq *cmdq)
65 {
66         int i;
67
68         cmdq->wait_pool =
69                 devm_kcalloc(&dev->pdev->dev, cmdq->max_outstandings,
70                              sizeof(struct erdma_comp_wait), GFP_KERNEL);
71         if (!cmdq->wait_pool)
72                 return -ENOMEM;
73
74         spin_lock_init(&cmdq->lock);
75         cmdq->comp_wait_bitmap = devm_bitmap_zalloc(
76                 &dev->pdev->dev, cmdq->max_outstandings, GFP_KERNEL);
77         if (!cmdq->comp_wait_bitmap)
78                 return -ENOMEM;
79
80         for (i = 0; i < cmdq->max_outstandings; i++) {
81                 init_completion(&cmdq->wait_pool[i].wait_event);
82                 cmdq->wait_pool[i].ctx_id = i;
83         }
84
85         return 0;
86 }
87
88 static int erdma_cmdq_sq_init(struct erdma_dev *dev)
89 {
90         struct erdma_cmdq *cmdq = &dev->cmdq;
91         struct erdma_cmdq_sq *sq = &cmdq->sq;
92
93         sq->wqebb_cnt = SQEBB_COUNT(ERDMA_CMDQ_SQE_SIZE);
94         sq->depth = cmdq->max_outstandings * sq->wqebb_cnt;
95
96         sq->qbuf = dma_alloc_coherent(&dev->pdev->dev, sq->depth << SQEBB_SHIFT,
97                                       &sq->qbuf_dma_addr, GFP_KERNEL);
98         if (!sq->qbuf)
99                 return -ENOMEM;
100
101         sq->dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &sq->dbrec_dma);
102         if (!sq->dbrec)
103                 goto err_out;
104
105         spin_lock_init(&sq->lock);
106
107         erdma_reg_write32(dev, ERDMA_REGS_CMDQ_SQ_ADDR_H_REG,
108                           upper_32_bits(sq->qbuf_dma_addr));
109         erdma_reg_write32(dev, ERDMA_REGS_CMDQ_SQ_ADDR_L_REG,
110                           lower_32_bits(sq->qbuf_dma_addr));
111         erdma_reg_write32(dev, ERDMA_REGS_CMDQ_DEPTH_REG, sq->depth);
112         erdma_reg_write64(dev, ERDMA_CMDQ_SQ_DB_HOST_ADDR_REG, sq->dbrec_dma);
113
114         return 0;
115
116 err_out:
117         dma_free_coherent(&dev->pdev->dev, sq->depth << SQEBB_SHIFT,
118                           sq->qbuf, sq->qbuf_dma_addr);
119
120         return -ENOMEM;
121 }
122
123 static int erdma_cmdq_cq_init(struct erdma_dev *dev)
124 {
125         struct erdma_cmdq *cmdq = &dev->cmdq;
126         struct erdma_cmdq_cq *cq = &cmdq->cq;
127
128         cq->depth = cmdq->sq.depth;
129         cq->qbuf = dma_alloc_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
130                                       &cq->qbuf_dma_addr, GFP_KERNEL);
131         if (!cq->qbuf)
132                 return -ENOMEM;
133
134         spin_lock_init(&cq->lock);
135
136         cq->dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &cq->dbrec_dma);
137         if (!cq->dbrec)
138                 goto err_out;
139
140         atomic64_set(&cq->armed_num, 0);
141
142         erdma_reg_write32(dev, ERDMA_REGS_CMDQ_CQ_ADDR_H_REG,
143                           upper_32_bits(cq->qbuf_dma_addr));
144         erdma_reg_write32(dev, ERDMA_REGS_CMDQ_CQ_ADDR_L_REG,
145                           lower_32_bits(cq->qbuf_dma_addr));
146         erdma_reg_write64(dev, ERDMA_CMDQ_CQ_DB_HOST_ADDR_REG, cq->dbrec_dma);
147
148         return 0;
149
150 err_out:
151         dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, cq->qbuf,
152                           cq->qbuf_dma_addr);
153
154         return -ENOMEM;
155 }
156
157 static int erdma_cmdq_eq_init(struct erdma_dev *dev)
158 {
159         struct erdma_cmdq *cmdq = &dev->cmdq;
160         struct erdma_eq *eq = &cmdq->eq;
161         int ret;
162
163         ret = erdma_eq_common_init(dev, eq, cmdq->max_outstandings);
164         if (ret)
165                 return ret;
166
167         eq->db = dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG;
168
169         erdma_reg_write32(dev, ERDMA_REGS_CMDQ_EQ_ADDR_H_REG,
170                           upper_32_bits(eq->qbuf_dma_addr));
171         erdma_reg_write32(dev, ERDMA_REGS_CMDQ_EQ_ADDR_L_REG,
172                           lower_32_bits(eq->qbuf_dma_addr));
173         erdma_reg_write32(dev, ERDMA_REGS_CMDQ_EQ_DEPTH_REG, eq->depth);
174         erdma_reg_write64(dev, ERDMA_CMDQ_EQ_DB_HOST_ADDR_REG, eq->dbrec_dma);
175
176         return 0;
177 }
178
179 int erdma_cmdq_init(struct erdma_dev *dev)
180 {
181         struct erdma_cmdq *cmdq = &dev->cmdq;
182         int err;
183
184         cmdq->max_outstandings = ERDMA_CMDQ_MAX_OUTSTANDING;
185
186         sema_init(&cmdq->credits, cmdq->max_outstandings);
187
188         err = erdma_cmdq_wait_res_init(dev, cmdq);
189         if (err)
190                 return err;
191
192         err = erdma_cmdq_sq_init(dev);
193         if (err)
194                 return err;
195
196         err = erdma_cmdq_cq_init(dev);
197         if (err)
198                 goto err_destroy_sq;
199
200         err = erdma_cmdq_eq_init(dev);
201         if (err)
202                 goto err_destroy_cq;
203
204         set_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state);
205
206         return 0;
207
208 err_destroy_cq:
209         dma_free_coherent(&dev->pdev->dev, cmdq->cq.depth << CQE_SHIFT,
210                           cmdq->cq.qbuf, cmdq->cq.qbuf_dma_addr);
211
212         dma_pool_free(dev->db_pool, cmdq->cq.dbrec, cmdq->cq.dbrec_dma);
213
214 err_destroy_sq:
215         dma_free_coherent(&dev->pdev->dev, cmdq->sq.depth << SQEBB_SHIFT,
216                           cmdq->sq.qbuf, cmdq->sq.qbuf_dma_addr);
217
218         dma_pool_free(dev->db_pool, cmdq->sq.dbrec, cmdq->sq.dbrec_dma);
219
220         return err;
221 }
222
223 void erdma_finish_cmdq_init(struct erdma_dev *dev)
224 {
225         arm_cmdq_cq(&dev->cmdq);
226 }
227
228 void erdma_cmdq_destroy(struct erdma_dev *dev)
229 {
230         struct erdma_cmdq *cmdq = &dev->cmdq;
231
232         clear_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state);
233
234         erdma_eq_destroy(dev, &cmdq->eq);
235
236         dma_free_coherent(&dev->pdev->dev, cmdq->sq.depth << SQEBB_SHIFT,
237                           cmdq->sq.qbuf, cmdq->sq.qbuf_dma_addr);
238
239         dma_pool_free(dev->db_pool, cmdq->sq.dbrec, cmdq->sq.dbrec_dma);
240
241         dma_free_coherent(&dev->pdev->dev, cmdq->cq.depth << CQE_SHIFT,
242                           cmdq->cq.qbuf, cmdq->cq.qbuf_dma_addr);
243
244         dma_pool_free(dev->db_pool, cmdq->cq.dbrec, cmdq->cq.dbrec_dma);
245 }
246
247 static void *get_next_valid_cmdq_cqe(struct erdma_cmdq *cmdq)
248 {
249         __be32 *cqe = get_queue_entry(cmdq->cq.qbuf, cmdq->cq.ci,
250                                       cmdq->cq.depth, CQE_SHIFT);
251         u32 owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK,
252                               be32_to_cpu(READ_ONCE(*cqe)));
253
254         return owner ^ !!(cmdq->cq.ci & cmdq->cq.depth) ? cqe : NULL;
255 }
256
257 static void push_cmdq_sqe(struct erdma_cmdq *cmdq, u64 *req, size_t req_len,
258                           struct erdma_comp_wait *comp_wait)
259 {
260         __le64 *wqe;
261         u64 hdr = *req;
262
263         comp_wait->cmd_status = ERDMA_CMD_STATUS_ISSUED;
264         reinit_completion(&comp_wait->wait_event);
265         comp_wait->sq_pi = cmdq->sq.pi;
266
267         wqe = get_queue_entry(cmdq->sq.qbuf, cmdq->sq.pi, cmdq->sq.depth,
268                               SQEBB_SHIFT);
269         memcpy(wqe, req, req_len);
270
271         cmdq->sq.pi += cmdq->sq.wqebb_cnt;
272         hdr |= FIELD_PREP(ERDMA_CMD_HDR_WQEBB_INDEX_MASK, cmdq->sq.pi) |
273                FIELD_PREP(ERDMA_CMD_HDR_CONTEXT_COOKIE_MASK,
274                           comp_wait->ctx_id) |
275                FIELD_PREP(ERDMA_CMD_HDR_WQEBB_CNT_MASK, cmdq->sq.wqebb_cnt - 1);
276         *wqe = cpu_to_le64(hdr);
277
278         kick_cmdq_db(cmdq);
279 }
280
281 static int erdma_poll_single_cmd_completion(struct erdma_cmdq *cmdq)
282 {
283         struct erdma_comp_wait *comp_wait;
284         u32 hdr0, sqe_idx;
285         __be32 *cqe;
286         u16 ctx_id;
287         u64 *sqe;
288
289         cqe = get_next_valid_cmdq_cqe(cmdq);
290         if (!cqe)
291                 return -EAGAIN;
292
293         cmdq->cq.ci++;
294
295         dma_rmb();
296         hdr0 = be32_to_cpu(*cqe);
297         sqe_idx = be32_to_cpu(*(cqe + 1));
298
299         sqe = get_queue_entry(cmdq->sq.qbuf, sqe_idx, cmdq->sq.depth,
300                               SQEBB_SHIFT);
301         ctx_id = FIELD_GET(ERDMA_CMD_HDR_CONTEXT_COOKIE_MASK, *sqe);
302         comp_wait = &cmdq->wait_pool[ctx_id];
303         if (comp_wait->cmd_status != ERDMA_CMD_STATUS_ISSUED)
304                 return -EIO;
305
306         comp_wait->cmd_status = ERDMA_CMD_STATUS_FINISHED;
307         comp_wait->comp_status = FIELD_GET(ERDMA_CQE_HDR_SYNDROME_MASK, hdr0);
308         cmdq->sq.ci += cmdq->sq.wqebb_cnt;
309         /* Copy 16B comp data after cqe hdr to outer */
310         be32_to_cpu_array(comp_wait->comp_data, cqe + 2, 4);
311
312         complete(&comp_wait->wait_event);
313
314         return 0;
315 }
316
317 static void erdma_polling_cmd_completions(struct erdma_cmdq *cmdq)
318 {
319         unsigned long flags;
320         u16 comp_num;
321
322         spin_lock_irqsave(&cmdq->cq.lock, flags);
323
324         /* We must have less than # of max_outstandings
325          * completions at one time.
326          */
327         for (comp_num = 0; comp_num < cmdq->max_outstandings; comp_num++)
328                 if (erdma_poll_single_cmd_completion(cmdq))
329                         break;
330
331         spin_unlock_irqrestore(&cmdq->cq.lock, flags);
332 }
333
334 void erdma_cmdq_completion_handler(struct erdma_cmdq *cmdq)
335 {
336         int got_event = 0;
337
338         if (!test_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state))
339                 return;
340
341         while (get_next_valid_eqe(&cmdq->eq)) {
342                 cmdq->eq.ci++;
343                 got_event++;
344         }
345
346         if (got_event) {
347                 cmdq->cq.cmdsn++;
348                 erdma_polling_cmd_completions(cmdq);
349                 arm_cmdq_cq(cmdq);
350         }
351
352         notify_eq(&cmdq->eq);
353 }
354
355 static int erdma_poll_cmd_completion(struct erdma_comp_wait *comp_ctx,
356                                      struct erdma_cmdq *cmdq, u32 timeout)
357 {
358         unsigned long comp_timeout = jiffies + msecs_to_jiffies(timeout);
359
360         while (1) {
361                 erdma_polling_cmd_completions(cmdq);
362                 if (comp_ctx->cmd_status != ERDMA_CMD_STATUS_ISSUED)
363                         break;
364
365                 if (time_is_before_jiffies(comp_timeout))
366                         return -ETIME;
367
368                 udelay(20);
369         }
370
371         return 0;
372 }
373
374 static int erdma_wait_cmd_completion(struct erdma_comp_wait *comp_ctx,
375                                      struct erdma_cmdq *cmdq, u32 timeout)
376 {
377         unsigned long flags = 0;
378
379         wait_for_completion_timeout(&comp_ctx->wait_event,
380                                     msecs_to_jiffies(timeout));
381
382         if (unlikely(comp_ctx->cmd_status != ERDMA_CMD_STATUS_FINISHED)) {
383                 spin_lock_irqsave(&cmdq->cq.lock, flags);
384                 comp_ctx->cmd_status = ERDMA_CMD_STATUS_TIMEOUT;
385                 spin_unlock_irqrestore(&cmdq->cq.lock, flags);
386                 return -ETIME;
387         }
388
389         return 0;
390 }
391
392 void erdma_cmdq_build_reqhdr(u64 *hdr, u32 mod, u32 op)
393 {
394         *hdr = FIELD_PREP(ERDMA_CMD_HDR_SUB_MOD_MASK, mod) |
395                FIELD_PREP(ERDMA_CMD_HDR_OPCODE_MASK, op);
396 }
397
398 int erdma_post_cmd_wait(struct erdma_cmdq *cmdq, void *req, u32 req_size,
399                         u64 *resp0, u64 *resp1, bool sleepable)
400 {
401         struct erdma_comp_wait *comp_wait;
402         int ret;
403
404         if (!test_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state))
405                 return -ENODEV;
406
407         if (!sleepable) {
408                 while (down_trylock(&cmdq->credits))
409                         ;
410         } else {
411                 down(&cmdq->credits);
412         }
413
414         comp_wait = get_comp_wait(cmdq);
415         if (IS_ERR(comp_wait)) {
416                 clear_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state);
417                 set_bit(ERDMA_CMDQ_STATE_CTX_ERR_BIT, &cmdq->state);
418                 up(&cmdq->credits);
419                 return PTR_ERR(comp_wait);
420         }
421
422         spin_lock(&cmdq->sq.lock);
423         push_cmdq_sqe(cmdq, req, req_size, comp_wait);
424         spin_unlock(&cmdq->sq.lock);
425
426         if (sleepable)
427                 ret = erdma_wait_cmd_completion(comp_wait, cmdq,
428                                                 ERDMA_CMDQ_TIMEOUT_MS);
429         else
430                 ret = erdma_poll_cmd_completion(comp_wait, cmdq,
431                                                 ERDMA_CMDQ_TIMEOUT_MS);
432
433         if (ret) {
434                 set_bit(ERDMA_CMDQ_STATE_TIMEOUT_BIT, &cmdq->state);
435                 clear_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state);
436                 goto out;
437         }
438
439         if (comp_wait->comp_status)
440                 ret = -EIO;
441
442         if (resp0 && resp1) {
443                 *resp0 = *((u64 *)&comp_wait->comp_data[0]);
444                 *resp1 = *((u64 *)&comp_wait->comp_data[2]);
445         }
446         put_comp_wait(cmdq, comp_wait);
447
448 out:
449         up(&cmdq->credits);
450
451         return ret;
452 }
This page took 0.058879 seconds and 4 git commands to generate.