drivers/infiniband/hw/hfi1/verbs_txreq.c

   1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
   2 /*
   3  * Copyright(c) 2016 - 2018 Intel Corporation.
   4  */
   5
   6 #include "hfi.h"
   7 #include "verbs_txreq.h"
   8 #include "qp.h"
   9 #include "trace.h"
  10
  11 #define TXREQ_LEN 24
  12
  13 void hfi1_put_txreq(struct verbs_txreq *tx)
  14 {
  15         struct hfi1_ibdev *dev;
  16         struct rvt_qp *qp;
  17         unsigned long flags;
  18         unsigned int seq;
  19         struct hfi1_qp_priv *priv;
  20
  21         qp = tx->qp;
  22         dev = to_idev(qp->ibqp.device);
  23
  24         if (tx->mr)
  25                 rvt_put_mr(tx->mr);
  26
  27         sdma_txclean(dd_from_dev(dev), &tx->txreq);
  28
  29         /* Free verbs_txreq and return to slab cache */
  30         kmem_cache_free(dev->verbs_txreq_cache, tx);
  31
  32         do {
  33                 seq = read_seqbegin(&dev->txwait_lock);
  34                 if (!list_empty(&dev->txwait)) {
  35                         struct iowait *wait;
  36
  37                         write_seqlock_irqsave(&dev->txwait_lock, flags);
  38                         wait = list_first_entry(&dev->txwait, struct iowait,
  39                                                 list);
  40                         qp = iowait_to_qp(wait);
  41                         priv = qp->priv;
  42                         list_del_init(&priv->s_iowait.list);
  43                         /* refcount held until actual wake up */
  44                         write_sequnlock_irqrestore(&dev->txwait_lock, flags);
  45                         hfi1_qp_wakeup(qp, RVT_S_WAIT_TX);
  46                         break;
  47                 }
  48         } while (read_seqretry(&dev->txwait_lock, seq));
  49 }
  50
  51 struct verbs_txreq *__get_txreq(struct hfi1_ibdev *dev,
  52                                 struct rvt_qp *qp)
  53         __must_hold(&qp->s_lock)
  54 {
  55         struct verbs_txreq *tx = NULL;
  56
  57         write_seqlock(&dev->txwait_lock);
  58         if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) {
  59                 struct hfi1_qp_priv *priv;
  60
  61                 tx = kmem_cache_alloc(dev->verbs_txreq_cache, VERBS_TXREQ_GFP);
  62                 if (tx)
  63                         goto out;
  64                 priv = qp->priv;
  65                 if (list_empty(&priv->s_iowait.list)) {
  66                         dev->n_txwait++;
  67                         qp->s_flags |= RVT_S_WAIT_TX;
  68                         list_add_tail(&priv->s_iowait.list, &dev->txwait);
  69                         priv->s_iowait.lock = &dev->txwait_lock;
  70                         trace_hfi1_qpsleep(qp, RVT_S_WAIT_TX);
  71                         rvt_get_qp(qp);
  72                 }
  73                 qp->s_flags &= ~RVT_S_BUSY;
  74         }
  75 out:
  76         write_sequnlock(&dev->txwait_lock);
  77         return tx;
  78 }
  79
  80 int verbs_txreq_init(struct hfi1_ibdev *dev)
  81 {
  82         char buf[TXREQ_LEN];
  83         struct hfi1_devdata *dd = dd_from_dev(dev);
  84
  85         snprintf(buf, sizeof(buf), "hfi1_%u_vtxreq_cache", dd->unit);
  86         dev->verbs_txreq_cache = kmem_cache_create(buf,
  87                                                    sizeof(struct verbs_txreq),
  88                                                    0, SLAB_HWCACHE_ALIGN,
  89                                                    NULL);
  90         if (!dev->verbs_txreq_cache)
  91                 return -ENOMEM;
  92         return 0;
  93 }
  94
  95 void verbs_txreq_exit(struct hfi1_ibdev *dev)
  96 {
  97         kmem_cache_destroy(dev->verbs_txreq_cache);
  98         dev->verbs_txreq_cache = NULL;
  99 }