drivers/infiniband/hw/qib/qib_sdma.c

   1 /*
   2  * Copyright (c) 2012 Intel Corporation. All rights reserved.
   3  * Copyright (c) 2007 - 2012 QLogic Corporation. All rights reserved.
   4  *
   5  * This software is available to you under a choice of one of two
   6  * licenses.  You may choose to be licensed under the terms of the GNU
   7  * General Public License (GPL) Version 2, available from the file
   8  * COPYING in the main directory of this source tree, or the
   9  * OpenIB.org BSD license below:
  10  *
  11  *     Redistribution and use in source and binary forms, with or
  12  *     without modification, are permitted provided that the following
  13  *     conditions are met:
  14  *
  15  *      - Redistributions of source code must retain the above
  16  *        copyright notice, this list of conditions and the following
  17  *        disclaimer.
  18  *
  19  *      - Redistributions in binary form must reproduce the above
  20  *        copyright notice, this list of conditions and the following
  21  *        disclaimer in the documentation and/or other materials
  22  *        provided with the distribution.
  23  *
  24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31  * SOFTWARE.
  32  */
  33
  34 #include <linux/spinlock.h>
  35 #include <linux/netdevice.h>
  36 #include <linux/moduleparam.h>
  37
  38 #include "qib.h"
  39 #include "qib_common.h"
  40
  41 /* default pio off, sdma on */
  42 static ushort sdma_descq_cnt = 256;
  43 module_param_named(sdma_descq_cnt, sdma_descq_cnt, ushort, S_IRUGO);
  44 MODULE_PARM_DESC(sdma_descq_cnt, "Number of SDMA descq entries");
  45
  46 /*
  47  * Bits defined in the send DMA descriptor.
  48  */
  49 #define SDMA_DESC_LAST          (1ULL << 11)
  50 #define SDMA_DESC_FIRST         (1ULL << 12)
  51 #define SDMA_DESC_DMA_HEAD      (1ULL << 13)
  52 #define SDMA_DESC_USE_LARGE_BUF (1ULL << 14)
  53 #define SDMA_DESC_INTR          (1ULL << 15)
  54 #define SDMA_DESC_COUNT_LSB     16
  55 #define SDMA_DESC_GEN_LSB       30
  56
  57 /* declare all statics here rather than keep sorting */
  58 static int alloc_sdma(struct qib_pportdata *);
  59 static void sdma_complete(struct kref *);
  60 static void sdma_finalput(struct qib_sdma_state *);
  61 static void sdma_get(struct qib_sdma_state *);
  62 static void sdma_put(struct qib_sdma_state *);
  63 static void sdma_set_state(struct qib_pportdata *, enum qib_sdma_states);
  64 static void sdma_start_sw_clean_up(struct qib_pportdata *);
  65 static void sdma_sw_clean_up_task(unsigned long);
  66 static void unmap_desc(struct qib_pportdata *, unsigned);
  67
  68 static void sdma_get(struct qib_sdma_state *ss)
  69 {
  70         kref_get(&ss->kref);
  71 }
  72
  73 static void sdma_complete(struct kref *kref)
  74 {
  75         struct qib_sdma_state *ss =
  76                 container_of(kref, struct qib_sdma_state, kref);
  77
  78         complete(&ss->comp);
  79 }
  80
  81 static void sdma_put(struct qib_sdma_state *ss)
  82 {
  83         kref_put(&ss->kref, sdma_complete);
  84 }
  85
  86 static void sdma_finalput(struct qib_sdma_state *ss)
  87 {
  88         sdma_put(ss);
  89         wait_for_completion(&ss->comp);
  90 }
  91
  92 /*
  93  * Complete all the sdma requests on the active list, in the correct
  94  * order, and with appropriate processing.   Called when cleaning up
  95  * after sdma shutdown, and when new sdma requests are submitted for
  96  * a link that is down.   This matches what is done for requests
  97  * that complete normally, it's just the full list.
  98  *
  99  * Must be called with sdma_lock held
 100  */
 101 static void clear_sdma_activelist(struct qib_pportdata *ppd)
 102 {
 103         struct qib_sdma_txreq *txp, *txp_next;
 104
 105         list_for_each_entry_safe(txp, txp_next, &ppd->sdma_activelist, list) {
 106                 list_del_init(&txp->list);
 107                 if (txp->flags & QIB_SDMA_TXREQ_F_FREEDESC) {
 108                         unsigned idx;
 109
 110                         idx = txp->start_idx;
 111                         while (idx != txp->next_descq_idx) {
 112                                 unmap_desc(ppd, idx);
 113                                 if (++idx == ppd->sdma_descq_cnt)
 114                                         idx = 0;
 115                         }
 116                 }
 117                 if (txp->callback)
 118                         (*txp->callback)(txp, QIB_SDMA_TXREQ_S_ABORTED);
 119         }
 120 }
 121
 122 static void sdma_sw_clean_up_task(unsigned long opaque)
 123 {
 124         struct qib_pportdata *ppd = (struct qib_pportdata *) opaque;
 125         unsigned long flags;
 126
 127         spin_lock_irqsave(&ppd->sdma_lock, flags);
 128
 129         /*
 130          * At this point, the following should always be true:
 131          * - We are halted, so no more descriptors are getting retired.
 132          * - We are not running, so no one is submitting new work.
 133          * - Only we can send the e40_sw_cleaned, so we can't start
 134          *   running again until we say so.  So, the active list and
 135          *   descq are ours to play with.
 136          */
 137
 138         /* Process all retired requests. */
 139         qib_sdma_make_progress(ppd);
 140
 141         clear_sdma_activelist(ppd);
 142
 143         /*
 144          * Resync count of added and removed.  It is VERY important that
 145          * sdma_descq_removed NEVER decrement - user_sdma depends on it.
 146          */
 147         ppd->sdma_descq_removed = ppd->sdma_descq_added;
 148
 149         /*
 150          * Reset our notion of head and tail.
 151          * Note that the HW registers will be reset when switching states
 152          * due to calling __qib_sdma_process_event() below.
 153          */
 154         ppd->sdma_descq_tail = 0;
 155         ppd->sdma_descq_head = 0;
 156         ppd->sdma_head_dma[0] = 0;
 157         ppd->sdma_generation = 0;
 158
 159         __qib_sdma_process_event(ppd, qib_sdma_event_e40_sw_cleaned);
 160
 161         spin_unlock_irqrestore(&ppd->sdma_lock, flags);
 162 }
 163
 164 /*
 165  * This is called when changing to state qib_sdma_state_s10_hw_start_up_wait
 166  * as a result of send buffer errors or send DMA descriptor errors.
 167  * We want to disarm the buffers in these cases.
 168  */
 169 static void sdma_hw_start_up(struct qib_pportdata *ppd)
 170 {
 171         struct qib_sdma_state *ss = &ppd->sdma_state;
 172         unsigned bufno;
 173
 174         for (bufno = ss->first_sendbuf; bufno < ss->last_sendbuf; ++bufno)
 175                 ppd->dd->f_sendctrl(ppd, QIB_SENDCTRL_DISARM_BUF(bufno));
 176
 177         ppd->dd->f_sdma_hw_start_up(ppd);
 178 }
 179
 180 static void sdma_sw_tear_down(struct qib_pportdata *ppd)
 181 {
 182         struct qib_sdma_state *ss = &ppd->sdma_state;
 183
 184         /* Releasing this reference means the state machine has stopped. */
 185         sdma_put(ss);
 186 }
 187
 188 static void sdma_start_sw_clean_up(struct qib_pportdata *ppd)
 189 {
 190         tasklet_hi_schedule(&ppd->sdma_sw_clean_up_task);
 191 }
 192
 193 static void sdma_set_state(struct qib_pportdata *ppd,
 194         enum qib_sdma_states next_state)
 195 {
 196         struct qib_sdma_state *ss = &ppd->sdma_state;
 197         struct sdma_set_state_action *action = ss->set_state_action;
 198         unsigned op = 0;
 199
 200         /* debugging bookkeeping */
 201         ss->previous_state = ss->current_state;
 202         ss->previous_op = ss->current_op;
 203
 204         ss->current_state = next_state;
 205
 206         if (action[next_state].op_enable)
 207                 op |= QIB_SDMA_SENDCTRL_OP_ENABLE;
 208
 209         if (action[next_state].op_intenable)
 210                 op |= QIB_SDMA_SENDCTRL_OP_INTENABLE;
 211
 212         if (action[next_state].op_halt)
 213                 op |= QIB_SDMA_SENDCTRL_OP_HALT;
 214
 215         if (action[next_state].op_drain)
 216                 op |= QIB_SDMA_SENDCTRL_OP_DRAIN;
 217
 218         if (action[next_state].go_s99_running_tofalse)
 219                 ss->go_s99_running = 0;
 220
 221         if (action[next_state].go_s99_running_totrue)
 222                 ss->go_s99_running = 1;
 223
 224         ss->current_op = op;
 225
 226         ppd->dd->f_sdma_sendctrl(ppd, ss->current_op);
 227 }
 228
 229 static void unmap_desc(struct qib_pportdata *ppd, unsigned head)
 230 {
 231         __le64 *descqp = &ppd->sdma_descq[head].qw[0];
 232         u64 desc[2];
 233         dma_addr_t addr;
 234         size_t len;
 235
 236         desc[0] = le64_to_cpu(descqp[0]);
 237         desc[1] = le64_to_cpu(descqp[1]);
 238
 239         addr = (desc[1] << 32) | (desc[0] >> 32);
 240         len = (desc[0] >> 14) & (0x7ffULL << 2);
 241         dma_unmap_single(&ppd->dd->pcidev->dev, addr, len, DMA_TO_DEVICE);
 242 }
 243
 244 static int alloc_sdma(struct qib_pportdata *ppd)
 245 {
 246         ppd->sdma_descq_cnt = sdma_descq_cnt;
 247         if (!ppd->sdma_descq_cnt)
 248                 ppd->sdma_descq_cnt = 256;
 249
 250         /* Allocate memory for SendDMA descriptor FIFO */
 251         ppd->sdma_descq = dma_alloc_coherent(&ppd->dd->pcidev->dev,
 252                 ppd->sdma_descq_cnt * sizeof(u64[2]), &ppd->sdma_descq_phys,
 253                 GFP_KERNEL);
 254
 255         if (!ppd->sdma_descq) {
 256                 qib_dev_err(ppd->dd,
 257                         "failed to allocate SendDMA descriptor FIFO memory\n");
 258                 goto bail;
 259         }
 260
 261         /* Allocate memory for DMA of head register to memory */
 262         ppd->sdma_head_dma = dma_alloc_coherent(&ppd->dd->pcidev->dev,
 263                 PAGE_SIZE, &ppd->sdma_head_phys, GFP_KERNEL);
 264         if (!ppd->sdma_head_dma) {
 265                 qib_dev_err(ppd->dd,
 266                         "failed to allocate SendDMA head memory\n");
 267                 goto cleanup_descq;
 268         }
 269         ppd->sdma_head_dma[0] = 0;
 270         return 0;
 271
 272 cleanup_descq:
 273         dma_free_coherent(&ppd->dd->pcidev->dev,
 274                 ppd->sdma_descq_cnt * sizeof(u64[2]), (void *)ppd->sdma_descq,
 275                 ppd->sdma_descq_phys);
 276         ppd->sdma_descq = NULL;
 277         ppd->sdma_descq_phys = 0;
 278 bail:
 279         ppd->sdma_descq_cnt = 0;
 280         return -ENOMEM;
 281 }
 282
 283 static void free_sdma(struct qib_pportdata *ppd)
 284 {
 285         struct qib_devdata *dd = ppd->dd;
 286
 287         if (ppd->sdma_head_dma) {
 288                 dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
 289                                   (void *)ppd->sdma_head_dma,
 290                                   ppd->sdma_head_phys);
 291                 ppd->sdma_head_dma = NULL;
 292                 ppd->sdma_head_phys = 0;
 293         }
 294
 295         if (ppd->sdma_descq) {
 296                 dma_free_coherent(&dd->pcidev->dev,
 297                                   ppd->sdma_descq_cnt * sizeof(u64[2]),
 298                                   ppd->sdma_descq, ppd->sdma_descq_phys);
 299                 ppd->sdma_descq = NULL;
 300                 ppd->sdma_descq_phys = 0;
 301         }
 302 }
 303
 304 static inline void make_sdma_desc(struct qib_pportdata *ppd,
 305                                   u64 *sdmadesc, u64 addr, u64 dwlen,
 306                                   u64 dwoffset)
 307 {
 308
 309         WARN_ON(addr & 3);
 310         /* SDmaPhyAddr[47:32] */
 311         sdmadesc[1] = addr >> 32;
 312         /* SDmaPhyAddr[31:0] */
 313         sdmadesc[0] = (addr & 0xfffffffcULL) << 32;
 314         /* SDmaGeneration[1:0] */
 315         sdmadesc[0] |= (ppd->sdma_generation & 3ULL) <<
 316                 SDMA_DESC_GEN_LSB;
 317         /* SDmaDwordCount[10:0] */
 318         sdmadesc[0] |= (dwlen & 0x7ffULL) << SDMA_DESC_COUNT_LSB;
 319         /* SDmaBufOffset[12:2] */
 320         sdmadesc[0] |= dwoffset & 0x7ffULL;
 321 }
 322
 323 /* sdma_lock must be held */
 324 int qib_sdma_make_progress(struct qib_pportdata *ppd)
 325 {
 326         struct list_head *lp = NULL;
 327         struct qib_sdma_txreq *txp = NULL;
 328         struct qib_devdata *dd = ppd->dd;
 329         int progress = 0;
 330         u16 hwhead;
 331         u16 idx = 0;
 332
 333         hwhead = dd->f_sdma_gethead(ppd);
 334
 335         /* The reason for some of the complexity of this code is that
 336          * not all descriptors have corresponding txps.  So, we have to
 337          * be able to skip over descs until we wander into the range of
 338          * the next txp on the list.
 339          */
 340
 341         if (!list_empty(&ppd->sdma_activelist)) {
 342                 lp = ppd->sdma_activelist.next;
 343                 txp = list_entry(lp, struct qib_sdma_txreq, list);
 344                 idx = txp->start_idx;
 345         }
 346
 347         while (ppd->sdma_descq_head != hwhead) {
 348                 /* if desc is part of this txp, unmap if needed */
 349                 if (txp && (txp->flags & QIB_SDMA_TXREQ_F_FREEDESC) &&
 350                     (idx == ppd->sdma_descq_head)) {
 351                         unmap_desc(ppd, ppd->sdma_descq_head);
 352                         if (++idx == ppd->sdma_descq_cnt)
 353                                 idx = 0;
 354                 }
 355
 356                 /* increment dequed desc count */
 357                 ppd->sdma_descq_removed++;
 358
 359                 /* advance head, wrap if needed */
 360                 if (++ppd->sdma_descq_head == ppd->sdma_descq_cnt)
 361                         ppd->sdma_descq_head = 0;
 362
 363                 /* if now past this txp's descs, do the callback */
 364                 if (txp && txp->next_descq_idx == ppd->sdma_descq_head) {
 365                         /* remove from active list */
 366                         list_del_init(&txp->list);
 367                         if (txp->callback)
 368                                 (*txp->callback)(txp, QIB_SDMA_TXREQ_S_OK);
 369                         /* see if there is another txp */
 370                         if (list_empty(&ppd->sdma_activelist))
 371                                 txp = NULL;
 372                         else {
 373                                 lp = ppd->sdma_activelist.next;
 374                                 txp = list_entry(lp, struct qib_sdma_txreq,
 375                                         list);
 376                                 idx = txp->start_idx;
 377                         }
 378                 }
 379                 progress = 1;
 380         }
 381         if (progress)
 382                 qib_verbs_sdma_desc_avail(ppd, qib_sdma_descq_freecnt(ppd));
 383         return progress;
 384 }
 385
 386 /*
 387  * This is called from interrupt context.
 388  */
 389 void qib_sdma_intr(struct qib_pportdata *ppd)
 390 {
 391         unsigned long flags;
 392
 393         spin_lock_irqsave(&ppd->sdma_lock, flags);
 394
 395         __qib_sdma_intr(ppd);
 396
 397         spin_unlock_irqrestore(&ppd->sdma_lock, flags);
 398 }
 399
 400 void __qib_sdma_intr(struct qib_pportdata *ppd)
 401 {
 402         if (__qib_sdma_running(ppd)) {
 403                 qib_sdma_make_progress(ppd);
 404                 if (!list_empty(&ppd->sdma_userpending))
 405                         qib_user_sdma_send_desc(ppd, &ppd->sdma_userpending);
 406         }
 407 }
 408
 409 int qib_setup_sdma(struct qib_pportdata *ppd)
 410 {
 411         struct qib_devdata *dd = ppd->dd;
 412         unsigned long flags;
 413         int ret = 0;
 414
 415         ret = alloc_sdma(ppd);
 416         if (ret)
 417                 goto bail;
 418
 419         /* set consistent sdma state */
 420         ppd->dd->f_sdma_init_early(ppd);
 421         spin_lock_irqsave(&ppd->sdma_lock, flags);
 422         sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
 423         spin_unlock_irqrestore(&ppd->sdma_lock, flags);
 424
 425         /* set up reference counting */
 426         kref_init(&ppd->sdma_state.kref);
 427         init_completion(&ppd->sdma_state.comp);
 428
 429         ppd->sdma_generation = 0;
 430         ppd->sdma_descq_head = 0;
 431         ppd->sdma_descq_removed = 0;
 432         ppd->sdma_descq_added = 0;
 433
 434         ppd->sdma_intrequest = 0;
 435         INIT_LIST_HEAD(&ppd->sdma_userpending);
 436
 437         INIT_LIST_HEAD(&ppd->sdma_activelist);
 438
 439         tasklet_init(&ppd->sdma_sw_clean_up_task, sdma_sw_clean_up_task,
 440                 (unsigned long)ppd);
 441
 442         ret = dd->f_init_sdma_regs(ppd);
 443         if (ret)
 444                 goto bail_alloc;
 445
 446         qib_sdma_process_event(ppd, qib_sdma_event_e10_go_hw_start);
 447
 448         return 0;
 449
 450 bail_alloc:
 451         qib_teardown_sdma(ppd);
 452 bail:
 453         return ret;
 454 }
 455
 456 void qib_teardown_sdma(struct qib_pportdata *ppd)
 457 {
 458         qib_sdma_process_event(ppd, qib_sdma_event_e00_go_hw_down);
 459
 460         /*
 461          * This waits for the state machine to exit so it is not
 462          * necessary to kill the sdma_sw_clean_up_task to make sure
 463          * it is not running.
 464          */
 465         sdma_finalput(&ppd->sdma_state);
 466
 467         free_sdma(ppd);
 468 }
 469
 470 int qib_sdma_running(struct qib_pportdata *ppd)
 471 {
 472         unsigned long flags;
 473         int ret;
 474
 475         spin_lock_irqsave(&ppd->sdma_lock, flags);
 476         ret = __qib_sdma_running(ppd);
 477         spin_unlock_irqrestore(&ppd->sdma_lock, flags);
 478
 479         return ret;
 480 }
 481
 482 /*
 483  * Complete a request when sdma not running; likely only request
 484  * but to simplify the code, always queue it, then process the full
 485  * activelist.  We process the entire list to ensure that this particular
 486  * request does get it's callback, but in the correct order.
 487  * Must be called with sdma_lock held
 488  */
 489 static void complete_sdma_err_req(struct qib_pportdata *ppd,
 490                                   struct qib_verbs_txreq *tx)
 491 {
 492         struct qib_qp_priv *priv = tx->qp->priv;
 493
 494         atomic_inc(&priv->s_dma_busy);
 495         /* no sdma descriptors, so no unmap_desc */
 496         tx->txreq.start_idx = 0;
 497         tx->txreq.next_descq_idx = 0;
 498         list_add_tail(&tx->txreq.list, &ppd->sdma_activelist);
 499         clear_sdma_activelist(ppd);
 500 }
 501
 502 /*
 503  * This function queues one IB packet onto the send DMA queue per call.
 504  * The caller is responsible for checking:
 505  * 1) The number of send DMA descriptor entries is less than the size of
 506  *    the descriptor queue.
 507  * 2) The IB SGE addresses and lengths are 32-bit aligned
 508  *    (except possibly the last SGE's length)
 509  * 3) The SGE addresses are suitable for passing to dma_map_single().
 510  */
 511 int qib_sdma_verbs_send(struct qib_pportdata *ppd,
 512                         struct rvt_sge_state *ss, u32 dwords,
 513                         struct qib_verbs_txreq *tx)
 514 {
 515         unsigned long flags;
 516         struct rvt_sge *sge;
 517         struct rvt_qp *qp;
 518         int ret = 0;
 519         u16 tail;
 520         __le64 *descqp;
 521         u64 sdmadesc[2];
 522         u32 dwoffset;
 523         dma_addr_t addr;
 524         struct qib_qp_priv *priv;
 525
 526         spin_lock_irqsave(&ppd->sdma_lock, flags);
 527
 528 retry:
 529         if (unlikely(!__qib_sdma_running(ppd))) {
 530                 complete_sdma_err_req(ppd, tx);
 531                 goto unlock;
 532         }
 533
 534         if (tx->txreq.sg_count > qib_sdma_descq_freecnt(ppd)) {
 535                 if (qib_sdma_make_progress(ppd))
 536                         goto retry;
 537                 if (ppd->dd->flags & QIB_HAS_SDMA_TIMEOUT)
 538                         ppd->dd->f_sdma_set_desc_cnt(ppd,
 539                                         ppd->sdma_descq_cnt / 2);
 540                 goto busy;
 541         }
 542
 543         dwoffset = tx->hdr_dwords;
 544         make_sdma_desc(ppd, sdmadesc, (u64) tx->txreq.addr, dwoffset, 0);
 545
 546         sdmadesc[0] |= SDMA_DESC_FIRST;
 547         if (tx->txreq.flags & QIB_SDMA_TXREQ_F_USELARGEBUF)
 548                 sdmadesc[0] |= SDMA_DESC_USE_LARGE_BUF;
 549
 550         /* write to the descq */
 551         tail = ppd->sdma_descq_tail;
 552         descqp = &ppd->sdma_descq[tail].qw[0];
 553         *descqp++ = cpu_to_le64(sdmadesc[0]);
 554         *descqp++ = cpu_to_le64(sdmadesc[1]);
 555
 556         /* increment the tail */
 557         if (++tail == ppd->sdma_descq_cnt) {
 558                 tail = 0;
 559                 descqp = &ppd->sdma_descq[0].qw[0];
 560                 ++ppd->sdma_generation;
 561         }
 562
 563         tx->txreq.start_idx = tail;
 564
 565         sge = &ss->sge;
 566         while (dwords) {
 567                 u32 dw;
 568                 u32 len;
 569
 570                 len = dwords << 2;
 571                 if (len > sge->length)
 572                         len = sge->length;
 573                 if (len > sge->sge_length)
 574                         len = sge->sge_length;
 575                 BUG_ON(len == 0);
 576                 dw = (len + 3) >> 2;
 577                 addr = dma_map_single(&ppd->dd->pcidev->dev, sge->vaddr,
 578                                       dw << 2, DMA_TO_DEVICE);
 579                 if (dma_mapping_error(&ppd->dd->pcidev->dev, addr))
 580                         goto unmap;
 581                 sdmadesc[0] = 0;
 582                 make_sdma_desc(ppd, sdmadesc, (u64) addr, dw, dwoffset);
 583                 /* SDmaUseLargeBuf has to be set in every descriptor */
 584                 if (tx->txreq.flags & QIB_SDMA_TXREQ_F_USELARGEBUF)
 585                         sdmadesc[0] |= SDMA_DESC_USE_LARGE_BUF;
 586                 /* write to the descq */
 587                 *descqp++ = cpu_to_le64(sdmadesc[0]);
 588                 *descqp++ = cpu_to_le64(sdmadesc[1]);
 589
 590                 /* increment the tail */
 591                 if (++tail == ppd->sdma_descq_cnt) {
 592                         tail = 0;
 593                         descqp = &ppd->sdma_descq[0].qw[0];
 594                         ++ppd->sdma_generation;
 595                 }
 596                 sge->vaddr += len;
 597                 sge->length -= len;
 598                 sge->sge_length -= len;
 599                 if (sge->sge_length == 0) {
 600                         if (--ss->num_sge)
 601                                 *sge = *ss->sg_list++;
 602                 } else if (sge->length == 0 && sge->mr->lkey) {
 603                         if (++sge->n >= RVT_SEGSZ) {
 604                                 if (++sge->m >= sge->mr->mapsz)
 605                                         break;
 606                                 sge->n = 0;
 607                         }
 608                         sge->vaddr =
 609                                 sge->mr->map[sge->m]->segs[sge->n].vaddr;
 610                         sge->length =
 611                                 sge->mr->map[sge->m]->segs[sge->n].length;
 612                 }
 613
 614                 dwoffset += dw;
 615                 dwords -= dw;
 616         }
 617
 618         if (!tail)
 619                 descqp = &ppd->sdma_descq[ppd->sdma_descq_cnt].qw[0];
 620         descqp -= 2;
 621         descqp[0] |= cpu_to_le64(SDMA_DESC_LAST);
 622         if (tx->txreq.flags & QIB_SDMA_TXREQ_F_HEADTOHOST)
 623                 descqp[0] |= cpu_to_le64(SDMA_DESC_DMA_HEAD);
 624         if (tx->txreq.flags & QIB_SDMA_TXREQ_F_INTREQ)
 625                 descqp[0] |= cpu_to_le64(SDMA_DESC_INTR);
 626         priv = tx->qp->priv;
 627         atomic_inc(&priv->s_dma_busy);
 628         tx->txreq.next_descq_idx = tail;
 629         ppd->dd->f_sdma_update_tail(ppd, tail);
 630         ppd->sdma_descq_added += tx->txreq.sg_count;
 631         list_add_tail(&tx->txreq.list, &ppd->sdma_activelist);
 632         goto unlock;
 633
 634 unmap:
 635         for (;;) {
 636                 if (!tail)
 637                         tail = ppd->sdma_descq_cnt - 1;
 638                 else
 639                         tail--;
 640                 if (tail == ppd->sdma_descq_tail)
 641                         break;
 642                 unmap_desc(ppd, tail);
 643         }
 644         qp = tx->qp;
 645         priv = qp->priv;
 646         qib_put_txreq(tx);
 647         spin_lock(&qp->r_lock);
 648         spin_lock(&qp->s_lock);
 649         if (qp->ibqp.qp_type == IB_QPT_RC) {
 650                 /* XXX what about error sending RDMA read responses? */
 651                 if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)
 652                         rvt_error_qp(qp, IB_WC_GENERAL_ERR);
 653         } else if (qp->s_wqe)
 654                 rvt_send_complete(qp, qp->s_wqe, IB_WC_GENERAL_ERR);
 655         spin_unlock(&qp->s_lock);
 656         spin_unlock(&qp->r_lock);
 657         /* return zero to process the next send work request */
 658         goto unlock;
 659
 660 busy:
 661         qp = tx->qp;
 662         priv = qp->priv;
 663         spin_lock(&qp->s_lock);
 664         if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) {
 665                 struct qib_ibdev *dev;
 666
 667                 /*
 668                  * If we couldn't queue the DMA request, save the info
 669                  * and try again later rather than destroying the
 670                  * buffer and undoing the side effects of the copy.
 671                  */
 672                 tx->ss = ss;
 673                 tx->dwords = dwords;
 674                 priv->s_tx = tx;
 675                 dev = &ppd->dd->verbs_dev;
 676                 spin_lock(&dev->rdi.pending_lock);
 677                 if (list_empty(&priv->iowait)) {
 678                         struct qib_ibport *ibp;
 679
 680                         ibp = &ppd->ibport_data;
 681                         ibp->rvp.n_dmawait++;
 682                         qp->s_flags |= RVT_S_WAIT_DMA_DESC;
 683                         list_add_tail(&priv->iowait, &dev->dmawait);
 684                 }
 685                 spin_unlock(&dev->rdi.pending_lock);
 686                 qp->s_flags &= ~RVT_S_BUSY;
 687                 spin_unlock(&qp->s_lock);
 688                 ret = -EBUSY;
 689         } else {
 690                 spin_unlock(&qp->s_lock);
 691                 qib_put_txreq(tx);
 692         }
 693 unlock:
 694         spin_unlock_irqrestore(&ppd->sdma_lock, flags);
 695         return ret;
 696 }
 697
 698 /*
 699  * sdma_lock should be acquired before calling this routine
 700  */
 701 void dump_sdma_state(struct qib_pportdata *ppd)
 702 {
 703         struct qib_sdma_desc *descq;
 704         struct qib_sdma_txreq *txp, *txpnext;
 705         __le64 *descqp;
 706         u64 desc[2];
 707         u64 addr;
 708         u16 gen, dwlen, dwoffset;
 709         u16 head, tail, cnt;
 710
 711         head = ppd->sdma_descq_head;
 712         tail = ppd->sdma_descq_tail;
 713         cnt = qib_sdma_descq_freecnt(ppd);
 714         descq = ppd->sdma_descq;
 715
 716         qib_dev_porterr(ppd->dd, ppd->port,
 717                 "SDMA ppd->sdma_descq_head: %u\n", head);
 718         qib_dev_porterr(ppd->dd, ppd->port,
 719                 "SDMA ppd->sdma_descq_tail: %u\n", tail);
 720         qib_dev_porterr(ppd->dd, ppd->port,
 721                 "SDMA sdma_descq_freecnt: %u\n", cnt);
 722
 723         /* print info for each entry in the descriptor queue */
 724         while (head != tail) {
 725                 char flags[6] = { 'x', 'x', 'x', 'x', 'x', 0 };
 726
 727                 descqp = &descq[head].qw[0];
 728                 desc[0] = le64_to_cpu(descqp[0]);
 729                 desc[1] = le64_to_cpu(descqp[1]);
 730                 flags[0] = (desc[0] & 1<<15) ? 'I' : '-';
 731                 flags[1] = (desc[0] & 1<<14) ? 'L' : 'S';
 732                 flags[2] = (desc[0] & 1<<13) ? 'H' : '-';
 733                 flags[3] = (desc[0] & 1<<12) ? 'F' : '-';
 734                 flags[4] = (desc[0] & 1<<11) ? 'L' : '-';
 735                 addr = (desc[1] << 32) | ((desc[0] >> 32) & 0xfffffffcULL);
 736                 gen = (desc[0] >> 30) & 3ULL;
 737                 dwlen = (desc[0] >> 14) & (0x7ffULL << 2);
 738                 dwoffset = (desc[0] & 0x7ffULL) << 2;
 739                 qib_dev_porterr(ppd->dd, ppd->port,
 740                         "SDMA sdmadesc[%u]: flags:%s addr:0x%016llx gen:%u len:%u bytes offset:%u bytes\n",
 741                          head, flags, addr, gen, dwlen, dwoffset);
 742                 if (++head == ppd->sdma_descq_cnt)
 743                         head = 0;
 744         }
 745
 746         /* print dma descriptor indices from the TX requests */
 747         list_for_each_entry_safe(txp, txpnext, &ppd->sdma_activelist,
 748                                  list)
 749                 qib_dev_porterr(ppd->dd, ppd->port,
 750                         "SDMA txp->start_idx: %u txp->next_descq_idx: %u\n",
 751                         txp->start_idx, txp->next_descq_idx);
 752 }
 753
 754 void qib_sdma_process_event(struct qib_pportdata *ppd,
 755         enum qib_sdma_events event)
 756 {
 757         unsigned long flags;
 758
 759         spin_lock_irqsave(&ppd->sdma_lock, flags);
 760
 761         __qib_sdma_process_event(ppd, event);
 762
 763         if (ppd->sdma_state.current_state == qib_sdma_state_s99_running)
 764                 qib_verbs_sdma_desc_avail(ppd, qib_sdma_descq_freecnt(ppd));
 765
 766         spin_unlock_irqrestore(&ppd->sdma_lock, flags);
 767 }
 768
 769 void __qib_sdma_process_event(struct qib_pportdata *ppd,
 770         enum qib_sdma_events event)
 771 {
 772         struct qib_sdma_state *ss = &ppd->sdma_state;
 773
 774         switch (ss->current_state) {
 775         case qib_sdma_state_s00_hw_down:
 776                 switch (event) {
 777                 case qib_sdma_event_e00_go_hw_down:
 778                         break;
 779                 case qib_sdma_event_e30_go_running:
 780                         /*
 781                          * If down, but running requested (usually result
 782                          * of link up, then we need to start up.
 783                          * This can happen when hw down is requested while
 784                          * bringing the link up with traffic active on
 785                          * 7220, e.g. */
 786                         ss->go_s99_running = 1;
 787                         /* fall through -- and start dma engine */
 788                 case qib_sdma_event_e10_go_hw_start:
 789                         /* This reference means the state machine is started */
 790                         sdma_get(&ppd->sdma_state);
 791                         sdma_set_state(ppd,
 792                                        qib_sdma_state_s10_hw_start_up_wait);
 793                         break;
 794                 case qib_sdma_event_e20_hw_started:
 795                         break;
 796                 case qib_sdma_event_e40_sw_cleaned:
 797                         sdma_sw_tear_down(ppd);
 798                         break;
 799                 case qib_sdma_event_e50_hw_cleaned:
 800                         break;
 801                 case qib_sdma_event_e60_hw_halted:
 802                         break;
 803                 case qib_sdma_event_e70_go_idle:
 804                         break;
 805                 case qib_sdma_event_e7220_err_halted:
 806                         break;
 807                 case qib_sdma_event_e7322_err_halted:
 808                         break;
 809                 case qib_sdma_event_e90_timer_tick:
 810                         break;
 811                 }
 812                 break;
 813
 814         case qib_sdma_state_s10_hw_start_up_wait:
 815                 switch (event) {
 816                 case qib_sdma_event_e00_go_hw_down:
 817                         sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
 818                         sdma_sw_tear_down(ppd);
 819                         break;
 820                 case qib_sdma_event_e10_go_hw_start:
 821                         break;
 822                 case qib_sdma_event_e20_hw_started:
 823                         sdma_set_state(ppd, ss->go_s99_running ?
 824                                        qib_sdma_state_s99_running :
 825                                        qib_sdma_state_s20_idle);
 826                         break;
 827                 case qib_sdma_event_e30_go_running:
 828                         ss->go_s99_running = 1;
 829                         break;
 830                 case qib_sdma_event_e40_sw_cleaned:
 831                         break;
 832                 case qib_sdma_event_e50_hw_cleaned:
 833                         break;
 834                 case qib_sdma_event_e60_hw_halted:
 835                         break;
 836                 case qib_sdma_event_e70_go_idle:
 837                         ss->go_s99_running = 0;
 838                         break;
 839                 case qib_sdma_event_e7220_err_halted:
 840                         break;
 841                 case qib_sdma_event_e7322_err_halted:
 842                         break;
 843                 case qib_sdma_event_e90_timer_tick:
 844                         break;
 845                 }
 846                 break;
 847
 848         case qib_sdma_state_s20_idle:
 849                 switch (event) {
 850                 case qib_sdma_event_e00_go_hw_down:
 851                         sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
 852                         sdma_sw_tear_down(ppd);
 853                         break;
 854                 case qib_sdma_event_e10_go_hw_start:
 855                         break;
 856                 case qib_sdma_event_e20_hw_started:
 857                         break;
 858                 case qib_sdma_event_e30_go_running:
 859                         sdma_set_state(ppd, qib_sdma_state_s99_running);
 860                         ss->go_s99_running = 1;
 861                         break;
 862                 case qib_sdma_event_e40_sw_cleaned:
 863                         break;
 864                 case qib_sdma_event_e50_hw_cleaned:
 865                         break;
 866                 case qib_sdma_event_e60_hw_halted:
 867                         break;
 868                 case qib_sdma_event_e70_go_idle:
 869                         break;
 870                 case qib_sdma_event_e7220_err_halted:
 871                         break;
 872                 case qib_sdma_event_e7322_err_halted:
 873                         break;
 874                 case qib_sdma_event_e90_timer_tick:
 875                         break;
 876                 }
 877                 break;
 878
 879         case qib_sdma_state_s30_sw_clean_up_wait:
 880                 switch (event) {
 881                 case qib_sdma_event_e00_go_hw_down:
 882                         sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
 883                         break;
 884                 case qib_sdma_event_e10_go_hw_start:
 885                         break;
 886                 case qib_sdma_event_e20_hw_started:
 887                         break;
 888                 case qib_sdma_event_e30_go_running:
 889                         ss->go_s99_running = 1;
 890                         break;
 891                 case qib_sdma_event_e40_sw_cleaned:
 892                         sdma_set_state(ppd,
 893                                        qib_sdma_state_s10_hw_start_up_wait);
 894                         sdma_hw_start_up(ppd);
 895                         break;
 896                 case qib_sdma_event_e50_hw_cleaned:
 897                         break;
 898                 case qib_sdma_event_e60_hw_halted:
 899                         break;
 900                 case qib_sdma_event_e70_go_idle:
 901                         ss->go_s99_running = 0;
 902                         break;
 903                 case qib_sdma_event_e7220_err_halted:
 904                         break;
 905                 case qib_sdma_event_e7322_err_halted:
 906                         break;
 907                 case qib_sdma_event_e90_timer_tick:
 908                         break;
 909                 }
 910                 break;
 911
 912         case qib_sdma_state_s40_hw_clean_up_wait:
 913                 switch (event) {
 914                 case qib_sdma_event_e00_go_hw_down:
 915                         sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
 916                         sdma_start_sw_clean_up(ppd);
 917                         break;
 918                 case qib_sdma_event_e10_go_hw_start:
 919                         break;
 920                 case qib_sdma_event_e20_hw_started:
 921                         break;
 922                 case qib_sdma_event_e30_go_running:
 923                         ss->go_s99_running = 1;
 924                         break;
 925                 case qib_sdma_event_e40_sw_cleaned:
 926                         break;
 927                 case qib_sdma_event_e50_hw_cleaned:
 928                         sdma_set_state(ppd,
 929                                        qib_sdma_state_s30_sw_clean_up_wait);
 930                         sdma_start_sw_clean_up(ppd);
 931                         break;
 932                 case qib_sdma_event_e60_hw_halted:
 933                         break;
 934                 case qib_sdma_event_e70_go_idle:
 935                         ss->go_s99_running = 0;
 936                         break;
 937                 case qib_sdma_event_e7220_err_halted:
 938                         break;
 939                 case qib_sdma_event_e7322_err_halted:
 940                         break;
 941                 case qib_sdma_event_e90_timer_tick:
 942                         break;
 943                 }
 944                 break;
 945
 946         case qib_sdma_state_s50_hw_halt_wait:
 947                 switch (event) {
 948                 case qib_sdma_event_e00_go_hw_down:
 949                         sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
 950                         sdma_start_sw_clean_up(ppd);
 951                         break;
 952                 case qib_sdma_event_e10_go_hw_start:
 953                         break;
 954                 case qib_sdma_event_e20_hw_started:
 955                         break;
 956                 case qib_sdma_event_e30_go_running:
 957                         ss->go_s99_running = 1;
 958                         break;
 959                 case qib_sdma_event_e40_sw_cleaned:
 960                         break;
 961                 case qib_sdma_event_e50_hw_cleaned:
 962                         break;
 963                 case qib_sdma_event_e60_hw_halted:
 964                         sdma_set_state(ppd,
 965                                        qib_sdma_state_s40_hw_clean_up_wait);
 966                         ppd->dd->f_sdma_hw_clean_up(ppd);
 967                         break;
 968                 case qib_sdma_event_e70_go_idle:
 969                         ss->go_s99_running = 0;
 970                         break;
 971                 case qib_sdma_event_e7220_err_halted:
 972                         break;
 973                 case qib_sdma_event_e7322_err_halted:
 974                         break;
 975                 case qib_sdma_event_e90_timer_tick:
 976                         break;
 977                 }
 978                 break;
 979
 980         case qib_sdma_state_s99_running:
 981                 switch (event) {
 982                 case qib_sdma_event_e00_go_hw_down:
 983                         sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
 984                         sdma_start_sw_clean_up(ppd);
 985                         break;
 986                 case qib_sdma_event_e10_go_hw_start:
 987                         break;
 988                 case qib_sdma_event_e20_hw_started:
 989                         break;
 990                 case qib_sdma_event_e30_go_running:
 991                         break;
 992                 case qib_sdma_event_e40_sw_cleaned:
 993                         break;
 994                 case qib_sdma_event_e50_hw_cleaned:
 995                         break;
 996                 case qib_sdma_event_e60_hw_halted:
 997                         sdma_set_state(ppd,
 998                                        qib_sdma_state_s30_sw_clean_up_wait);
 999                         sdma_start_sw_clean_up(ppd);
1000                         break;
1001                 case qib_sdma_event_e70_go_idle:
1002                         sdma_set_state(ppd, qib_sdma_state_s50_hw_halt_wait);
1003                         ss->go_s99_running = 0;
1004                         break;
1005                 case qib_sdma_event_e7220_err_halted:
1006                         sdma_set_state(ppd,
1007                                        qib_sdma_state_s30_sw_clean_up_wait);
1008                         sdma_start_sw_clean_up(ppd);
1009                         break;
1010                 case qib_sdma_event_e7322_err_halted:
1011                         sdma_set_state(ppd, qib_sdma_state_s50_hw_halt_wait);
1012                         break;
1013                 case qib_sdma_event_e90_timer_tick:
1014                         break;
1015                 }
1016                 break;
1017         }
1018
1019         ss->last_event = event;
1020 }