drivers/usb/cdns3/cdnsp-ring.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Cadence CDNSP DRD Driver.
   4  *
   5  * Copyright (C) 2020 Cadence.
   6  *
   7  * Author: Pawel Laszczak <[email protected]>
   8  *
   9  * Code based on Linux XHCI driver.
  10  * Origin: Copyright (C) 2008 Intel Corp
  11  */
  12
  13 /*
  14  * Ring initialization rules:
  15  * 1. Each segment is initialized to zero, except for link TRBs.
  16  * 2. Ring cycle state = 0. This represents Producer Cycle State (PCS) or
  17  *    Consumer Cycle State (CCS), depending on ring function.
  18  * 3. Enqueue pointer = dequeue pointer = address of first TRB in the segment.
  19  *
  20  * Ring behavior rules:
  21  * 1. A ring is empty if enqueue == dequeue. This means there will always be at
  22  *    least one free TRB in the ring. This is useful if you want to turn that
  23  *    into a link TRB and expand the ring.
  24  * 2. When incrementing an enqueue or dequeue pointer, if the next TRB is a
  25  *    link TRB, then load the pointer with the address in the link TRB. If the
  26  *    link TRB had its toggle bit set, you may need to update the ring cycle
  27  *    state (see cycle bit rules). You may have to do this multiple times
  28  *    until you reach a non-link TRB.
  29  * 3. A ring is full if enqueue++ (for the definition of increment above)
  30  *    equals the dequeue pointer.
  31  *
  32  * Cycle bit rules:
  33  * 1. When a consumer increments a dequeue pointer and encounters a toggle bit
  34  *    in a link TRB, it must toggle the ring cycle state.
  35  * 2. When a producer increments an enqueue pointer and encounters a toggle bit
  36  *    in a link TRB, it must toggle the ring cycle state.
  37  *
  38  * Producer rules:
  39  * 1. Check if ring is full before you enqueue.
  40  * 2. Write the ring cycle state to the cycle bit in the TRB you're enqueuing.
  41  *    Update enqueue pointer between each write (which may update the ring
  42  *    cycle state).
  43  * 3. Notify consumer. If SW is producer, it rings the doorbell for command
  44  *    and endpoint rings. If controller is the producer for the event ring,
  45  *    and it generates an interrupt according to interrupt modulation rules.
  46  *
  47  * Consumer rules:
  48  * 1. Check if TRB belongs to you. If the cycle bit == your ring cycle state,
  49  *    the TRB is owned by the consumer.
  50  * 2. Update dequeue pointer (which may update the ring cycle state) and
  51  *    continue processing TRBs until you reach a TRB which is not owned by you.
  52  * 3. Notify the producer. SW is the consumer for the event ring, and it
  53  *    updates event ring dequeue pointer. Controller is the consumer for the
  54  *    command and endpoint rings; it generates events on the event ring
  55  *    for these.
  56  */
  57
  58 #include <linux/scatterlist.h>
  59 #include <linux/dma-mapping.h>
  60 #include <linux/delay.h>
  61 #include <linux/slab.h>
  62 #include <linux/irq.h>
  63
  64 #include "cdnsp-trace.h"
  65 #include "cdnsp-gadget.h"
  66
  67 /*
  68  * Returns zero if the TRB isn't in this segment, otherwise it returns the DMA
  69  * address of the TRB.
  70  */
  71 dma_addr_t cdnsp_trb_virt_to_dma(struct cdnsp_segment *seg,
  72                                  union cdnsp_trb *trb)
  73 {
  74         unsigned long segment_offset = trb - seg->trbs;
  75
  76         if (trb < seg->trbs || segment_offset >= TRBS_PER_SEGMENT)
  77                 return 0;
  78
  79         return seg->dma + (segment_offset * sizeof(*trb));
  80 }
  81
  82 static bool cdnsp_trb_is_noop(union cdnsp_trb *trb)
  83 {
  84         return TRB_TYPE_NOOP_LE32(trb->generic.field[3]);
  85 }
  86
  87 static bool cdnsp_trb_is_link(union cdnsp_trb *trb)
  88 {
  89         return TRB_TYPE_LINK_LE32(trb->link.control);
  90 }
  91
  92 bool cdnsp_last_trb_on_seg(struct cdnsp_segment *seg, union cdnsp_trb *trb)
  93 {
  94         return trb == &seg->trbs[TRBS_PER_SEGMENT - 1];
  95 }
  96
  97 bool cdnsp_last_trb_on_ring(struct cdnsp_ring *ring,
  98                             struct cdnsp_segment *seg,
  99                             union cdnsp_trb *trb)
 100 {
 101         return cdnsp_last_trb_on_seg(seg, trb) && (seg->next == ring->first_seg);
 102 }
 103
 104 static bool cdnsp_link_trb_toggles_cycle(union cdnsp_trb *trb)
 105 {
 106         return le32_to_cpu(trb->link.control) & LINK_TOGGLE;
 107 }
 108
 109 static void cdnsp_trb_to_noop(union cdnsp_trb *trb, u32 noop_type)
 110 {
 111         if (cdnsp_trb_is_link(trb)) {
 112                 /* Unchain chained link TRBs. */
 113                 trb->link.control &= cpu_to_le32(~TRB_CHAIN);
 114         } else {
 115                 trb->generic.field[0] = 0;
 116                 trb->generic.field[1] = 0;
 117                 trb->generic.field[2] = 0;
 118                 /* Preserve only the cycle bit of this TRB. */
 119                 trb->generic.field[3] &= cpu_to_le32(TRB_CYCLE);
 120                 trb->generic.field[3] |= cpu_to_le32(TRB_TYPE(noop_type));
 121         }
 122 }
 123
 124 /*
 125  * Updates trb to point to the next TRB in the ring, and updates seg if the next
 126  * TRB is in a new segment. This does not skip over link TRBs, and it does not
 127  * effect the ring dequeue or enqueue pointers.
 128  */
 129 static void cdnsp_next_trb(struct cdnsp_device *pdev,
 130                            struct cdnsp_ring *ring,
 131                            struct cdnsp_segment **seg,
 132                            union cdnsp_trb **trb)
 133 {
 134         if (cdnsp_trb_is_link(*trb)) {
 135                 *seg = (*seg)->next;
 136                 *trb = ((*seg)->trbs);
 137         } else {
 138                 (*trb)++;
 139         }
 140 }
 141
 142 /*
 143  * See Cycle bit rules. SW is the consumer for the event ring only.
 144  * Don't make a ring full of link TRBs. That would be dumb and this would loop.
 145  */
 146 void cdnsp_inc_deq(struct cdnsp_device *pdev, struct cdnsp_ring *ring)
 147 {
 148         /* event ring doesn't have link trbs, check for last trb. */
 149         if (ring->type == TYPE_EVENT) {
 150                 if (!cdnsp_last_trb_on_seg(ring->deq_seg, ring->dequeue)) {
 151                         ring->dequeue++;
 152                         goto out;
 153                 }
 154
 155                 if (cdnsp_last_trb_on_ring(ring, ring->deq_seg, ring->dequeue))
 156                         ring->cycle_state ^= 1;
 157
 158                 ring->deq_seg = ring->deq_seg->next;
 159                 ring->dequeue = ring->deq_seg->trbs;
 160                 goto out;
 161         }
 162
 163         /* All other rings have link trbs. */
 164         if (!cdnsp_trb_is_link(ring->dequeue)) {
 165                 ring->dequeue++;
 166                 ring->num_trbs_free++;
 167         }
 168         while (cdnsp_trb_is_link(ring->dequeue)) {
 169                 ring->deq_seg = ring->deq_seg->next;
 170                 ring->dequeue = ring->deq_seg->trbs;
 171         }
 172 out:
 173         trace_cdnsp_inc_deq(ring);
 174 }
 175
 176 /*
 177  * See Cycle bit rules. SW is the consumer for the event ring only.
 178  * Don't make a ring full of link TRBs. That would be dumb and this would loop.
 179  *
 180  * If we've just enqueued a TRB that is in the middle of a TD (meaning the
 181  * chain bit is set), then set the chain bit in all the following link TRBs.
 182  * If we've enqueued the last TRB in a TD, make sure the following link TRBs
 183  * have their chain bit cleared (so that each Link TRB is a separate TD).
 184  *
 185  * @more_trbs_coming:   Will you enqueue more TRBs before ringing the doorbell.
 186  */
 187 static void cdnsp_inc_enq(struct cdnsp_device *pdev,
 188                           struct cdnsp_ring *ring,
 189                           bool more_trbs_coming)
 190 {
 191         union cdnsp_trb *next;
 192         u32 chain;
 193
 194         chain = le32_to_cpu(ring->enqueue->generic.field[3]) & TRB_CHAIN;
 195
 196         /* If this is not event ring, there is one less usable TRB. */
 197         if (!cdnsp_trb_is_link(ring->enqueue))
 198                 ring->num_trbs_free--;
 199         next = ++(ring->enqueue);
 200
 201         /* Update the dequeue pointer further if that was a link TRB */
 202         while (cdnsp_trb_is_link(next)) {
 203                 /*
 204                  * If the caller doesn't plan on enqueuing more TDs before
 205                  * ringing the doorbell, then we don't want to give the link TRB
 206                  * to the hardware just yet. We'll give the link TRB back in
 207                  * cdnsp_prepare_ring() just before we enqueue the TD at the
 208                  * top of the ring.
 209                  */
 210                 if (!chain && !more_trbs_coming)
 211                         break;
 212
 213                 next->link.control &= cpu_to_le32(~TRB_CHAIN);
 214                 next->link.control |= cpu_to_le32(chain);
 215
 216                 /* Give this link TRB to the hardware */
 217                 wmb();
 218                 next->link.control ^= cpu_to_le32(TRB_CYCLE);
 219
 220                 /* Toggle the cycle bit after the last ring segment. */
 221                 if (cdnsp_link_trb_toggles_cycle(next))
 222                         ring->cycle_state ^= 1;
 223
 224                 ring->enq_seg = ring->enq_seg->next;
 225                 ring->enqueue = ring->enq_seg->trbs;
 226                 next = ring->enqueue;
 227         }
 228
 229         trace_cdnsp_inc_enq(ring);
 230 }
 231
 232 /*
 233  * Check to see if there's room to enqueue num_trbs on the ring and make sure
 234  * enqueue pointer will not advance into dequeue segment.
 235  */
 236 static bool cdnsp_room_on_ring(struct cdnsp_device *pdev,
 237                                struct cdnsp_ring *ring,
 238                                unsigned int num_trbs)
 239 {
 240         int num_trbs_in_deq_seg;
 241
 242         if (ring->num_trbs_free < num_trbs)
 243                 return false;
 244
 245         if (ring->type != TYPE_COMMAND && ring->type != TYPE_EVENT) {
 246                 num_trbs_in_deq_seg = ring->dequeue - ring->deq_seg->trbs;
 247
 248                 if (ring->num_trbs_free < num_trbs + num_trbs_in_deq_seg)
 249                         return false;
 250         }
 251
 252         return true;
 253 }
 254
 255 /*
 256  * Workaround for L1: controller has issue with resuming from L1 after
 257  * setting doorbell for endpoint during L1 state. This function forces
 258  * resume signal in such case.
 259  */
 260 static void cdnsp_force_l0_go(struct cdnsp_device *pdev)
 261 {
 262         if (pdev->active_port == &pdev->usb2_port && pdev->gadget.lpm_capable)
 263                 cdnsp_set_link_state(pdev, &pdev->active_port->regs->portsc, XDEV_U0);
 264 }
 265
 266 /* Ring the doorbell after placing a command on the ring. */
 267 void cdnsp_ring_cmd_db(struct cdnsp_device *pdev)
 268 {
 269         writel(DB_VALUE_CMD, &pdev->dba->cmd_db);
 270 }
 271
 272 /*
 273  * Ring the doorbell after placing a transfer on the ring.
 274  * Returns true if doorbell was set, otherwise false.
 275  */
 276 static bool cdnsp_ring_ep_doorbell(struct cdnsp_device *pdev,
 277                                    struct cdnsp_ep *pep,
 278                                    unsigned int stream_id)
 279 {
 280         __le32 __iomem *reg_addr = &pdev->dba->ep_db;
 281         unsigned int ep_state = pep->ep_state;
 282         unsigned int db_value;
 283
 284         /*
 285          * Don't ring the doorbell for this endpoint if endpoint is halted or
 286          * disabled.
 287          */
 288         if (ep_state & EP_HALTED || !(ep_state & EP_ENABLED))
 289                 return false;
 290
 291         /* For stream capable endpoints driver can ring doorbell only twice. */
 292         if (pep->ep_state & EP_HAS_STREAMS) {
 293                 if (pep->stream_info.drbls_count >= 2)
 294                         return false;
 295
 296                 pep->stream_info.drbls_count++;
 297         }
 298
 299         pep->ep_state &= ~EP_STOPPED;
 300
 301         if (pep->idx == 0 && pdev->ep0_stage == CDNSP_DATA_STAGE &&
 302             !pdev->ep0_expect_in)
 303                 db_value = DB_VALUE_EP0_OUT(pep->idx, stream_id);
 304         else
 305                 db_value = DB_VALUE(pep->idx, stream_id);
 306
 307         trace_cdnsp_tr_drbl(pep, stream_id);
 308
 309         writel(db_value, reg_addr);
 310
 311         cdnsp_force_l0_go(pdev);
 312
 313         /* Doorbell was set. */
 314         return true;
 315 }
 316
 317 /*
 318  * Get the right ring for the given pep and stream_id.
 319  * If the endpoint supports streams, boundary check the USB request's stream ID.
 320  * If the endpoint doesn't support streams, return the singular endpoint ring.
 321  */
 322 static struct cdnsp_ring *cdnsp_get_transfer_ring(struct cdnsp_device *pdev,
 323                                                   struct cdnsp_ep *pep,
 324                                                   unsigned int stream_id)
 325 {
 326         if (!(pep->ep_state & EP_HAS_STREAMS))
 327                 return pep->ring;
 328
 329         if (stream_id == 0 || stream_id >= pep->stream_info.num_streams) {
 330                 dev_err(pdev->dev, "ERR: %s ring doesn't exist for SID: %d.\n",
 331                         pep->name, stream_id);
 332                 return NULL;
 333         }
 334
 335         return pep->stream_info.stream_rings[stream_id];
 336 }
 337
 338 static struct cdnsp_ring *
 339         cdnsp_request_to_transfer_ring(struct cdnsp_device *pdev,
 340                                        struct cdnsp_request *preq)
 341 {
 342         return cdnsp_get_transfer_ring(pdev, preq->pep,
 343                                        preq->request.stream_id);
 344 }
 345
 346 /* Ring the doorbell for any rings with pending requests. */
 347 void cdnsp_ring_doorbell_for_active_rings(struct cdnsp_device *pdev,
 348                                           struct cdnsp_ep *pep)
 349 {
 350         struct cdnsp_stream_info *stream_info;
 351         unsigned int stream_id;
 352         int ret;
 353
 354         if (pep->ep_state & EP_DIS_IN_RROGRESS)
 355                 return;
 356
 357         /* A ring has pending Request if its TD list is not empty. */
 358         if (!(pep->ep_state & EP_HAS_STREAMS) && pep->number) {
 359                 if (pep->ring && !list_empty(&pep->ring->td_list))
 360                         cdnsp_ring_ep_doorbell(pdev, pep, 0);
 361                 return;
 362         }
 363
 364         stream_info = &pep->stream_info;
 365
 366         for (stream_id = 1; stream_id < stream_info->num_streams; stream_id++) {
 367                 struct cdnsp_td *td, *td_temp;
 368                 struct cdnsp_ring *ep_ring;
 369
 370                 if (stream_info->drbls_count >= 2)
 371                         return;
 372
 373                 ep_ring = cdnsp_get_transfer_ring(pdev, pep, stream_id);
 374                 if (!ep_ring)
 375                         continue;
 376
 377                 if (!ep_ring->stream_active || ep_ring->stream_rejected)
 378                         continue;
 379
 380                 list_for_each_entry_safe(td, td_temp, &ep_ring->td_list,
 381                                          td_list) {
 382                         if (td->drbl)
 383                                 continue;
 384
 385                         ret = cdnsp_ring_ep_doorbell(pdev, pep, stream_id);
 386                         if (ret)
 387                                 td->drbl = 1;
 388                 }
 389         }
 390 }
 391
 392 /*
 393  * Get the hw dequeue pointer controller stopped on, either directly from the
 394  * endpoint context, or if streams are in use from the stream context.
 395  * The returned hw_dequeue contains the lowest four bits with cycle state
 396  * and possible stream context type.
 397  */
 398 static u64 cdnsp_get_hw_deq(struct cdnsp_device *pdev,
 399                             unsigned int ep_index,
 400                             unsigned int stream_id)
 401 {
 402         struct cdnsp_stream_ctx *st_ctx;
 403         struct cdnsp_ep *pep;
 404
 405         pep = &pdev->eps[stream_id];
 406
 407         if (pep->ep_state & EP_HAS_STREAMS) {
 408                 st_ctx = &pep->stream_info.stream_ctx_array[stream_id];
 409                 return le64_to_cpu(st_ctx->stream_ring);
 410         }
 411
 412         return le64_to_cpu(pep->out_ctx->deq);
 413 }
 414
 415 /*
 416  * Move the controller endpoint ring dequeue pointer past cur_td.
 417  * Record the new state of the controller endpoint ring dequeue segment,
 418  * dequeue pointer, and new consumer cycle state in state.
 419  * Update internal representation of the ring's dequeue pointer.
 420  *
 421  * We do this in three jumps:
 422  *  - First we update our new ring state to be the same as when the
 423  *    controller stopped.
 424  *  - Then we traverse the ring to find the segment that contains
 425  *    the last TRB in the TD. We toggle the controller new cycle state
 426  *    when we pass any link TRBs with the toggle cycle bit set.
 427  *  - Finally we move the dequeue state one TRB further, toggling the cycle bit
 428  *    if we've moved it past a link TRB with the toggle cycle bit set.
 429  */
 430 static void cdnsp_find_new_dequeue_state(struct cdnsp_device *pdev,
 431                                          struct cdnsp_ep *pep,
 432                                          unsigned int stream_id,
 433                                          struct cdnsp_td *cur_td,
 434                                          struct cdnsp_dequeue_state *state)
 435 {
 436         bool td_last_trb_found = false;
 437         struct cdnsp_segment *new_seg;
 438         struct cdnsp_ring *ep_ring;
 439         union cdnsp_trb *new_deq;
 440         bool cycle_found = false;
 441         u64 hw_dequeue;
 442
 443         ep_ring = cdnsp_get_transfer_ring(pdev, pep, stream_id);
 444         if (!ep_ring)
 445                 return;
 446
 447         /*
 448          * Dig out the cycle state saved by the controller during the
 449          * stop endpoint command.
 450          */
 451         hw_dequeue = cdnsp_get_hw_deq(pdev, pep->idx, stream_id);
 452         new_seg = ep_ring->deq_seg;
 453         new_deq = ep_ring->dequeue;
 454         state->new_cycle_state = hw_dequeue & 0x1;
 455         state->stream_id = stream_id;
 456
 457         /*
 458          * We want to find the pointer, segment and cycle state of the new trb
 459          * (the one after current TD's last_trb). We know the cycle state at
 460          * hw_dequeue, so walk the ring until both hw_dequeue and last_trb are
 461          * found.
 462          */
 463         do {
 464                 if (!cycle_found && cdnsp_trb_virt_to_dma(new_seg, new_deq)
 465                     == (dma_addr_t)(hw_dequeue & ~0xf)) {
 466                         cycle_found = true;
 467
 468                         if (td_last_trb_found)
 469                                 break;
 470                 }
 471
 472                 if (new_deq == cur_td->last_trb)
 473                         td_last_trb_found = true;
 474
 475                 if (cycle_found && cdnsp_trb_is_link(new_deq) &&
 476                     cdnsp_link_trb_toggles_cycle(new_deq))
 477                         state->new_cycle_state ^= 0x1;
 478
 479                 cdnsp_next_trb(pdev, ep_ring, &new_seg, &new_deq);
 480
 481                 /* Search wrapped around, bail out. */
 482                 if (new_deq == pep->ring->dequeue) {
 483                         dev_err(pdev->dev,
 484                                 "Error: Failed finding new dequeue state\n");
 485                         state->new_deq_seg = NULL;
 486                         state->new_deq_ptr = NULL;
 487                         return;
 488                 }
 489
 490         } while (!cycle_found || !td_last_trb_found);
 491
 492         state->new_deq_seg = new_seg;
 493         state->new_deq_ptr = new_deq;
 494
 495         trace_cdnsp_new_deq_state(state);
 496 }
 497
 498 /*
 499  * flip_cycle means flip the cycle bit of all but the first and last TRB.
 500  * (The last TRB actually points to the ring enqueue pointer, which is not part
 501  * of this TD.) This is used to remove partially enqueued isoc TDs from a ring.
 502  */
 503 static void cdnsp_td_to_noop(struct cdnsp_device *pdev,
 504                              struct cdnsp_ring *ep_ring,
 505                              struct cdnsp_td *td,
 506                              bool flip_cycle)
 507 {
 508         struct cdnsp_segment *seg = td->start_seg;
 509         union cdnsp_trb *trb = td->first_trb;
 510
 511         while (1) {
 512                 cdnsp_trb_to_noop(trb, TRB_TR_NOOP);
 513
 514                 /* flip cycle if asked to */
 515                 if (flip_cycle && trb != td->first_trb && trb != td->last_trb)
 516                         trb->generic.field[3] ^= cpu_to_le32(TRB_CYCLE);
 517
 518                 if (trb == td->last_trb)
 519                         break;
 520
 521                 cdnsp_next_trb(pdev, ep_ring, &seg, &trb);
 522         }
 523 }
 524
 525 /*
 526  * This TD is defined by the TRBs starting at start_trb in start_seg and ending
 527  * at end_trb, which may be in another segment. If the suspect DMA address is a
 528  * TRB in this TD, this function returns that TRB's segment. Otherwise it
 529  * returns 0.
 530  */
 531 static struct cdnsp_segment *cdnsp_trb_in_td(struct cdnsp_device *pdev,
 532                                              struct cdnsp_segment *start_seg,
 533                                              union cdnsp_trb *start_trb,
 534                                              union cdnsp_trb *end_trb,
 535                                              dma_addr_t suspect_dma)
 536 {
 537         struct cdnsp_segment *cur_seg;
 538         union cdnsp_trb *temp_trb;
 539         dma_addr_t end_seg_dma;
 540         dma_addr_t end_trb_dma;
 541         dma_addr_t start_dma;
 542
 543         start_dma = cdnsp_trb_virt_to_dma(start_seg, start_trb);
 544         cur_seg = start_seg;
 545
 546         do {
 547                 if (start_dma == 0)
 548                         return NULL;
 549
 550                 temp_trb = &cur_seg->trbs[TRBS_PER_SEGMENT - 1];
 551                 /* We may get an event for a Link TRB in the middle of a TD */
 552                 end_seg_dma = cdnsp_trb_virt_to_dma(cur_seg, temp_trb);
 553                 /* If the end TRB isn't in this segment, this is set to 0 */
 554                 end_trb_dma = cdnsp_trb_virt_to_dma(cur_seg, end_trb);
 555
 556                 trace_cdnsp_looking_trb_in_td(suspect_dma, start_dma,
 557                                               end_trb_dma, cur_seg->dma,
 558                                               end_seg_dma);
 559
 560                 if (end_trb_dma > 0) {
 561                         /*
 562                          * The end TRB is in this segment, so suspect should
 563                          * be here
 564                          */
 565                         if (start_dma <= end_trb_dma) {
 566                                 if (suspect_dma >= start_dma &&
 567                                     suspect_dma <= end_trb_dma) {
 568                                         return cur_seg;
 569                                 }
 570                         } else {
 571                                 /*
 572                                  * Case for one segment with a
 573                                  * TD wrapped around to the top
 574                                  */
 575                                 if ((suspect_dma >= start_dma &&
 576                                      suspect_dma <= end_seg_dma) ||
 577                                     (suspect_dma >= cur_seg->dma &&
 578                                      suspect_dma <= end_trb_dma)) {
 579                                         return cur_seg;
 580                                 }
 581                         }
 582
 583                         return NULL;
 584                 }
 585
 586                 /* Might still be somewhere in this segment */
 587                 if (suspect_dma >= start_dma && suspect_dma <= end_seg_dma)
 588                         return cur_seg;
 589
 590                 cur_seg = cur_seg->next;
 591                 start_dma = cdnsp_trb_virt_to_dma(cur_seg, &cur_seg->trbs[0]);
 592         } while (cur_seg != start_seg);
 593
 594         return NULL;
 595 }
 596
 597 static void cdnsp_unmap_td_bounce_buffer(struct cdnsp_device *pdev,
 598                                          struct cdnsp_ring *ring,
 599                                          struct cdnsp_td *td)
 600 {
 601         struct cdnsp_segment *seg = td->bounce_seg;
 602         struct cdnsp_request *preq;
 603         size_t len;
 604
 605         if (!seg)
 606                 return;
 607
 608         preq = td->preq;
 609
 610         trace_cdnsp_bounce_unmap(td->preq, seg->bounce_len, seg->bounce_offs,
 611                                  seg->bounce_dma, 0);
 612
 613         if (!preq->direction) {
 614                 dma_unmap_single(pdev->dev, seg->bounce_dma,
 615                                  ring->bounce_buf_len,  DMA_TO_DEVICE);
 616                 return;
 617         }
 618
 619         dma_unmap_single(pdev->dev, seg->bounce_dma, ring->bounce_buf_len,
 620                          DMA_FROM_DEVICE);
 621
 622         /* For in transfers we need to copy the data from bounce to sg */
 623         len = sg_pcopy_from_buffer(preq->request.sg, preq->request.num_sgs,
 624                                    seg->bounce_buf, seg->bounce_len,
 625                                    seg->bounce_offs);
 626         if (len != seg->bounce_len)
 627                 dev_warn(pdev->dev, "WARN Wrong bounce buffer read length: %zu != %d\n",
 628                          len, seg->bounce_len);
 629
 630         seg->bounce_len = 0;
 631         seg->bounce_offs = 0;
 632 }
 633
 634 static int cdnsp_cmd_set_deq(struct cdnsp_device *pdev,
 635                              struct cdnsp_ep *pep,
 636                              struct cdnsp_dequeue_state *deq_state)
 637 {
 638         struct cdnsp_ring *ep_ring;
 639         int ret;
 640
 641         if (!deq_state->new_deq_ptr || !deq_state->new_deq_seg) {
 642                 cdnsp_ring_doorbell_for_active_rings(pdev, pep);
 643                 return 0;
 644         }
 645
 646         cdnsp_queue_new_dequeue_state(pdev, pep, deq_state);
 647         cdnsp_ring_cmd_db(pdev);
 648         ret = cdnsp_wait_for_cmd_compl(pdev);
 649
 650         trace_cdnsp_handle_cmd_set_deq(cdnsp_get_slot_ctx(&pdev->out_ctx));
 651         trace_cdnsp_handle_cmd_set_deq_ep(pep->out_ctx);
 652
 653         /*
 654          * Update the ring's dequeue segment and dequeue pointer
 655          * to reflect the new position.
 656          */
 657         ep_ring = cdnsp_get_transfer_ring(pdev, pep, deq_state->stream_id);
 658
 659         if (cdnsp_trb_is_link(ep_ring->dequeue)) {
 660                 ep_ring->deq_seg = ep_ring->deq_seg->next;
 661                 ep_ring->dequeue = ep_ring->deq_seg->trbs;
 662         }
 663
 664         while (ep_ring->dequeue != deq_state->new_deq_ptr) {
 665                 ep_ring->num_trbs_free++;
 666                 ep_ring->dequeue++;
 667
 668                 if (cdnsp_trb_is_link(ep_ring->dequeue)) {
 669                         if (ep_ring->dequeue == deq_state->new_deq_ptr)
 670                                 break;
 671
 672                         ep_ring->deq_seg = ep_ring->deq_seg->next;
 673                         ep_ring->dequeue = ep_ring->deq_seg->trbs;
 674                 }
 675         }
 676
 677         /*
 678          * Probably there was TIMEOUT during handling Set Dequeue Pointer
 679          * command. It's critical error and controller will be stopped.
 680          */
 681         if (ret)
 682                 return -ESHUTDOWN;
 683
 684         /* Restart any rings with pending requests */
 685         cdnsp_ring_doorbell_for_active_rings(pdev, pep);
 686
 687         return 0;
 688 }
 689
 690 int cdnsp_remove_request(struct cdnsp_device *pdev,
 691                          struct cdnsp_request *preq,
 692                          struct cdnsp_ep *pep)
 693 {
 694         struct cdnsp_dequeue_state deq_state;
 695         struct cdnsp_td *cur_td = NULL;
 696         struct cdnsp_ring *ep_ring;
 697         struct cdnsp_segment *seg;
 698         int status = -ECONNRESET;
 699         int ret = 0;
 700         u64 hw_deq;
 701
 702         memset(&deq_state, 0, sizeof(deq_state));
 703
 704         trace_cdnsp_remove_request(pep->out_ctx);
 705         trace_cdnsp_remove_request_td(preq);
 706
 707         cur_td = &preq->td;
 708         ep_ring = cdnsp_request_to_transfer_ring(pdev, preq);
 709
 710         /*
 711          * If we stopped on the TD we need to cancel, then we have to
 712          * move the controller endpoint ring dequeue pointer past
 713          * this TD.
 714          */
 715         hw_deq = cdnsp_get_hw_deq(pdev, pep->idx, preq->request.stream_id);
 716         hw_deq &= ~0xf;
 717
 718         seg = cdnsp_trb_in_td(pdev, cur_td->start_seg, cur_td->first_trb,
 719                               cur_td->last_trb, hw_deq);
 720
 721         if (seg && (pep->ep_state & EP_ENABLED))
 722                 cdnsp_find_new_dequeue_state(pdev, pep, preq->request.stream_id,
 723                                              cur_td, &deq_state);
 724         else
 725                 cdnsp_td_to_noop(pdev, ep_ring, cur_td, false);
 726
 727         /*
 728          * The event handler won't see a completion for this TD anymore,
 729          * so remove it from the endpoint ring's TD list.
 730          */
 731         list_del_init(&cur_td->td_list);
 732         ep_ring->num_tds--;
 733         pep->stream_info.td_count--;
 734
 735         /*
 736          * During disconnecting all endpoint will be disabled so we don't
 737          * have to worry about updating dequeue pointer.
 738          */
 739         if (pdev->cdnsp_state & CDNSP_STATE_DISCONNECT_PENDING) {
 740                 status = -ESHUTDOWN;
 741                 ret = cdnsp_cmd_set_deq(pdev, pep, &deq_state);
 742         }
 743
 744         cdnsp_unmap_td_bounce_buffer(pdev, ep_ring, cur_td);
 745         cdnsp_gadget_giveback(pep, cur_td->preq, status);
 746
 747         return ret;
 748 }
 749
 750 static int cdnsp_update_port_id(struct cdnsp_device *pdev, u32 port_id)
 751 {
 752         struct cdnsp_port *port = pdev->active_port;
 753         u8 old_port = 0;
 754
 755         if (port && port->port_num == port_id)
 756                 return 0;
 757
 758         if (port)
 759                 old_port = port->port_num;
 760
 761         if (port_id == pdev->usb2_port.port_num) {
 762                 port = &pdev->usb2_port;
 763         } else if (port_id == pdev->usb3_port.port_num) {
 764                 port  = &pdev->usb3_port;
 765         } else {
 766                 dev_err(pdev->dev, "Port event with invalid port ID %d\n",
 767                         port_id);
 768                 return -EINVAL;
 769         }
 770
 771         if (port_id != old_port) {
 772                 cdnsp_disable_slot(pdev);
 773                 pdev->active_port = port;
 774                 cdnsp_enable_slot(pdev);
 775         }
 776
 777         if (port_id == pdev->usb2_port.port_num)
 778                 cdnsp_set_usb2_hardware_lpm(pdev, NULL, 1);
 779         else
 780                 writel(PORT_U1_TIMEOUT(1) | PORT_U2_TIMEOUT(1),
 781                        &pdev->usb3_port.regs->portpmsc);
 782
 783         return 0;
 784 }
 785
 786 static void cdnsp_handle_port_status(struct cdnsp_device *pdev,
 787                                      union cdnsp_trb *event)
 788 {
 789         struct cdnsp_port_regs __iomem *port_regs;
 790         u32 portsc, cmd_regs;
 791         bool port2 = false;
 792         u32 link_state;
 793         u32 port_id;
 794
 795         /* Port status change events always have a successful completion code */
 796         if (GET_COMP_CODE(le32_to_cpu(event->generic.field[2])) != COMP_SUCCESS)
 797                 dev_err(pdev->dev, "ERR: incorrect PSC event\n");
 798
 799         port_id = GET_PORT_ID(le32_to_cpu(event->generic.field[0]));
 800
 801         if (cdnsp_update_port_id(pdev, port_id))
 802                 goto cleanup;
 803
 804         port_regs = pdev->active_port->regs;
 805
 806         if (port_id == pdev->usb2_port.port_num)
 807                 port2 = true;
 808
 809 new_event:
 810         portsc = readl(&port_regs->portsc);
 811         writel(cdnsp_port_state_to_neutral(portsc) |
 812                (portsc & PORT_CHANGE_BITS), &port_regs->portsc);
 813
 814         trace_cdnsp_handle_port_status(pdev->active_port->port_num, portsc);
 815
 816         pdev->gadget.speed = cdnsp_port_speed(portsc);
 817         link_state = portsc & PORT_PLS_MASK;
 818
 819         /* Port Link State change detected. */
 820         if ((portsc & PORT_PLC)) {
 821                 if (!(pdev->cdnsp_state & CDNSP_WAKEUP_PENDING)  &&
 822                     link_state == XDEV_RESUME) {
 823                         cmd_regs = readl(&pdev->op_regs->command);
 824                         if (!(cmd_regs & CMD_R_S))
 825                                 goto cleanup;
 826
 827                         if (DEV_SUPERSPEED_ANY(portsc)) {
 828                                 cdnsp_set_link_state(pdev, &port_regs->portsc,
 829                                                      XDEV_U0);
 830
 831                                 cdnsp_resume_gadget(pdev);
 832                         }
 833                 }
 834
 835                 if ((pdev->cdnsp_state & CDNSP_WAKEUP_PENDING) &&
 836                     link_state == XDEV_U0) {
 837                         pdev->cdnsp_state &= ~CDNSP_WAKEUP_PENDING;
 838
 839                         cdnsp_force_header_wakeup(pdev, 1);
 840                         cdnsp_ring_cmd_db(pdev);
 841                         cdnsp_wait_for_cmd_compl(pdev);
 842                 }
 843
 844                 if (link_state == XDEV_U0 && pdev->link_state == XDEV_U3 &&
 845                     !DEV_SUPERSPEED_ANY(portsc))
 846                         cdnsp_resume_gadget(pdev);
 847
 848                 if (link_state == XDEV_U3 &&  pdev->link_state != XDEV_U3)
 849                         cdnsp_suspend_gadget(pdev);
 850
 851                 pdev->link_state = link_state;
 852         }
 853
 854         if (portsc & PORT_CSC) {
 855                 /* Detach device. */
 856                 if (pdev->gadget.connected && !(portsc & PORT_CONNECT))
 857                         cdnsp_disconnect_gadget(pdev);
 858
 859                 /* Attach device. */
 860                 if (portsc & PORT_CONNECT) {
 861                         if (!port2)
 862                                 cdnsp_irq_reset(pdev);
 863
 864                         usb_gadget_set_state(&pdev->gadget, USB_STATE_ATTACHED);
 865                 }
 866         }
 867
 868         /* Port reset. */
 869         if ((portsc & (PORT_RC | PORT_WRC)) && (portsc & PORT_CONNECT)) {
 870                 cdnsp_irq_reset(pdev);
 871                 pdev->u1_allowed = 0;
 872                 pdev->u2_allowed = 0;
 873                 pdev->may_wakeup = 0;
 874         }
 875
 876         if (portsc & PORT_CEC)
 877                 dev_err(pdev->dev, "Port Over Current detected\n");
 878
 879         if (portsc & PORT_CEC)
 880                 dev_err(pdev->dev, "Port Configure Error detected\n");
 881
 882         if (readl(&port_regs->portsc) & PORT_CHANGE_BITS)
 883                 goto new_event;
 884
 885 cleanup:
 886         cdnsp_inc_deq(pdev, pdev->event_ring);
 887 }
 888
 889 static void cdnsp_td_cleanup(struct cdnsp_device *pdev,
 890                              struct cdnsp_td *td,
 891                              struct cdnsp_ring *ep_ring,
 892                              int *status)
 893 {
 894         struct cdnsp_request *preq = td->preq;
 895
 896         /* if a bounce buffer was used to align this td then unmap it */
 897         cdnsp_unmap_td_bounce_buffer(pdev, ep_ring, td);
 898
 899         /*
 900          * If the controller said we transferred more data than the buffer
 901          * length, Play it safe and say we didn't transfer anything.
 902          */
 903         if (preq->request.actual > preq->request.length) {
 904                 preq->request.actual = 0;
 905                 *status = 0;
 906         }
 907
 908         list_del_init(&td->td_list);
 909         ep_ring->num_tds--;
 910         preq->pep->stream_info.td_count--;
 911
 912         cdnsp_gadget_giveback(preq->pep, preq, *status);
 913 }
 914
 915 static void cdnsp_finish_td(struct cdnsp_device *pdev,
 916                             struct cdnsp_td *td,
 917                             struct cdnsp_transfer_event *event,
 918                             struct cdnsp_ep *ep,
 919                             int *status)
 920 {
 921         struct cdnsp_ring *ep_ring;
 922         u32 trb_comp_code;
 923
 924         ep_ring = cdnsp_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
 925         trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
 926
 927         if (trb_comp_code == COMP_STOPPED_LENGTH_INVALID ||
 928             trb_comp_code == COMP_STOPPED ||
 929             trb_comp_code == COMP_STOPPED_SHORT_PACKET) {
 930                 /*
 931                  * The Endpoint Stop Command completion will take care of any
 932                  * stopped TDs. A stopped TD may be restarted, so don't update
 933                  * the ring dequeue pointer or take this TD off any lists yet.
 934                  */
 935                 return;
 936         }
 937
 938         /* Update ring dequeue pointer */
 939         while (ep_ring->dequeue != td->last_trb)
 940                 cdnsp_inc_deq(pdev, ep_ring);
 941
 942         cdnsp_inc_deq(pdev, ep_ring);
 943
 944         cdnsp_td_cleanup(pdev, td, ep_ring, status);
 945 }
 946
 947 /* sum trb lengths from ring dequeue up to stop_trb, _excluding_ stop_trb */
 948 static int cdnsp_sum_trb_lengths(struct cdnsp_device *pdev,
 949                                  struct cdnsp_ring *ring,
 950                                  union cdnsp_trb *stop_trb)
 951 {
 952         struct cdnsp_segment *seg = ring->deq_seg;
 953         union cdnsp_trb *trb = ring->dequeue;
 954         u32 sum;
 955
 956         for (sum = 0; trb != stop_trb; cdnsp_next_trb(pdev, ring, &seg, &trb)) {
 957                 if (!cdnsp_trb_is_noop(trb) && !cdnsp_trb_is_link(trb))
 958                         sum += TRB_LEN(le32_to_cpu(trb->generic.field[2]));
 959         }
 960         return sum;
 961 }
 962
 963 static int cdnsp_giveback_first_trb(struct cdnsp_device *pdev,
 964                                     struct cdnsp_ep *pep,
 965                                     unsigned int stream_id,
 966                                     int start_cycle,
 967                                     struct cdnsp_generic_trb *start_trb)
 968 {
 969         /*
 970          * Pass all the TRBs to the hardware at once and make sure this write
 971          * isn't reordered.
 972          */
 973         wmb();
 974
 975         if (start_cycle)
 976                 start_trb->field[3] |= cpu_to_le32(start_cycle);
 977         else
 978                 start_trb->field[3] &= cpu_to_le32(~TRB_CYCLE);
 979
 980         if ((pep->ep_state & EP_HAS_STREAMS) &&
 981             !pep->stream_info.first_prime_det) {
 982                 trace_cdnsp_wait_for_prime(pep, stream_id);
 983                 return 0;
 984         }
 985
 986         return cdnsp_ring_ep_doorbell(pdev, pep, stream_id);
 987 }
 988
 989 /*
 990  * Process control tds, update USB request status and actual_length.
 991  */
 992 static void cdnsp_process_ctrl_td(struct cdnsp_device *pdev,
 993                                   struct cdnsp_td *td,
 994                                   union cdnsp_trb *event_trb,
 995                                   struct cdnsp_transfer_event *event,
 996                                   struct cdnsp_ep *pep,
 997                                   int *status)
 998 {
 999         struct cdnsp_ring *ep_ring;
1000         u32 remaining;
1001         u32 trb_type;
1002
1003         trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(event_trb->generic.field[3]));
1004         ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer));
1005         remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
1006
1007         /*
1008          * if on data stage then update the actual_length of the USB
1009          * request and flag it as set, so it won't be overwritten in the event
1010          * for the last TRB.
1011          */
1012         if (trb_type == TRB_DATA) {
1013                 td->request_length_set = true;
1014                 td->preq->request.actual = td->preq->request.length - remaining;
1015         }
1016
1017         /* at status stage */
1018         if (!td->request_length_set)
1019                 td->preq->request.actual = td->preq->request.length;
1020
1021         if (pdev->ep0_stage == CDNSP_DATA_STAGE && pep->number == 0 &&
1022             pdev->three_stage_setup) {
1023                 td = list_entry(ep_ring->td_list.next, struct cdnsp_td,
1024                                 td_list);
1025                 pdev->ep0_stage = CDNSP_STATUS_STAGE;
1026
1027                 cdnsp_giveback_first_trb(pdev, pep, 0, ep_ring->cycle_state,
1028                                          &td->last_trb->generic);
1029                 return;
1030         }
1031
1032         cdnsp_finish_td(pdev, td, event, pep, status);
1033 }
1034
1035 /*
1036  * Process isochronous tds, update usb request status and actual_length.
1037  */
1038 static void cdnsp_process_isoc_td(struct cdnsp_device *pdev,
1039                                   struct cdnsp_td *td,
1040                                   union cdnsp_trb *ep_trb,
1041                                   struct cdnsp_transfer_event *event,
1042                                   struct cdnsp_ep *pep,
1043                                   int status)
1044 {
1045         struct cdnsp_request *preq = td->preq;
1046         u32 remaining, requested, ep_trb_len;
1047         bool sum_trbs_for_length = false;
1048         struct cdnsp_ring *ep_ring;
1049         u32 trb_comp_code;
1050         u32 td_length;
1051
1052         ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer));
1053         trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
1054         remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
1055         ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2]));
1056
1057         requested = preq->request.length;
1058
1059         /* handle completion code */
1060         switch (trb_comp_code) {
1061         case COMP_SUCCESS:
1062                 preq->request.status = 0;
1063                 break;
1064         case COMP_SHORT_PACKET:
1065                 preq->request.status = 0;
1066                 sum_trbs_for_length = true;
1067                 break;
1068         case COMP_ISOCH_BUFFER_OVERRUN:
1069         case COMP_BABBLE_DETECTED_ERROR:
1070                 preq->request.status = -EOVERFLOW;
1071                 break;
1072         case COMP_STOPPED:
1073                 sum_trbs_for_length = true;
1074                 break;
1075         case COMP_STOPPED_SHORT_PACKET:
1076                 /* field normally containing residue now contains transferred */
1077                 preq->request.status  = 0;
1078                 requested = remaining;
1079                 break;
1080         case COMP_STOPPED_LENGTH_INVALID:
1081                 requested = 0;
1082                 remaining = 0;
1083                 break;
1084         default:
1085                 sum_trbs_for_length = true;
1086                 preq->request.status = -1;
1087                 break;
1088         }
1089
1090         if (sum_trbs_for_length) {
1091                 td_length = cdnsp_sum_trb_lengths(pdev, ep_ring, ep_trb);
1092                 td_length += ep_trb_len - remaining;
1093         } else {
1094                 td_length = requested;
1095         }
1096
1097         td->preq->request.actual += td_length;
1098
1099         cdnsp_finish_td(pdev, td, event, pep, &status);
1100 }
1101
1102 static void cdnsp_skip_isoc_td(struct cdnsp_device *pdev,
1103                                struct cdnsp_td *td,
1104                                struct cdnsp_transfer_event *event,
1105                                struct cdnsp_ep *pep,
1106                                int status)
1107 {
1108         struct cdnsp_ring *ep_ring;
1109
1110         ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer));
1111         td->preq->request.status = -EXDEV;
1112         td->preq->request.actual = 0;
1113
1114         /* Update ring dequeue pointer */
1115         while (ep_ring->dequeue != td->last_trb)
1116                 cdnsp_inc_deq(pdev, ep_ring);
1117
1118         cdnsp_inc_deq(pdev, ep_ring);
1119
1120         cdnsp_td_cleanup(pdev, td, ep_ring, &status);
1121 }
1122
1123 /*
1124  * Process bulk and interrupt tds, update usb request status and actual_length.
1125  */
1126 static void cdnsp_process_bulk_intr_td(struct cdnsp_device *pdev,
1127                                        struct cdnsp_td *td,
1128                                        union cdnsp_trb *ep_trb,
1129                                        struct cdnsp_transfer_event *event,
1130                                        struct cdnsp_ep *ep,
1131                                        int *status)
1132 {
1133         u32 remaining, requested, ep_trb_len;
1134         struct cdnsp_ring *ep_ring;
1135         u32 trb_comp_code;
1136
1137         ep_ring = cdnsp_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
1138         trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
1139         remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
1140         ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2]));
1141         requested = td->preq->request.length;
1142
1143         switch (trb_comp_code) {
1144         case COMP_SUCCESS:
1145         case COMP_SHORT_PACKET:
1146                 *status = 0;
1147                 break;
1148         case COMP_STOPPED_SHORT_PACKET:
1149                 td->preq->request.actual = remaining;
1150                 goto finish_td;
1151         case COMP_STOPPED_LENGTH_INVALID:
1152                 /* Stopped on ep trb with invalid length, exclude it. */
1153                 ep_trb_len = 0;
1154                 remaining = 0;
1155                 break;
1156         }
1157
1158         if (ep_trb == td->last_trb)
1159                 ep_trb_len = requested - remaining;
1160         else
1161                 ep_trb_len = cdnsp_sum_trb_lengths(pdev, ep_ring, ep_trb) +
1162                                                    ep_trb_len - remaining;
1163         td->preq->request.actual = ep_trb_len;
1164
1165 finish_td:
1166         ep->stream_info.drbls_count--;
1167
1168         cdnsp_finish_td(pdev, td, event, ep, status);
1169 }
1170
1171 static void cdnsp_handle_tx_nrdy(struct cdnsp_device *pdev,
1172                                  struct cdnsp_transfer_event *event)
1173 {
1174         struct cdnsp_generic_trb *generic;
1175         struct cdnsp_ring *ep_ring;
1176         struct cdnsp_ep *pep;
1177         int cur_stream;
1178         int ep_index;
1179         int host_sid;
1180         int dev_sid;
1181
1182         generic = (struct cdnsp_generic_trb *)event;
1183         ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
1184         dev_sid = TRB_TO_DEV_STREAM(le32_to_cpu(generic->field[0]));
1185         host_sid = TRB_TO_HOST_STREAM(le32_to_cpu(generic->field[2]));
1186
1187         pep = &pdev->eps[ep_index];
1188
1189         if (!(pep->ep_state & EP_HAS_STREAMS))
1190                 return;
1191
1192         if (host_sid == STREAM_PRIME_ACK) {
1193                 pep->stream_info.first_prime_det = 1;
1194                 for (cur_stream = 1; cur_stream < pep->stream_info.num_streams;
1195                     cur_stream++) {
1196                         ep_ring = pep->stream_info.stream_rings[cur_stream];
1197                         ep_ring->stream_active = 1;
1198                         ep_ring->stream_rejected = 0;
1199                 }
1200         }
1201
1202         if (host_sid == STREAM_REJECTED) {
1203                 struct cdnsp_td *td, *td_temp;
1204
1205                 pep->stream_info.drbls_count--;
1206                 ep_ring = pep->stream_info.stream_rings[dev_sid];
1207                 ep_ring->stream_active = 0;
1208                 ep_ring->stream_rejected = 1;
1209
1210                 list_for_each_entry_safe(td, td_temp, &ep_ring->td_list,
1211                                          td_list) {
1212                         td->drbl = 0;
1213                 }
1214         }
1215
1216         cdnsp_ring_doorbell_for_active_rings(pdev, pep);
1217 }
1218
1219 /*
1220  * If this function returns an error condition, it means it got a Transfer
1221  * event with a corrupted TRB DMA address or endpoint is disabled.
1222  */
1223 static int cdnsp_handle_tx_event(struct cdnsp_device *pdev,
1224                                  struct cdnsp_transfer_event *event)
1225 {
1226         const struct usb_endpoint_descriptor *desc;
1227         bool handling_skipped_tds = false;
1228         struct cdnsp_segment *ep_seg;
1229         struct cdnsp_ring *ep_ring;
1230         int status = -EINPROGRESS;
1231         union cdnsp_trb *ep_trb;
1232         dma_addr_t ep_trb_dma;
1233         struct cdnsp_ep *pep;
1234         struct cdnsp_td *td;
1235         u32 trb_comp_code;
1236         int invalidate;
1237         int ep_index;
1238
1239         invalidate = le32_to_cpu(event->flags) & TRB_EVENT_INVALIDATE;
1240         ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
1241         trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
1242         ep_trb_dma = le64_to_cpu(event->buffer);
1243
1244         pep = &pdev->eps[ep_index];
1245         ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer));
1246
1247         /*
1248          * If device is disconnect then all requests will be dequeued
1249          * by upper layers as part of disconnect sequence.
1250          * We don't want handle such event to avoid racing.
1251          */
1252         if (invalidate || !pdev->gadget.connected)
1253                 goto cleanup;
1254
1255         if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_DISABLED) {
1256                 trace_cdnsp_ep_disabled(pep->out_ctx);
1257                 goto err_out;
1258         }
1259
1260         /* Some transfer events don't always point to a trb*/
1261         if (!ep_ring) {
1262                 switch (trb_comp_code) {
1263                 case COMP_INVALID_STREAM_TYPE_ERROR:
1264                 case COMP_INVALID_STREAM_ID_ERROR:
1265                 case COMP_RING_UNDERRUN:
1266                 case COMP_RING_OVERRUN:
1267                         goto cleanup;
1268                 default:
1269                         dev_err(pdev->dev, "ERROR: %s event for unknown ring\n",
1270                                 pep->name);
1271                         goto err_out;
1272                 }
1273         }
1274
1275         /* Look for some error cases that need special treatment. */
1276         switch (trb_comp_code) {
1277         case COMP_BABBLE_DETECTED_ERROR:
1278                 status = -EOVERFLOW;
1279                 break;
1280         case COMP_RING_UNDERRUN:
1281         case COMP_RING_OVERRUN:
1282                 /*
1283                  * When the Isoch ring is empty, the controller will generate
1284                  * a Ring Overrun Event for IN Isoch endpoint or Ring
1285                  * Underrun Event for OUT Isoch endpoint.
1286                  */
1287                 goto cleanup;
1288         case COMP_MISSED_SERVICE_ERROR:
1289                 /*
1290                  * When encounter missed service error, one or more isoc tds
1291                  * may be missed by controller.
1292                  * Set skip flag of the ep_ring; Complete the missed tds as
1293                  * short transfer when process the ep_ring next time.
1294                  */
1295                 pep->skip = true;
1296                 break;
1297         }
1298
1299         do {
1300                 /*
1301                  * This TRB should be in the TD at the head of this ring's TD
1302                  * list.
1303                  */
1304                 if (list_empty(&ep_ring->td_list)) {
1305                         /*
1306                          * Don't print warnings if it's due to a stopped
1307                          * endpoint generating an extra completion event, or
1308                          * a event for the last TRB of a short TD we already
1309                          * got a short event for.
1310                          * The short TD is already removed from the TD list.
1311                          */
1312                         if (!(trb_comp_code == COMP_STOPPED ||
1313                               trb_comp_code == COMP_STOPPED_LENGTH_INVALID ||
1314                               ep_ring->last_td_was_short))
1315                                 trace_cdnsp_trb_without_td(ep_ring,
1316                                         (struct cdnsp_generic_trb *)event);
1317
1318                         if (pep->skip) {
1319                                 pep->skip = false;
1320                                 trace_cdnsp_ep_list_empty_with_skip(pep, 0);
1321                         }
1322
1323                         goto cleanup;
1324                 }
1325
1326                 td = list_entry(ep_ring->td_list.next, struct cdnsp_td,
1327                                 td_list);
1328
1329                 /* Is this a TRB in the currently executing TD? */
1330                 ep_seg = cdnsp_trb_in_td(pdev, ep_ring->deq_seg,
1331                                          ep_ring->dequeue, td->last_trb,
1332                                          ep_trb_dma);
1333
1334                 /*
1335                  * Skip the Force Stopped Event. The event_trb(ep_trb_dma)
1336                  * of FSE is not in the current TD pointed by ep_ring->dequeue
1337                  * because that the hardware dequeue pointer still at the
1338                  * previous TRB of the current TD. The previous TRB maybe a
1339                  * Link TD or the last TRB of the previous TD. The command
1340                  * completion handle will take care the rest.
1341                  */
1342                 if (!ep_seg && (trb_comp_code == COMP_STOPPED ||
1343                                 trb_comp_code == COMP_STOPPED_LENGTH_INVALID)) {
1344                         pep->skip = false;
1345                         goto cleanup;
1346                 }
1347
1348                 desc = td->preq->pep->endpoint.desc;
1349                 if (!ep_seg) {
1350                         if (!pep->skip || !usb_endpoint_xfer_isoc(desc)) {
1351                                 /* Something is busted, give up! */
1352                                 dev_err(pdev->dev,
1353                                         "ERROR Transfer event TRB DMA ptr not "
1354                                         "part of current TD ep_index %d "
1355                                         "comp_code %u\n", ep_index,
1356                                         trb_comp_code);
1357                                 return -EINVAL;
1358                         }
1359
1360                         cdnsp_skip_isoc_td(pdev, td, event, pep, status);
1361                         goto cleanup;
1362                 }
1363
1364                 if (trb_comp_code == COMP_SHORT_PACKET)
1365                         ep_ring->last_td_was_short = true;
1366                 else
1367                         ep_ring->last_td_was_short = false;
1368
1369                 if (pep->skip) {
1370                         pep->skip = false;
1371                         cdnsp_skip_isoc_td(pdev, td, event, pep, status);
1372                         goto cleanup;
1373                 }
1374
1375                 ep_trb = &ep_seg->trbs[(ep_trb_dma - ep_seg->dma)
1376                                        / sizeof(*ep_trb)];
1377
1378                 trace_cdnsp_handle_transfer(ep_ring,
1379                                             (struct cdnsp_generic_trb *)ep_trb);
1380
1381                 if (cdnsp_trb_is_noop(ep_trb))
1382                         goto cleanup;
1383
1384                 if (usb_endpoint_xfer_control(desc))
1385                         cdnsp_process_ctrl_td(pdev, td, ep_trb, event, pep,
1386                                               &status);
1387                 else if (usb_endpoint_xfer_isoc(desc))
1388                         cdnsp_process_isoc_td(pdev, td, ep_trb, event, pep,
1389                                               status);
1390                 else
1391                         cdnsp_process_bulk_intr_td(pdev, td, ep_trb, event, pep,
1392                                                    &status);
1393 cleanup:
1394                 handling_skipped_tds = pep->skip;
1395
1396                 /*
1397                  * Do not update event ring dequeue pointer if we're in a loop
1398                  * processing missed tds.
1399                  */
1400                 if (!handling_skipped_tds)
1401                         cdnsp_inc_deq(pdev, pdev->event_ring);
1402
1403         /*
1404          * If ep->skip is set, it means there are missed tds on the
1405          * endpoint ring need to take care of.
1406          * Process them as short transfer until reach the td pointed by
1407          * the event.
1408          */
1409         } while (handling_skipped_tds);
1410         return 0;
1411
1412 err_out:
1413         dev_err(pdev->dev, "@%016llx %08x %08x %08x %08x\n",
1414                 (unsigned long long)
1415                 cdnsp_trb_virt_to_dma(pdev->event_ring->deq_seg,
1416                                       pdev->event_ring->dequeue),
1417                  lower_32_bits(le64_to_cpu(event->buffer)),
1418                  upper_32_bits(le64_to_cpu(event->buffer)),
1419                  le32_to_cpu(event->transfer_len),
1420                  le32_to_cpu(event->flags));
1421         return -EINVAL;
1422 }
1423
1424 /*
1425  * This function handles all events on the event ring.
1426  * Returns true for "possibly more events to process" (caller should call
1427  * again), otherwise false if done.
1428  */
1429 static bool cdnsp_handle_event(struct cdnsp_device *pdev)
1430 {
1431         unsigned int comp_code;
1432         union cdnsp_trb *event;
1433         bool update_ptrs = true;
1434         u32 cycle_bit;
1435         int ret = 0;
1436         u32 flags;
1437
1438         event = pdev->event_ring->dequeue;
1439         flags = le32_to_cpu(event->event_cmd.flags);
1440         cycle_bit = (flags & TRB_CYCLE);
1441
1442         /* Does the controller or driver own the TRB? */
1443         if (cycle_bit != pdev->event_ring->cycle_state)
1444                 return false;
1445
1446         trace_cdnsp_handle_event(pdev->event_ring, &event->generic);
1447
1448         /*
1449          * Barrier between reading the TRB_CYCLE (valid) flag above and any
1450          * reads of the event's flags/data below.
1451          */
1452         rmb();
1453
1454         switch (flags & TRB_TYPE_BITMASK) {
1455         case TRB_TYPE(TRB_COMPLETION):
1456                 /*
1457                  * Command can't be handled in interrupt context so just
1458                  * increment command ring dequeue pointer.
1459                  */
1460                 cdnsp_inc_deq(pdev, pdev->cmd_ring);
1461                 break;
1462         case TRB_TYPE(TRB_PORT_STATUS):
1463                 cdnsp_handle_port_status(pdev, event);
1464                 update_ptrs = false;
1465                 break;
1466         case TRB_TYPE(TRB_TRANSFER):
1467                 ret = cdnsp_handle_tx_event(pdev, &event->trans_event);
1468                 if (ret >= 0)
1469                         update_ptrs = false;
1470                 break;
1471         case TRB_TYPE(TRB_SETUP):
1472                 pdev->ep0_stage = CDNSP_SETUP_STAGE;
1473                 pdev->setup_id = TRB_SETUPID_TO_TYPE(flags);
1474                 pdev->setup_speed = TRB_SETUP_SPEEDID(flags);
1475                 pdev->setup = *((struct usb_ctrlrequest *)
1476                                 &event->trans_event.buffer);
1477
1478                 cdnsp_setup_analyze(pdev);
1479                 break;
1480         case TRB_TYPE(TRB_ENDPOINT_NRDY):
1481                 cdnsp_handle_tx_nrdy(pdev, &event->trans_event);
1482                 break;
1483         case TRB_TYPE(TRB_HC_EVENT): {
1484                 comp_code = GET_COMP_CODE(le32_to_cpu(event->generic.field[2]));
1485
1486                 switch (comp_code) {
1487                 case COMP_EVENT_RING_FULL_ERROR:
1488                         dev_err(pdev->dev, "Event Ring Full\n");
1489                         break;
1490                 default:
1491                         dev_err(pdev->dev, "Controller error code 0x%02x\n",
1492                                 comp_code);
1493                 }
1494
1495                 break;
1496         }
1497         case TRB_TYPE(TRB_MFINDEX_WRAP):
1498         case TRB_TYPE(TRB_DRB_OVERFLOW):
1499                 break;
1500         default:
1501                 dev_warn(pdev->dev, "ERROR unknown event type %ld\n",
1502                          TRB_FIELD_TO_TYPE(flags));
1503         }
1504
1505         if (update_ptrs)
1506                 /* Update SW event ring dequeue pointer. */
1507                 cdnsp_inc_deq(pdev, pdev->event_ring);
1508
1509         /*
1510          * Caller will call us again to check if there are more items
1511          * on the event ring.
1512          */
1513         return true;
1514 }
1515
1516 irqreturn_t cdnsp_thread_irq_handler(int irq, void *data)
1517 {
1518         struct cdnsp_device *pdev = (struct cdnsp_device *)data;
1519         union cdnsp_trb *event_ring_deq;
1520         int counter = 0;
1521
1522         spin_lock(&pdev->lock);
1523
1524         if (pdev->cdnsp_state & (CDNSP_STATE_HALTED | CDNSP_STATE_DYING)) {
1525                 cdnsp_died(pdev);
1526                 spin_unlock(&pdev->lock);
1527                 return IRQ_HANDLED;
1528         }
1529
1530         event_ring_deq = pdev->event_ring->dequeue;
1531
1532         while (cdnsp_handle_event(pdev)) {
1533                 if (++counter >= TRBS_PER_EV_DEQ_UPDATE) {
1534                         cdnsp_update_erst_dequeue(pdev, event_ring_deq, 0);
1535                         event_ring_deq = pdev->event_ring->dequeue;
1536                         counter = 0;
1537                 }
1538         }
1539
1540         cdnsp_update_erst_dequeue(pdev, event_ring_deq, 1);
1541
1542         spin_unlock(&pdev->lock);
1543
1544         return IRQ_HANDLED;
1545 }
1546
1547 irqreturn_t cdnsp_irq_handler(int irq, void *priv)
1548 {
1549         struct cdnsp_device *pdev = (struct cdnsp_device *)priv;
1550         u32 irq_pending;
1551         u32 status;
1552
1553         status = readl(&pdev->op_regs->status);
1554
1555         if (status == ~(u32)0) {
1556                 cdnsp_died(pdev);
1557                 return IRQ_HANDLED;
1558         }
1559
1560         if (!(status & STS_EINT))
1561                 return IRQ_NONE;
1562
1563         writel(status | STS_EINT, &pdev->op_regs->status);
1564         irq_pending = readl(&pdev->ir_set->irq_pending);
1565         irq_pending |= IMAN_IP;
1566         writel(irq_pending, &pdev->ir_set->irq_pending);
1567
1568         if (status & STS_FATAL) {
1569                 cdnsp_died(pdev);
1570                 return IRQ_HANDLED;
1571         }
1572
1573         return IRQ_WAKE_THREAD;
1574 }
1575
1576 /*
1577  * Generic function for queuing a TRB on a ring.
1578  * The caller must have checked to make sure there's room on the ring.
1579  *
1580  * @more_trbs_coming:   Will you enqueue more TRBs before setting doorbell?
1581  */
1582 static void cdnsp_queue_trb(struct cdnsp_device *pdev, struct cdnsp_ring *ring,
1583                             bool more_trbs_coming, u32 field1, u32 field2,
1584                             u32 field3, u32 field4)
1585 {
1586         struct cdnsp_generic_trb *trb;
1587
1588         trb = &ring->enqueue->generic;
1589
1590         trb->field[0] = cpu_to_le32(field1);
1591         trb->field[1] = cpu_to_le32(field2);
1592         trb->field[2] = cpu_to_le32(field3);
1593         trb->field[3] = cpu_to_le32(field4);
1594
1595         trace_cdnsp_queue_trb(ring, trb);
1596         cdnsp_inc_enq(pdev, ring, more_trbs_coming);
1597 }
1598
1599 /*
1600  * Does various checks on the endpoint ring, and makes it ready to
1601  * queue num_trbs.
1602  */
1603 static int cdnsp_prepare_ring(struct cdnsp_device *pdev,
1604                               struct cdnsp_ring *ep_ring,
1605                               u32 ep_state, unsigned
1606                               int num_trbs,
1607                               gfp_t mem_flags)
1608 {
1609         unsigned int num_trbs_needed;
1610
1611         /* Make sure the endpoint has been added to controller schedule. */
1612         switch (ep_state) {
1613         case EP_STATE_STOPPED:
1614         case EP_STATE_RUNNING:
1615         case EP_STATE_HALTED:
1616                 break;
1617         default:
1618                 dev_err(pdev->dev, "ERROR: incorrect endpoint state\n");
1619                 return -EINVAL;
1620         }
1621
1622         while (1) {
1623                 if (cdnsp_room_on_ring(pdev, ep_ring, num_trbs))
1624                         break;
1625
1626                 trace_cdnsp_no_room_on_ring("try ring expansion");
1627
1628                 num_trbs_needed = num_trbs - ep_ring->num_trbs_free;
1629                 if (cdnsp_ring_expansion(pdev, ep_ring, num_trbs_needed,
1630                                          mem_flags)) {
1631                         dev_err(pdev->dev, "Ring expansion failed\n");
1632                         return -ENOMEM;
1633                 }
1634         }
1635
1636         while (cdnsp_trb_is_link(ep_ring->enqueue)) {
1637                 ep_ring->enqueue->link.control |= cpu_to_le32(TRB_CHAIN);
1638                 /* The cycle bit must be set as the last operation. */
1639                 wmb();
1640                 ep_ring->enqueue->link.control ^= cpu_to_le32(TRB_CYCLE);
1641
1642                 /* Toggle the cycle bit after the last ring segment. */
1643                 if (cdnsp_link_trb_toggles_cycle(ep_ring->enqueue))
1644                         ep_ring->cycle_state ^= 1;
1645                 ep_ring->enq_seg = ep_ring->enq_seg->next;
1646                 ep_ring->enqueue = ep_ring->enq_seg->trbs;
1647         }
1648         return 0;
1649 }
1650
1651 static int cdnsp_prepare_transfer(struct cdnsp_device *pdev,
1652                                   struct cdnsp_request *preq,
1653                                   unsigned int num_trbs)
1654 {
1655         struct cdnsp_ring *ep_ring;
1656         int ret;
1657
1658         ep_ring = cdnsp_get_transfer_ring(pdev, preq->pep,
1659                                           preq->request.stream_id);
1660         if (!ep_ring)
1661                 return -EINVAL;
1662
1663         ret = cdnsp_prepare_ring(pdev, ep_ring,
1664                                  GET_EP_CTX_STATE(preq->pep->out_ctx),
1665                                  num_trbs, GFP_ATOMIC);
1666         if (ret)
1667                 return ret;
1668
1669         INIT_LIST_HEAD(&preq->td.td_list);
1670         preq->td.preq = preq;
1671
1672         /* Add this TD to the tail of the endpoint ring's TD list. */
1673         list_add_tail(&preq->td.td_list, &ep_ring->td_list);
1674         ep_ring->num_tds++;
1675         preq->pep->stream_info.td_count++;
1676
1677         preq->td.start_seg = ep_ring->enq_seg;
1678         preq->td.first_trb = ep_ring->enqueue;
1679
1680         return 0;
1681 }
1682
1683 static unsigned int cdnsp_count_trbs(u64 addr, u64 len)
1684 {
1685         unsigned int num_trbs;
1686
1687         num_trbs = DIV_ROUND_UP(len + (addr & (TRB_MAX_BUFF_SIZE - 1)),
1688                                 TRB_MAX_BUFF_SIZE);
1689         if (num_trbs == 0)
1690                 num_trbs++;
1691
1692         return num_trbs;
1693 }
1694
1695 static unsigned int count_trbs_needed(struct cdnsp_request *preq)
1696 {
1697         return cdnsp_count_trbs(preq->request.dma, preq->request.length);
1698 }
1699
1700 static unsigned int count_sg_trbs_needed(struct cdnsp_request *preq)
1701 {
1702         unsigned int i, len, full_len, num_trbs = 0;
1703         struct scatterlist *sg;
1704
1705         full_len = preq->request.length;
1706
1707         for_each_sg(preq->request.sg, sg, preq->request.num_sgs, i) {
1708                 len = sg_dma_len(sg);
1709                 num_trbs += cdnsp_count_trbs(sg_dma_address(sg), len);
1710                 len = min(len, full_len);
1711                 full_len -= len;
1712                 if (full_len == 0)
1713                         break;
1714         }
1715
1716         return num_trbs;
1717 }
1718
1719 static unsigned int count_isoc_trbs_needed(struct cdnsp_request *preq)
1720 {
1721         return cdnsp_count_trbs(preq->request.dma, preq->request.length);
1722 }
1723
1724 static void cdnsp_check_trb_math(struct cdnsp_request *preq, int running_total)
1725 {
1726         if (running_total != preq->request.length)
1727                 dev_err(preq->pep->pdev->dev,
1728                         "%s - Miscalculated tx length, "
1729                         "queued %#x, asked for %#x (%d)\n",
1730                         preq->pep->name, running_total,
1731                         preq->request.length, preq->request.actual);
1732 }
1733
1734 /*
1735  * TD size is the number of max packet sized packets remaining in the TD
1736  * (*not* including this TRB).
1737  *
1738  * Total TD packet count = total_packet_count =
1739  *     DIV_ROUND_UP(TD size in bytes / wMaxPacketSize)
1740  *
1741  * Packets transferred up to and including this TRB = packets_transferred =
1742  *     rounddown(total bytes transferred including this TRB / wMaxPacketSize)
1743  *
1744  * TD size = total_packet_count - packets_transferred
1745  *
1746  * It must fit in bits 21:17, so it can't be bigger than 31.
1747  * This is taken care of in the TRB_TD_SIZE() macro
1748  *
1749  * The last TRB in a TD must have the TD size set to zero.
1750  */
1751 static u32 cdnsp_td_remainder(struct cdnsp_device *pdev,
1752                               int transferred,
1753                               int trb_buff_len,
1754                               unsigned int td_total_len,
1755                               struct cdnsp_request *preq,
1756                               bool more_trbs_coming)
1757 {
1758         u32 maxp, total_packet_count;
1759
1760         /* One TRB with a zero-length data packet. */
1761         if (!more_trbs_coming || (transferred == 0 && trb_buff_len == 0) ||
1762             trb_buff_len == td_total_len)
1763                 return 0;
1764
1765         maxp = usb_endpoint_maxp(preq->pep->endpoint.desc);
1766         total_packet_count = DIV_ROUND_UP(td_total_len, maxp);
1767
1768         /* Queuing functions don't count the current TRB into transferred. */
1769         return (total_packet_count - ((transferred + trb_buff_len) / maxp));
1770 }
1771
1772 static int cdnsp_align_td(struct cdnsp_device *pdev,
1773                           struct cdnsp_request *preq, u32 enqd_len,
1774                           u32 *trb_buff_len, struct cdnsp_segment *seg)
1775 {
1776         struct device *dev = pdev->dev;
1777         unsigned int unalign;
1778         unsigned int max_pkt;
1779         u32 new_buff_len;
1780
1781         max_pkt = usb_endpoint_maxp(preq->pep->endpoint.desc);
1782         unalign = (enqd_len + *trb_buff_len) % max_pkt;
1783
1784         /* We got lucky, last normal TRB data on segment is packet aligned. */
1785         if (unalign == 0)
1786                 return 0;
1787
1788         /* Is the last nornal TRB alignable by splitting it. */
1789         if (*trb_buff_len > unalign) {
1790                 *trb_buff_len -= unalign;
1791                 trace_cdnsp_bounce_align_td_split(preq, *trb_buff_len,
1792                                                   enqd_len, 0, unalign);
1793                 return 0;
1794         }
1795
1796         /*
1797          * We want enqd_len + trb_buff_len to sum up to a number aligned to
1798          * number which is divisible by the endpoint's wMaxPacketSize. IOW:
1799          * (size of currently enqueued TRBs + remainder) % wMaxPacketSize == 0.
1800          */
1801         new_buff_len = max_pkt - (enqd_len % max_pkt);
1802
1803         if (new_buff_len > (preq->request.length - enqd_len))
1804                 new_buff_len = (preq->request.length - enqd_len);
1805
1806         /* Create a max max_pkt sized bounce buffer pointed to by last trb. */
1807         if (preq->direction) {
1808                 sg_pcopy_to_buffer(preq->request.sg,
1809                                    preq->request.num_mapped_sgs,
1810                                    seg->bounce_buf, new_buff_len, enqd_len);
1811                 seg->bounce_dma = dma_map_single(dev, seg->bounce_buf,
1812                                                  max_pkt, DMA_TO_DEVICE);
1813         } else {
1814                 seg->bounce_dma = dma_map_single(dev, seg->bounce_buf,
1815                                                  max_pkt, DMA_FROM_DEVICE);
1816         }
1817
1818         if (dma_mapping_error(dev, seg->bounce_dma)) {
1819                 /* Try without aligning.*/
1820                 dev_warn(pdev->dev,
1821                          "Failed mapping bounce buffer, not aligning\n");
1822                 return 0;
1823         }
1824
1825         *trb_buff_len = new_buff_len;
1826         seg->bounce_len = new_buff_len;
1827         seg->bounce_offs = enqd_len;
1828
1829         trace_cdnsp_bounce_map(preq, new_buff_len, enqd_len, seg->bounce_dma,
1830                                unalign);
1831
1832         /*
1833          * Bounce buffer successful aligned and seg->bounce_dma will be used
1834          * in transfer TRB as new transfer buffer address.
1835          */
1836         return 1;
1837 }
1838
1839 int cdnsp_queue_bulk_tx(struct cdnsp_device *pdev, struct cdnsp_request *preq)
1840 {
1841         unsigned int enqd_len, block_len, trb_buff_len, full_len;
1842         unsigned int start_cycle, num_sgs = 0;
1843         struct cdnsp_generic_trb *start_trb;
1844         u32 field, length_field, remainder;
1845         struct scatterlist *sg = NULL;
1846         bool more_trbs_coming = true;
1847         bool need_zero_pkt = false;
1848         bool zero_len_trb = false;
1849         struct cdnsp_ring *ring;
1850         bool first_trb = true;
1851         unsigned int num_trbs;
1852         struct cdnsp_ep *pep;
1853         u64 addr, send_addr;
1854         int sent_len, ret;
1855
1856         ring = cdnsp_request_to_transfer_ring(pdev, preq);
1857         if (!ring)
1858                 return -EINVAL;
1859
1860         full_len = preq->request.length;
1861
1862         if (preq->request.num_sgs) {
1863                 num_sgs = preq->request.num_sgs;
1864                 sg = preq->request.sg;
1865                 addr = (u64)sg_dma_address(sg);
1866                 block_len = sg_dma_len(sg);
1867                 num_trbs = count_sg_trbs_needed(preq);
1868         } else {
1869                 num_trbs = count_trbs_needed(preq);
1870                 addr = (u64)preq->request.dma;
1871                 block_len = full_len;
1872         }
1873
1874         pep = preq->pep;
1875
1876         /* Deal with request.zero - need one more td/trb. */
1877         if (preq->request.zero && preq->request.length &&
1878             IS_ALIGNED(full_len, usb_endpoint_maxp(pep->endpoint.desc))) {
1879                 need_zero_pkt = true;
1880                 num_trbs++;
1881         }
1882
1883         ret = cdnsp_prepare_transfer(pdev, preq, num_trbs);
1884         if (ret)
1885                 return ret;
1886
1887         /*
1888          * Don't give the first TRB to the hardware (by toggling the cycle bit)
1889          * until we've finished creating all the other TRBs. The ring's cycle
1890          * state may change as we enqueue the other TRBs, so save it too.
1891          */
1892         start_trb = &ring->enqueue->generic;
1893         start_cycle = ring->cycle_state;
1894         send_addr = addr;
1895
1896         /* Queue the TRBs, even if they are zero-length */
1897         for (enqd_len = 0; zero_len_trb || first_trb || enqd_len < full_len;
1898              enqd_len += trb_buff_len) {
1899                 field = TRB_TYPE(TRB_NORMAL);
1900
1901                 /* TRB buffer should not cross 64KB boundaries */
1902                 trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(addr);
1903                 trb_buff_len = min(trb_buff_len, block_len);
1904                 if (enqd_len + trb_buff_len > full_len)
1905                         trb_buff_len = full_len - enqd_len;
1906
1907                 /* Don't change the cycle bit of the first TRB until later */
1908                 if (first_trb) {
1909                         first_trb = false;
1910                         if (start_cycle == 0)
1911                                 field |= TRB_CYCLE;
1912                 } else {
1913                         field |= ring->cycle_state;
1914                 }
1915
1916                 /*
1917                  * Chain all the TRBs together; clear the chain bit in the last
1918                  * TRB to indicate it's the last TRB in the chain.
1919                  */
1920                 if (enqd_len + trb_buff_len < full_len || need_zero_pkt) {
1921                         field |= TRB_CHAIN;
1922                         if (cdnsp_trb_is_link(ring->enqueue + 1)) {
1923                                 if (cdnsp_align_td(pdev, preq, enqd_len,
1924                                                    &trb_buff_len,
1925                                                    ring->enq_seg)) {
1926                                         send_addr = ring->enq_seg->bounce_dma;
1927                                         /* Assuming TD won't span 2 segs */
1928                                         preq->td.bounce_seg = ring->enq_seg;
1929                                 }
1930                         }
1931                 }
1932
1933                 if (enqd_len + trb_buff_len >= full_len) {
1934                         if (need_zero_pkt && zero_len_trb) {
1935                                 zero_len_trb = true;
1936                         } else {
1937                                 field &= ~TRB_CHAIN;
1938                                 field |= TRB_IOC;
1939                                 more_trbs_coming = false;
1940                                 need_zero_pkt = false;
1941                                 preq->td.last_trb = ring->enqueue;
1942                         }
1943                 }
1944
1945                 /* Only set interrupt on short packet for OUT endpoints. */
1946                 if (!preq->direction)
1947                         field |= TRB_ISP;
1948
1949                 /* Set the TRB length, TD size, and interrupter fields. */
1950                 remainder = cdnsp_td_remainder(pdev, enqd_len, trb_buff_len,
1951                                                full_len, preq,
1952                                                more_trbs_coming);
1953
1954                 length_field = TRB_LEN(trb_buff_len) | TRB_TD_SIZE(remainder) |
1955                         TRB_INTR_TARGET(0);
1956
1957                 cdnsp_queue_trb(pdev, ring, more_trbs_coming | need_zero_pkt,
1958                                 lower_32_bits(send_addr),
1959                                 upper_32_bits(send_addr),
1960                                 length_field,
1961                                 field);
1962
1963                 addr += trb_buff_len;
1964                 sent_len = trb_buff_len;
1965                 while (sg && sent_len >= block_len) {
1966                         /* New sg entry */
1967                         --num_sgs;
1968                         sent_len -= block_len;
1969                         if (num_sgs != 0) {
1970                                 sg = sg_next(sg);
1971                                 block_len = sg_dma_len(sg);
1972                                 addr = (u64)sg_dma_address(sg);
1973                                 addr += sent_len;
1974                         }
1975                 }
1976                 block_len -= sent_len;
1977                 send_addr = addr;
1978         }
1979
1980         cdnsp_check_trb_math(preq, enqd_len);
1981         ret = cdnsp_giveback_first_trb(pdev, pep, preq->request.stream_id,
1982                                        start_cycle, start_trb);
1983
1984         if (ret)
1985                 preq->td.drbl = 1;
1986
1987         return 0;
1988 }
1989
1990 int cdnsp_queue_ctrl_tx(struct cdnsp_device *pdev, struct cdnsp_request *preq)
1991 {
1992         u32 field, length_field, remainder;
1993         struct cdnsp_ep *pep = preq->pep;
1994         struct cdnsp_ring *ep_ring;
1995         int num_trbs;
1996         int ret;
1997
1998         ep_ring = cdnsp_request_to_transfer_ring(pdev, preq);
1999         if (!ep_ring)
2000                 return -EINVAL;
2001
2002         /* 1 TRB for data, 1 for status */
2003         num_trbs = (pdev->three_stage_setup) ? 2 : 1;
2004
2005         ret = cdnsp_prepare_transfer(pdev, preq, num_trbs);
2006         if (ret)
2007                 return ret;
2008
2009         /* If there's data, queue data TRBs */
2010         if (pdev->ep0_expect_in)
2011                 field = TRB_TYPE(TRB_DATA) | TRB_IOC;
2012         else
2013                 field = TRB_ISP | TRB_TYPE(TRB_DATA) | TRB_IOC;
2014
2015         if (preq->request.length > 0) {
2016                 remainder = cdnsp_td_remainder(pdev, 0, preq->request.length,
2017                                                preq->request.length, preq, 1);
2018
2019                 length_field = TRB_LEN(preq->request.length) |
2020                                 TRB_TD_SIZE(remainder) | TRB_INTR_TARGET(0);
2021
2022                 if (pdev->ep0_expect_in)
2023                         field |= TRB_DIR_IN;
2024
2025                 cdnsp_queue_trb(pdev, ep_ring, true,
2026                                 lower_32_bits(preq->request.dma),
2027                                 upper_32_bits(preq->request.dma), length_field,
2028                                 field | ep_ring->cycle_state |
2029                                 TRB_SETUPID(pdev->setup_id) |
2030                                 pdev->setup_speed);
2031
2032                 pdev->ep0_stage = CDNSP_DATA_STAGE;
2033         }
2034
2035         /* Save the DMA address of the last TRB in the TD. */
2036         preq->td.last_trb = ep_ring->enqueue;
2037
2038         /* Queue status TRB. */
2039         if (preq->request.length == 0)
2040                 field = ep_ring->cycle_state;
2041         else
2042                 field = (ep_ring->cycle_state ^ 1);
2043
2044         if (preq->request.length > 0 && pdev->ep0_expect_in)
2045                 field |= TRB_DIR_IN;
2046
2047         if (pep->ep_state & EP0_HALTED_STATUS) {
2048                 pep->ep_state &= ~EP0_HALTED_STATUS;
2049                 field |= TRB_SETUPSTAT(TRB_SETUPSTAT_STALL);
2050         } else {
2051                 field |= TRB_SETUPSTAT(TRB_SETUPSTAT_ACK);
2052         }
2053
2054         cdnsp_queue_trb(pdev, ep_ring, false, 0, 0, TRB_INTR_TARGET(0),
2055                         field | TRB_IOC | TRB_SETUPID(pdev->setup_id) |
2056                         TRB_TYPE(TRB_STATUS) | pdev->setup_speed);
2057
2058         cdnsp_ring_ep_doorbell(pdev, pep, preq->request.stream_id);
2059
2060         return 0;
2061 }
2062
2063 int cdnsp_cmd_stop_ep(struct cdnsp_device *pdev, struct cdnsp_ep *pep)
2064 {
2065         u32 ep_state = GET_EP_CTX_STATE(pep->out_ctx);
2066         int ret = 0;
2067
2068         if (ep_state == EP_STATE_STOPPED || ep_state == EP_STATE_DISABLED) {
2069                 trace_cdnsp_ep_stopped_or_disabled(pep->out_ctx);
2070                 goto ep_stopped;
2071         }
2072
2073         cdnsp_queue_stop_endpoint(pdev, pep->idx);
2074         cdnsp_ring_cmd_db(pdev);
2075         ret = cdnsp_wait_for_cmd_compl(pdev);
2076
2077         trace_cdnsp_handle_cmd_stop_ep(pep->out_ctx);
2078
2079 ep_stopped:
2080         pep->ep_state |= EP_STOPPED;
2081         return ret;
2082 }
2083
2084 int cdnsp_cmd_flush_ep(struct cdnsp_device *pdev, struct cdnsp_ep *pep)
2085 {
2086         int ret;
2087
2088         cdnsp_queue_flush_endpoint(pdev, pep->idx);
2089         cdnsp_ring_cmd_db(pdev);
2090         ret = cdnsp_wait_for_cmd_compl(pdev);
2091
2092         trace_cdnsp_handle_cmd_flush_ep(pep->out_ctx);
2093
2094         return ret;
2095 }
2096
2097 /*
2098  * The transfer burst count field of the isochronous TRB defines the number of
2099  * bursts that are required to move all packets in this TD. Only SuperSpeed
2100  * devices can burst up to bMaxBurst number of packets per service interval.
2101  * This field is zero based, meaning a value of zero in the field means one
2102  * burst. Basically, for everything but SuperSpeed devices, this field will be
2103  * zero.
2104  */
2105 static unsigned int cdnsp_get_burst_count(struct cdnsp_device *pdev,
2106                                           struct cdnsp_request *preq,
2107                                           unsigned int total_packet_count)
2108 {
2109         unsigned int max_burst;
2110
2111         if (pdev->gadget.speed < USB_SPEED_SUPER)
2112                 return 0;
2113
2114         max_burst = preq->pep->endpoint.comp_desc->bMaxBurst;
2115         return DIV_ROUND_UP(total_packet_count, max_burst + 1) - 1;
2116 }
2117
2118 /*
2119  * Returns the number of packets in the last "burst" of packets. This field is
2120  * valid for all speeds of devices. USB 2.0 devices can only do one "burst", so
2121  * the last burst packet count is equal to the total number of packets in the
2122  * TD. SuperSpeed endpoints can have up to 3 bursts. All but the last burst
2123  * must contain (bMaxBurst + 1) number of packets, but the last burst can
2124  * contain 1 to (bMaxBurst + 1) packets.
2125  */
2126 static unsigned int
2127         cdnsp_get_last_burst_packet_count(struct cdnsp_device *pdev,
2128                                           struct cdnsp_request *preq,
2129                                           unsigned int total_packet_count)
2130 {
2131         unsigned int max_burst;
2132         unsigned int residue;
2133
2134         if (pdev->gadget.speed >= USB_SPEED_SUPER) {
2135                 /* bMaxBurst is zero based: 0 means 1 packet per burst. */
2136                 max_burst = preq->pep->endpoint.comp_desc->bMaxBurst;
2137                 residue = total_packet_count % (max_burst + 1);
2138
2139                 /*
2140                  * If residue is zero, the last burst contains (max_burst + 1)
2141                  * number of packets, but the TLBPC field is zero-based.
2142                  */
2143                 if (residue == 0)
2144                         return max_burst;
2145
2146                 return residue - 1;
2147         }
2148         if (total_packet_count == 0)
2149                 return 0;
2150
2151         return total_packet_count - 1;
2152 }
2153
2154 /* Queue function isoc transfer */
2155 static int cdnsp_queue_isoc_tx(struct cdnsp_device *pdev,
2156                                struct cdnsp_request *preq)
2157 {
2158         int trb_buff_len, td_len, td_remain_len, ret;
2159         unsigned int burst_count, last_burst_pkt;
2160         unsigned int total_pkt_count, max_pkt;
2161         struct cdnsp_generic_trb *start_trb;
2162         bool more_trbs_coming = true;
2163         struct cdnsp_ring *ep_ring;
2164         int running_total = 0;
2165         u32 field, length_field;
2166         int start_cycle;
2167         int trbs_per_td;
2168         u64 addr;
2169         int i;
2170
2171         ep_ring = preq->pep->ring;
2172         start_trb = &ep_ring->enqueue->generic;
2173         start_cycle = ep_ring->cycle_state;
2174         td_len = preq->request.length;
2175         addr = (u64)preq->request.dma;
2176         td_remain_len = td_len;
2177
2178         max_pkt = usb_endpoint_maxp(preq->pep->endpoint.desc);
2179         total_pkt_count = DIV_ROUND_UP(td_len, max_pkt);
2180
2181         /* A zero-length transfer still involves at least one packet. */
2182         if (total_pkt_count == 0)
2183                 total_pkt_count++;
2184
2185         burst_count = cdnsp_get_burst_count(pdev, preq, total_pkt_count);
2186         last_burst_pkt = cdnsp_get_last_burst_packet_count(pdev, preq,
2187                                                            total_pkt_count);
2188         trbs_per_td = count_isoc_trbs_needed(preq);
2189
2190         ret = cdnsp_prepare_transfer(pdev, preq, trbs_per_td);
2191         if (ret)
2192                 goto cleanup;
2193
2194         /*
2195          * Set isoc specific data for the first TRB in a TD.
2196          * Prevent HW from getting the TRBs by keeping the cycle state
2197          * inverted in the first TDs isoc TRB.
2198          */
2199         field = TRB_TYPE(TRB_ISOC) | TRB_TLBPC(last_burst_pkt) |
2200                 start_cycle ? 0 : 1 | TRB_SIA | TRB_TBC(burst_count);
2201
2202         /* Fill the rest of the TRB fields, and remaining normal TRBs. */
2203         for (i = 0; i < trbs_per_td; i++) {
2204                 u32 remainder;
2205
2206                 /* Calculate TRB length. */
2207                 trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(addr);
2208                 if (trb_buff_len > td_remain_len)
2209                         trb_buff_len = td_remain_len;
2210
2211                 /* Set the TRB length, TD size, & interrupter fields. */
2212                 remainder = cdnsp_td_remainder(pdev, running_total,
2213                                                trb_buff_len, td_len, preq,
2214                                                more_trbs_coming);
2215
2216                 length_field = TRB_LEN(trb_buff_len) | TRB_INTR_TARGET(0);
2217
2218                 /* Only first TRB is isoc, overwrite otherwise. */
2219                 if (i) {
2220                         field = TRB_TYPE(TRB_NORMAL) | ep_ring->cycle_state;
2221                         length_field |= TRB_TD_SIZE(remainder);
2222                 } else {
2223                         length_field |= TRB_TD_SIZE_TBC(burst_count);
2224                 }
2225
2226                 /* Only set interrupt on short packet for OUT EPs. */
2227                 if (usb_endpoint_dir_out(preq->pep->endpoint.desc))
2228                         field |= TRB_ISP;
2229
2230                 /* Set the chain bit for all except the last TRB. */
2231                 if (i < trbs_per_td - 1) {
2232                         more_trbs_coming = true;
2233                         field |= TRB_CHAIN;
2234                 } else {
2235                         more_trbs_coming = false;
2236                         preq->td.last_trb = ep_ring->enqueue;
2237                         field |= TRB_IOC;
2238                 }
2239
2240                 cdnsp_queue_trb(pdev, ep_ring, more_trbs_coming,
2241                                 lower_32_bits(addr), upper_32_bits(addr),
2242                                 length_field, field);
2243
2244                 running_total += trb_buff_len;
2245                 addr += trb_buff_len;
2246                 td_remain_len -= trb_buff_len;
2247         }
2248
2249         /* Check TD length */
2250         if (running_total != td_len) {
2251                 dev_err(pdev->dev, "ISOC TD length unmatch\n");
2252                 ret = -EINVAL;
2253                 goto cleanup;
2254         }
2255
2256         cdnsp_giveback_first_trb(pdev, preq->pep, preq->request.stream_id,
2257                                  start_cycle, start_trb);
2258
2259         return 0;
2260
2261 cleanup:
2262         /* Clean up a partially enqueued isoc transfer. */
2263         list_del_init(&preq->td.td_list);
2264         ep_ring->num_tds--;
2265
2266         /*
2267          * Use the first TD as a temporary variable to turn the TDs we've
2268          * queued into No-ops with a software-owned cycle bit.
2269          * That way the hardware won't accidentally start executing bogus TDs
2270          * when we partially overwrite them.
2271          * td->first_trb and td->start_seg are already set.
2272          */
2273         preq->td.last_trb = ep_ring->enqueue;
2274         /* Every TRB except the first & last will have its cycle bit flipped. */
2275         cdnsp_td_to_noop(pdev, ep_ring, &preq->td, true);
2276
2277         /* Reset the ring enqueue back to the first TRB and its cycle bit. */
2278         ep_ring->enqueue = preq->td.first_trb;
2279         ep_ring->enq_seg = preq->td.start_seg;
2280         ep_ring->cycle_state = start_cycle;
2281         return ret;
2282 }
2283
2284 int cdnsp_queue_isoc_tx_prepare(struct cdnsp_device *pdev,
2285                                 struct cdnsp_request *preq)
2286 {
2287         struct cdnsp_ring *ep_ring;
2288         u32 ep_state;
2289         int num_trbs;
2290         int ret;
2291
2292         ep_ring = preq->pep->ring;
2293         ep_state = GET_EP_CTX_STATE(preq->pep->out_ctx);
2294         num_trbs = count_isoc_trbs_needed(preq);
2295
2296         /*
2297          * Check the ring to guarantee there is enough room for the whole
2298          * request. Do not insert any td of the USB Request to the ring if the
2299          * check failed.
2300          */
2301         ret = cdnsp_prepare_ring(pdev, ep_ring, ep_state, num_trbs, GFP_ATOMIC);
2302         if (ret)
2303                 return ret;
2304
2305         return cdnsp_queue_isoc_tx(pdev, preq);
2306 }
2307
2308 /****           Command Ring Operations         ****/
2309 /*
2310  * Generic function for queuing a command TRB on the command ring.
2311  * Driver queue only one command to ring in the moment.
2312  */
2313 static void cdnsp_queue_command(struct cdnsp_device *pdev,
2314                                 u32 field1,
2315                                 u32 field2,
2316                                 u32 field3,
2317                                 u32 field4)
2318 {
2319         cdnsp_prepare_ring(pdev, pdev->cmd_ring, EP_STATE_RUNNING, 1,
2320                            GFP_ATOMIC);
2321
2322         pdev->cmd.command_trb = pdev->cmd_ring->enqueue;
2323
2324         cdnsp_queue_trb(pdev, pdev->cmd_ring, false, field1, field2,
2325                         field3, field4 | pdev->cmd_ring->cycle_state);
2326 }
2327
2328 /* Queue a slot enable or disable request on the command ring */
2329 void cdnsp_queue_slot_control(struct cdnsp_device *pdev, u32 trb_type)
2330 {
2331         cdnsp_queue_command(pdev, 0, 0, 0, TRB_TYPE(trb_type) |
2332                             SLOT_ID_FOR_TRB(pdev->slot_id));
2333 }
2334
2335 /* Queue an address device command TRB */
2336 void cdnsp_queue_address_device(struct cdnsp_device *pdev,
2337                                 dma_addr_t in_ctx_ptr,
2338                                 enum cdnsp_setup_dev setup)
2339 {
2340         cdnsp_queue_command(pdev, lower_32_bits(in_ctx_ptr),
2341                             upper_32_bits(in_ctx_ptr), 0,
2342                             TRB_TYPE(TRB_ADDR_DEV) |
2343                             SLOT_ID_FOR_TRB(pdev->slot_id) |
2344                             (setup == SETUP_CONTEXT_ONLY ? TRB_BSR : 0));
2345 }
2346
2347 /* Queue a reset device command TRB */
2348 void cdnsp_queue_reset_device(struct cdnsp_device *pdev)
2349 {
2350         cdnsp_queue_command(pdev, 0, 0, 0, TRB_TYPE(TRB_RESET_DEV) |
2351                             SLOT_ID_FOR_TRB(pdev->slot_id));
2352 }
2353
2354 /* Queue a configure endpoint command TRB */
2355 void cdnsp_queue_configure_endpoint(struct cdnsp_device *pdev,
2356                                     dma_addr_t in_ctx_ptr)
2357 {
2358         cdnsp_queue_command(pdev, lower_32_bits(in_ctx_ptr),
2359                             upper_32_bits(in_ctx_ptr), 0,
2360                             TRB_TYPE(TRB_CONFIG_EP) |
2361                             SLOT_ID_FOR_TRB(pdev->slot_id));
2362 }
2363
2364 /*
2365  * Suspend is set to indicate "Stop Endpoint Command" is being issued to stop
2366  * activity on an endpoint that is about to be suspended.
2367  */
2368 void cdnsp_queue_stop_endpoint(struct cdnsp_device *pdev, unsigned int ep_index)
2369 {
2370         cdnsp_queue_command(pdev, 0, 0, 0, SLOT_ID_FOR_TRB(pdev->slot_id) |
2371                             EP_ID_FOR_TRB(ep_index) | TRB_TYPE(TRB_STOP_RING));
2372 }
2373
2374 /* Set Transfer Ring Dequeue Pointer command. */
2375 void cdnsp_queue_new_dequeue_state(struct cdnsp_device *pdev,
2376                                    struct cdnsp_ep *pep,
2377                                    struct cdnsp_dequeue_state *deq_state)
2378 {
2379         u32 trb_stream_id = STREAM_ID_FOR_TRB(deq_state->stream_id);
2380         u32 trb_slot_id = SLOT_ID_FOR_TRB(pdev->slot_id);
2381         u32 type = TRB_TYPE(TRB_SET_DEQ);
2382         u32 trb_sct = 0;
2383         dma_addr_t addr;
2384
2385         addr = cdnsp_trb_virt_to_dma(deq_state->new_deq_seg,
2386                                      deq_state->new_deq_ptr);
2387
2388         if (deq_state->stream_id)
2389                 trb_sct = SCT_FOR_TRB(SCT_PRI_TR);
2390
2391         cdnsp_queue_command(pdev, lower_32_bits(addr) | trb_sct |
2392                             deq_state->new_cycle_state, upper_32_bits(addr),
2393                             trb_stream_id, trb_slot_id |
2394                             EP_ID_FOR_TRB(pep->idx) | type);
2395 }
2396
2397 void cdnsp_queue_reset_ep(struct cdnsp_device *pdev, unsigned int ep_index)
2398 {
2399         return cdnsp_queue_command(pdev, 0, 0, 0,
2400                                    SLOT_ID_FOR_TRB(pdev->slot_id) |
2401                                    EP_ID_FOR_TRB(ep_index) |
2402                                    TRB_TYPE(TRB_RESET_EP));
2403 }
2404
2405 /*
2406  * Queue a halt endpoint request on the command ring.
2407  */
2408 void cdnsp_queue_halt_endpoint(struct cdnsp_device *pdev, unsigned int ep_index)
2409 {
2410         cdnsp_queue_command(pdev, 0, 0, 0, TRB_TYPE(TRB_HALT_ENDPOINT) |
2411                             SLOT_ID_FOR_TRB(pdev->slot_id) |
2412                             EP_ID_FOR_TRB(ep_index));
2413 }
2414
2415 /*
2416  * Queue a flush endpoint request on the command ring.
2417  */
2418 void  cdnsp_queue_flush_endpoint(struct cdnsp_device *pdev,
2419                                  unsigned int ep_index)
2420 {
2421         cdnsp_queue_command(pdev, 0, 0, 0, TRB_TYPE(TRB_FLUSH_ENDPOINT) |
2422                             SLOT_ID_FOR_TRB(pdev->slot_id) |
2423                             EP_ID_FOR_TRB(ep_index));
2424 }
2425
2426 void cdnsp_force_header_wakeup(struct cdnsp_device *pdev, int intf_num)
2427 {
2428         u32 lo, mid;
2429
2430         lo = TRB_FH_TO_PACKET_TYPE(TRB_FH_TR_PACKET) |
2431              TRB_FH_TO_DEVICE_ADDRESS(pdev->device_address);
2432         mid = TRB_FH_TR_PACKET_DEV_NOT |
2433               TRB_FH_TO_NOT_TYPE(TRB_FH_TR_PACKET_FUNCTION_WAKE) |
2434               TRB_FH_TO_INTERFACE(intf_num);
2435
2436         cdnsp_queue_command(pdev, lo, mid, 0,
2437                             TRB_TYPE(TRB_FORCE_HEADER) | SET_PORT_ID(2));
2438 }