drivers/usb/dwc3/gadget.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
   4  *
   5  * Copyright (C) 2010-2011 Texas Instruments Incorporated - https://www.ti.com
   6  *
   7  * Authors: Felipe Balbi <[email protected]>,
   8  *          Sebastian Andrzej Siewior <[email protected]>
   9  */
  10
  11 #include <linux/kernel.h>
  12 #include <linux/delay.h>
  13 #include <linux/slab.h>
  14 #include <linux/spinlock.h>
  15 #include <linux/platform_device.h>
  16 #include <linux/pm_runtime.h>
  17 #include <linux/interrupt.h>
  18 #include <linux/io.h>
  19 #include <linux/list.h>
  20 #include <linux/dma-mapping.h>
  21
  22 #include <linux/usb/ch9.h>
  23 #include <linux/usb/gadget.h>
  24
  25 #include "debug.h"
  26 #include "core.h"
  27 #include "gadget.h"
  28 #include "io.h"
  29
  30 #define DWC3_ALIGN_FRAME(d, n)  (((d)->frame_number + ((d)->interval * (n))) \
  31                                         & ~((d)->interval - 1))
  32
  33 /**
  34  * dwc3_gadget_set_test_mode - enables usb2 test modes
  35  * @dwc: pointer to our context structure
  36  * @mode: the mode to set (J, K SE0 NAK, Force Enable)
  37  *
  38  * Caller should take care of locking. This function will return 0 on
  39  * success or -EINVAL if wrong Test Selector is passed.
  40  */
  41 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
  42 {
  43         u32             reg;
  44
  45         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
  46         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
  47
  48         switch (mode) {
  49         case USB_TEST_J:
  50         case USB_TEST_K:
  51         case USB_TEST_SE0_NAK:
  52         case USB_TEST_PACKET:
  53         case USB_TEST_FORCE_ENABLE:
  54                 reg |= mode << 1;
  55                 break;
  56         default:
  57                 return -EINVAL;
  58         }
  59
  60         dwc3_gadget_dctl_write_safe(dwc, reg);
  61
  62         return 0;
  63 }
  64
  65 /**
  66  * dwc3_gadget_get_link_state - gets current state of usb link
  67  * @dwc: pointer to our context structure
  68  *
  69  * Caller should take care of locking. This function will
  70  * return the link state on success (>= 0) or -ETIMEDOUT.
  71  */
  72 int dwc3_gadget_get_link_state(struct dwc3 *dwc)
  73 {
  74         u32             reg;
  75
  76         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
  77
  78         return DWC3_DSTS_USBLNKST(reg);
  79 }
  80
  81 /**
  82  * dwc3_gadget_set_link_state - sets usb link to a particular state
  83  * @dwc: pointer to our context structure
  84  * @state: the state to put link into
  85  *
  86  * Caller should take care of locking. This function will
  87  * return 0 on success or -ETIMEDOUT.
  88  */
  89 int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
  90 {
  91         int             retries = 10000;
  92         u32             reg;
  93
  94         /*
  95          * Wait until device controller is ready. Only applies to 1.94a and
  96          * later RTL.
  97          */
  98         if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) {
  99                 while (--retries) {
 100                         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
 101                         if (reg & DWC3_DSTS_DCNRD)
 102                                 udelay(5);
 103                         else
 104                                 break;
 105                 }
 106
 107                 if (retries <= 0)
 108                         return -ETIMEDOUT;
 109         }
 110
 111         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
 112         reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
 113
 114         /* set no action before sending new link state change */
 115         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
 116
 117         /* set requested state */
 118         reg |= DWC3_DCTL_ULSTCHNGREQ(state);
 119         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
 120
 121         /*
 122          * The following code is racy when called from dwc3_gadget_wakeup,
 123          * and is not needed, at least on newer versions
 124          */
 125         if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
 126                 return 0;
 127
 128         /* wait for a change in DSTS */
 129         retries = 10000;
 130         while (--retries) {
 131                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
 132
 133                 if (DWC3_DSTS_USBLNKST(reg) == state)
 134                         return 0;
 135
 136                 udelay(5);
 137         }
 138
 139         return -ETIMEDOUT;
 140 }
 141
 142 /**
 143  * dwc3_ep_inc_trb - increment a trb index.
 144  * @index: Pointer to the TRB index to increment.
 145  *
 146  * The index should never point to the link TRB. After incrementing,
 147  * if it is point to the link TRB, wrap around to the beginning. The
 148  * link TRB is always at the last TRB entry.
 149  */
 150 static void dwc3_ep_inc_trb(u8 *index)
 151 {
 152         (*index)++;
 153         if (*index == (DWC3_TRB_NUM - 1))
 154                 *index = 0;
 155 }
 156
 157 /**
 158  * dwc3_ep_inc_enq - increment endpoint's enqueue pointer
 159  * @dep: The endpoint whose enqueue pointer we're incrementing
 160  */
 161 static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
 162 {
 163         dwc3_ep_inc_trb(&dep->trb_enqueue);
 164 }
 165
 166 /**
 167  * dwc3_ep_inc_deq - increment endpoint's dequeue pointer
 168  * @dep: The endpoint whose enqueue pointer we're incrementing
 169  */
 170 static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
 171 {
 172         dwc3_ep_inc_trb(&dep->trb_dequeue);
 173 }
 174
 175 static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep,
 176                 struct dwc3_request *req, int status)
 177 {
 178         struct dwc3                     *dwc = dep->dwc;
 179
 180         list_del(&req->list);
 181         req->remaining = 0;
 182         req->needs_extra_trb = false;
 183
 184         if (req->request.status == -EINPROGRESS)
 185                 req->request.status = status;
 186
 187         if (req->trb)
 188                 usb_gadget_unmap_request_by_dev(dwc->sysdev,
 189                                 &req->request, req->direction);
 190
 191         req->trb = NULL;
 192         trace_dwc3_gadget_giveback(req);
 193
 194         if (dep->number > 1)
 195                 pm_runtime_put(dwc->dev);
 196 }
 197
 198 /**
 199  * dwc3_gadget_giveback - call struct usb_request's ->complete callback
 200  * @dep: The endpoint to whom the request belongs to
 201  * @req: The request we're giving back
 202  * @status: completion code for the request
 203  *
 204  * Must be called with controller's lock held and interrupts disabled. This
 205  * function will unmap @req and call its ->complete() callback to notify upper
 206  * layers that it has completed.
 207  */
 208 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
 209                 int status)
 210 {
 211         struct dwc3                     *dwc = dep->dwc;
 212
 213         dwc3_gadget_del_and_unmap_request(dep, req, status);
 214         req->status = DWC3_REQUEST_STATUS_COMPLETED;
 215
 216         spin_unlock(&dwc->lock);
 217         usb_gadget_giveback_request(&dep->endpoint, &req->request);
 218         spin_lock(&dwc->lock);
 219 }
 220
 221 /**
 222  * dwc3_send_gadget_generic_command - issue a generic command for the controller
 223  * @dwc: pointer to the controller context
 224  * @cmd: the command to be issued
 225  * @param: command parameter
 226  *
 227  * Caller should take care of locking. Issue @cmd with a given @param to @dwc
 228  * and wait for its completion.
 229  */
 230 int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned cmd, u32 param)
 231 {
 232         u32             timeout = 500;
 233         int             status = 0;
 234         int             ret = 0;
 235         u32             reg;
 236
 237         dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
 238         dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
 239
 240         do {
 241                 reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
 242                 if (!(reg & DWC3_DGCMD_CMDACT)) {
 243                         status = DWC3_DGCMD_STATUS(reg);
 244                         if (status)
 245                                 ret = -EINVAL;
 246                         break;
 247                 }
 248         } while (--timeout);
 249
 250         if (!timeout) {
 251                 ret = -ETIMEDOUT;
 252                 status = -ETIMEDOUT;
 253         }
 254
 255         trace_dwc3_gadget_generic_cmd(cmd, param, status);
 256
 257         return ret;
 258 }
 259
 260 static int __dwc3_gadget_wakeup(struct dwc3 *dwc);
 261
 262 /**
 263  * dwc3_send_gadget_ep_cmd - issue an endpoint command
 264  * @dep: the endpoint to which the command is going to be issued
 265  * @cmd: the command to be issued
 266  * @params: parameters to the command
 267  *
 268  * Caller should handle locking. This function will issue @cmd with given
 269  * @params to @dep and wait for its completion.
 270  */
 271 int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned cmd,
 272                 struct dwc3_gadget_ep_cmd_params *params)
 273 {
 274         const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
 275         struct dwc3             *dwc = dep->dwc;
 276         u32                     timeout = 5000;
 277         u32                     saved_config = 0;
 278         u32                     reg;
 279
 280         int                     cmd_status = 0;
 281         int                     ret = -EINVAL;
 282
 283         /*
 284          * When operating in USB 2.0 speeds (HS/FS), if GUSB2PHYCFG.ENBLSLPM or
 285          * GUSB2PHYCFG.SUSPHY is set, it must be cleared before issuing an
 286          * endpoint command.
 287          *
 288          * Save and clear both GUSB2PHYCFG.ENBLSLPM and GUSB2PHYCFG.SUSPHY
 289          * settings. Restore them after the command is completed.
 290          *
 291          * DWC_usb3 3.30a and DWC_usb31 1.90a programming guide section 3.2.2
 292          */
 293         if (dwc->gadget.speed <= USB_SPEED_HIGH) {
 294                 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
 295                 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
 296                         saved_config |= DWC3_GUSB2PHYCFG_SUSPHY;
 297                         reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
 298                 }
 299
 300                 if (reg & DWC3_GUSB2PHYCFG_ENBLSLPM) {
 301                         saved_config |= DWC3_GUSB2PHYCFG_ENBLSLPM;
 302                         reg &= ~DWC3_GUSB2PHYCFG_ENBLSLPM;
 303                 }
 304
 305                 if (saved_config)
 306                         dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
 307         }
 308
 309         if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
 310                 int             needs_wakeup;
 311
 312                 needs_wakeup = (dwc->link_state == DWC3_LINK_STATE_U1 ||
 313                                 dwc->link_state == DWC3_LINK_STATE_U2 ||
 314                                 dwc->link_state == DWC3_LINK_STATE_U3);
 315
 316                 if (unlikely(needs_wakeup)) {
 317                         ret = __dwc3_gadget_wakeup(dwc);
 318                         dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
 319                                         ret);
 320                 }
 321         }
 322
 323         dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
 324         dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
 325         dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
 326
 327         /*
 328          * Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're
 329          * not relying on XferNotReady, we can make use of a special "No
 330          * Response Update Transfer" command where we should clear both CmdAct
 331          * and CmdIOC bits.
 332          *
 333          * With this, we don't need to wait for command completion and can
 334          * straight away issue further commands to the endpoint.
 335          *
 336          * NOTICE: We're making an assumption that control endpoints will never
 337          * make use of Update Transfer command. This is a safe assumption
 338          * because we can never have more than one request at a time with
 339          * Control Endpoints. If anybody changes that assumption, this chunk
 340          * needs to be updated accordingly.
 341          */
 342         if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER &&
 343                         !usb_endpoint_xfer_isoc(desc))
 344                 cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT);
 345         else
 346                 cmd |= DWC3_DEPCMD_CMDACT;
 347
 348         dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
 349         do {
 350                 reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
 351                 if (!(reg & DWC3_DEPCMD_CMDACT)) {
 352                         cmd_status = DWC3_DEPCMD_STATUS(reg);
 353
 354                         switch (cmd_status) {
 355                         case 0:
 356                                 ret = 0;
 357                                 break;
 358                         case DEPEVT_TRANSFER_NO_RESOURCE:
 359                                 dev_WARN(dwc->dev, "No resource for %s\n",
 360                                          dep->name);
 361                                 ret = -EINVAL;
 362                                 break;
 363                         case DEPEVT_TRANSFER_BUS_EXPIRY:
 364                                 /*
 365                                  * SW issues START TRANSFER command to
 366                                  * isochronous ep with future frame interval. If
 367                                  * future interval time has already passed when
 368                                  * core receives the command, it will respond
 369                                  * with an error status of 'Bus Expiry'.
 370                                  *
 371                                  * Instead of always returning -EINVAL, let's
 372                                  * give a hint to the gadget driver that this is
 373                                  * the case by returning -EAGAIN.
 374                                  */
 375                                 ret = -EAGAIN;
 376                                 break;
 377                         default:
 378                                 dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
 379                         }
 380
 381                         break;
 382                 }
 383         } while (--timeout);
 384
 385         if (timeout == 0) {
 386                 ret = -ETIMEDOUT;
 387                 cmd_status = -ETIMEDOUT;
 388         }
 389
 390         trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
 391
 392         if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
 393                 if (ret == 0)
 394                         dep->flags |= DWC3_EP_TRANSFER_STARTED;
 395
 396                 if (ret != -ETIMEDOUT)
 397                         dwc3_gadget_ep_get_transfer_index(dep);
 398         }
 399
 400         if (saved_config) {
 401                 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
 402                 reg |= saved_config;
 403                 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
 404         }
 405
 406         return ret;
 407 }
 408
 409 static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
 410 {
 411         struct dwc3 *dwc = dep->dwc;
 412         struct dwc3_gadget_ep_cmd_params params;
 413         u32 cmd = DWC3_DEPCMD_CLEARSTALL;
 414
 415         /*
 416          * As of core revision 2.60a the recommended programming model
 417          * is to set the ClearPendIN bit when issuing a Clear Stall EP
 418          * command for IN endpoints. This is to prevent an issue where
 419          * some (non-compliant) hosts may not send ACK TPs for pending
 420          * IN transfers due to a mishandled error condition. Synopsys
 421          * STAR 9000614252.
 422          */
 423         if (dep->direction &&
 424             !DWC3_VER_IS_PRIOR(DWC3, 260A) &&
 425             (dwc->gadget.speed >= USB_SPEED_SUPER))
 426                 cmd |= DWC3_DEPCMD_CLEARPENDIN;
 427
 428         memset(&params, 0, sizeof(params));
 429
 430         return dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 431 }
 432
 433 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
 434                 struct dwc3_trb *trb)
 435 {
 436         u32             offset = (char *) trb - (char *) dep->trb_pool;
 437
 438         return dep->trb_pool_dma + offset;
 439 }
 440
 441 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
 442 {
 443         struct dwc3             *dwc = dep->dwc;
 444
 445         if (dep->trb_pool)
 446                 return 0;
 447
 448         dep->trb_pool = dma_alloc_coherent(dwc->sysdev,
 449                         sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 450                         &dep->trb_pool_dma, GFP_KERNEL);
 451         if (!dep->trb_pool) {
 452                 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
 453                                 dep->name);
 454                 return -ENOMEM;
 455         }
 456
 457         return 0;
 458 }
 459
 460 static void dwc3_free_trb_pool(struct dwc3_ep *dep)
 461 {
 462         struct dwc3             *dwc = dep->dwc;
 463
 464         dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 465                         dep->trb_pool, dep->trb_pool_dma);
 466
 467         dep->trb_pool = NULL;
 468         dep->trb_pool_dma = 0;
 469 }
 470
 471 static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep)
 472 {
 473         struct dwc3_gadget_ep_cmd_params params;
 474
 475         memset(&params, 0x00, sizeof(params));
 476
 477         params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
 478
 479         return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
 480                         &params);
 481 }
 482
 483 /**
 484  * dwc3_gadget_start_config - configure ep resources
 485  * @dep: endpoint that is being enabled
 486  *
 487  * Issue a %DWC3_DEPCMD_DEPSTARTCFG command to @dep. After the command's
 488  * completion, it will set Transfer Resource for all available endpoints.
 489  *
 490  * The assignment of transfer resources cannot perfectly follow the data book
 491  * due to the fact that the controller driver does not have all knowledge of the
 492  * configuration in advance. It is given this information piecemeal by the
 493  * composite gadget framework after every SET_CONFIGURATION and
 494  * SET_INTERFACE. Trying to follow the databook programming model in this
 495  * scenario can cause errors. For two reasons:
 496  *
 497  * 1) The databook says to do %DWC3_DEPCMD_DEPSTARTCFG for every
 498  * %USB_REQ_SET_CONFIGURATION and %USB_REQ_SET_INTERFACE (8.1.5). This is
 499  * incorrect in the scenario of multiple interfaces.
 500  *
 501  * 2) The databook does not mention doing more %DWC3_DEPCMD_DEPXFERCFG for new
 502  * endpoint on alt setting (8.1.6).
 503  *
 504  * The following simplified method is used instead:
 505  *
 506  * All hardware endpoints can be assigned a transfer resource and this setting
 507  * will stay persistent until either a core reset or hibernation. So whenever we
 508  * do a %DWC3_DEPCMD_DEPSTARTCFG(0) we can go ahead and do
 509  * %DWC3_DEPCMD_DEPXFERCFG for every hardware endpoint as well. We are
 510  * guaranteed that there are as many transfer resources as endpoints.
 511  *
 512  * This function is called for each endpoint when it is being enabled but is
 513  * triggered only when called for EP0-out, which always happens first, and which
 514  * should only happen in one of the above conditions.
 515  */
 516 static int dwc3_gadget_start_config(struct dwc3_ep *dep)
 517 {
 518         struct dwc3_gadget_ep_cmd_params params;
 519         struct dwc3             *dwc;
 520         u32                     cmd;
 521         int                     i;
 522         int                     ret;
 523
 524         if (dep->number)
 525                 return 0;
 526
 527         memset(&params, 0x00, sizeof(params));
 528         cmd = DWC3_DEPCMD_DEPSTARTCFG;
 529         dwc = dep->dwc;
 530
 531         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 532         if (ret)
 533                 return ret;
 534
 535         for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
 536                 struct dwc3_ep *dep = dwc->eps[i];
 537
 538                 if (!dep)
 539                         continue;
 540
 541                 ret = dwc3_gadget_set_xfer_resource(dep);
 542                 if (ret)
 543                         return ret;
 544         }
 545
 546         return 0;
 547 }
 548
 549 static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action)
 550 {
 551         const struct usb_ss_ep_comp_descriptor *comp_desc;
 552         const struct usb_endpoint_descriptor *desc;
 553         struct dwc3_gadget_ep_cmd_params params;
 554         struct dwc3 *dwc = dep->dwc;
 555
 556         comp_desc = dep->endpoint.comp_desc;
 557         desc = dep->endpoint.desc;
 558
 559         memset(&params, 0x00, sizeof(params));
 560
 561         params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
 562                 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
 563
 564         /* Burst size is only needed in SuperSpeed mode */
 565         if (dwc->gadget.speed >= USB_SPEED_SUPER) {
 566                 u32 burst = dep->endpoint.maxburst;
 567                 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
 568         }
 569
 570         params.param0 |= action;
 571         if (action == DWC3_DEPCFG_ACTION_RESTORE)
 572                 params.param2 |= dep->saved_state;
 573
 574         if (usb_endpoint_xfer_control(desc))
 575                 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
 576
 577         if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc))
 578                 params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN;
 579
 580         if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
 581                 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
 582                         | DWC3_DEPCFG_XFER_COMPLETE_EN
 583                         | DWC3_DEPCFG_STREAM_EVENT_EN;
 584                 dep->stream_capable = true;
 585         }
 586
 587         if (!usb_endpoint_xfer_control(desc))
 588                 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
 589
 590         /*
 591          * We are doing 1:1 mapping for endpoints, meaning
 592          * Physical Endpoints 2 maps to Logical Endpoint 2 and
 593          * so on. We consider the direction bit as part of the physical
 594          * endpoint number. So USB endpoint 0x81 is 0x03.
 595          */
 596         params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
 597
 598         /*
 599          * We must use the lower 16 TX FIFOs even though
 600          * HW might have more
 601          */
 602         if (dep->direction)
 603                 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
 604
 605         if (desc->bInterval) {
 606                 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1);
 607                 dep->interval = 1 << (desc->bInterval - 1);
 608         }
 609
 610         return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, &params);
 611 }
 612
 613 static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
 614                 bool interrupt);
 615
 616 /**
 617  * __dwc3_gadget_ep_enable - initializes a hw endpoint
 618  * @dep: endpoint to be initialized
 619  * @action: one of INIT, MODIFY or RESTORE
 620  *
 621  * Caller should take care of locking. Execute all necessary commands to
 622  * initialize a HW endpoint so it can be used by a gadget driver.
 623  */
 624 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action)
 625 {
 626         const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
 627         struct dwc3             *dwc = dep->dwc;
 628
 629         u32                     reg;
 630         int                     ret;
 631
 632         if (!(dep->flags & DWC3_EP_ENABLED)) {
 633                 ret = dwc3_gadget_start_config(dep);
 634                 if (ret)
 635                         return ret;
 636         }
 637
 638         ret = dwc3_gadget_set_ep_config(dep, action);
 639         if (ret)
 640                 return ret;
 641
 642         if (!(dep->flags & DWC3_EP_ENABLED)) {
 643                 struct dwc3_trb *trb_st_hw;
 644                 struct dwc3_trb *trb_link;
 645
 646                 dep->type = usb_endpoint_type(desc);
 647                 dep->flags |= DWC3_EP_ENABLED;
 648
 649                 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 650                 reg |= DWC3_DALEPENA_EP(dep->number);
 651                 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 652
 653                 if (usb_endpoint_xfer_control(desc))
 654                         goto out;
 655
 656                 /* Initialize the TRB ring */
 657                 dep->trb_dequeue = 0;
 658                 dep->trb_enqueue = 0;
 659                 memset(dep->trb_pool, 0,
 660                        sizeof(struct dwc3_trb) * DWC3_TRB_NUM);
 661
 662                 /* Link TRB. The HWO bit is never reset */
 663                 trb_st_hw = &dep->trb_pool[0];
 664
 665                 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
 666                 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 667                 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 668                 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
 669                 trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
 670         }
 671
 672         /*
 673          * Issue StartTransfer here with no-op TRB so we can always rely on No
 674          * Response Update Transfer command.
 675          */
 676         if (usb_endpoint_xfer_bulk(desc) ||
 677                         usb_endpoint_xfer_int(desc)) {
 678                 struct dwc3_gadget_ep_cmd_params params;
 679                 struct dwc3_trb *trb;
 680                 dma_addr_t trb_dma;
 681                 u32 cmd;
 682
 683                 memset(&params, 0, sizeof(params));
 684                 trb = &dep->trb_pool[0];
 685                 trb_dma = dwc3_trb_dma_offset(dep, trb);
 686
 687                 params.param0 = upper_32_bits(trb_dma);
 688                 params.param1 = lower_32_bits(trb_dma);
 689
 690                 cmd = DWC3_DEPCMD_STARTTRANSFER;
 691
 692                 ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 693                 if (ret < 0)
 694                         return ret;
 695
 696                 if (dep->stream_capable) {
 697                         /*
 698                          * For streams, at start, there maybe a race where the
 699                          * host primes the endpoint before the function driver
 700                          * queues a request to initiate a stream. In that case,
 701                          * the controller will not see the prime to generate the
 702                          * ERDY and start stream. To workaround this, issue a
 703                          * no-op TRB as normal, but end it immediately. As a
 704                          * result, when the function driver queues the request,
 705                          * the next START_TRANSFER command will cause the
 706                          * controller to generate an ERDY to initiate the
 707                          * stream.
 708                          */
 709                         dwc3_stop_active_transfer(dep, true, true);
 710
 711                         /*
 712                          * All stream eps will reinitiate stream on NoStream
 713                          * rejection until we can determine that the host can
 714                          * prime after the first transfer.
 715                          */
 716                         dep->flags |= DWC3_EP_FORCE_RESTART_STREAM;
 717                 }
 718         }
 719
 720 out:
 721         trace_dwc3_gadget_ep_enable(dep);
 722
 723         return 0;
 724 }
 725
 726 static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
 727 {
 728         struct dwc3_request             *req;
 729
 730         dwc3_stop_active_transfer(dep, true, false);
 731
 732         /* - giveback all requests to gadget driver */
 733         while (!list_empty(&dep->started_list)) {
 734                 req = next_request(&dep->started_list);
 735
 736                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 737         }
 738
 739         while (!list_empty(&dep->pending_list)) {
 740                 req = next_request(&dep->pending_list);
 741
 742                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 743         }
 744
 745         while (!list_empty(&dep->cancelled_list)) {
 746                 req = next_request(&dep->cancelled_list);
 747
 748                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 749         }
 750 }
 751
 752 /**
 753  * __dwc3_gadget_ep_disable - disables a hw endpoint
 754  * @dep: the endpoint to disable
 755  *
 756  * This function undoes what __dwc3_gadget_ep_enable did and also removes
 757  * requests which are currently being processed by the hardware and those which
 758  * are not yet scheduled.
 759  *
 760  * Caller should take care of locking.
 761  */
 762 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
 763 {
 764         struct dwc3             *dwc = dep->dwc;
 765         u32                     reg;
 766
 767         trace_dwc3_gadget_ep_disable(dep);
 768
 769         dwc3_remove_requests(dwc, dep);
 770
 771         /* make sure HW endpoint isn't stalled */
 772         if (dep->flags & DWC3_EP_STALL)
 773                 __dwc3_gadget_ep_set_halt(dep, 0, false);
 774
 775         reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 776         reg &= ~DWC3_DALEPENA_EP(dep->number);
 777         dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 778
 779         dep->stream_capable = false;
 780         dep->type = 0;
 781         dep->flags = 0;
 782
 783         /* Clear out the ep descriptors for non-ep0 */
 784         if (dep->number > 1) {
 785                 dep->endpoint.comp_desc = NULL;
 786                 dep->endpoint.desc = NULL;
 787         }
 788
 789         return 0;
 790 }
 791
 792 /* -------------------------------------------------------------------------- */
 793
 794 static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
 795                 const struct usb_endpoint_descriptor *desc)
 796 {
 797         return -EINVAL;
 798 }
 799
 800 static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
 801 {
 802         return -EINVAL;
 803 }
 804
 805 /* -------------------------------------------------------------------------- */
 806
 807 static int dwc3_gadget_ep_enable(struct usb_ep *ep,
 808                 const struct usb_endpoint_descriptor *desc)
 809 {
 810         struct dwc3_ep                  *dep;
 811         struct dwc3                     *dwc;
 812         unsigned long                   flags;
 813         int                             ret;
 814
 815         if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
 816                 pr_debug("dwc3: invalid parameters\n");
 817                 return -EINVAL;
 818         }
 819
 820         if (!desc->wMaxPacketSize) {
 821                 pr_debug("dwc3: missing wMaxPacketSize\n");
 822                 return -EINVAL;
 823         }
 824
 825         dep = to_dwc3_ep(ep);
 826         dwc = dep->dwc;
 827
 828         if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
 829                                         "%s is already enabled\n",
 830                                         dep->name))
 831                 return 0;
 832
 833         spin_lock_irqsave(&dwc->lock, flags);
 834         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
 835         spin_unlock_irqrestore(&dwc->lock, flags);
 836
 837         return ret;
 838 }
 839
 840 static int dwc3_gadget_ep_disable(struct usb_ep *ep)
 841 {
 842         struct dwc3_ep                  *dep;
 843         struct dwc3                     *dwc;
 844         unsigned long                   flags;
 845         int                             ret;
 846
 847         if (!ep) {
 848                 pr_debug("dwc3: invalid parameters\n");
 849                 return -EINVAL;
 850         }
 851
 852         dep = to_dwc3_ep(ep);
 853         dwc = dep->dwc;
 854
 855         if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
 856                                         "%s is already disabled\n",
 857                                         dep->name))
 858                 return 0;
 859
 860         spin_lock_irqsave(&dwc->lock, flags);
 861         ret = __dwc3_gadget_ep_disable(dep);
 862         spin_unlock_irqrestore(&dwc->lock, flags);
 863
 864         return ret;
 865 }
 866
 867 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
 868                 gfp_t gfp_flags)
 869 {
 870         struct dwc3_request             *req;
 871         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
 872
 873         req = kzalloc(sizeof(*req), gfp_flags);
 874         if (!req)
 875                 return NULL;
 876
 877         req->direction  = dep->direction;
 878         req->epnum      = dep->number;
 879         req->dep        = dep;
 880         req->status     = DWC3_REQUEST_STATUS_UNKNOWN;
 881
 882         trace_dwc3_alloc_request(req);
 883
 884         return &req->request;
 885 }
 886
 887 static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
 888                 struct usb_request *request)
 889 {
 890         struct dwc3_request             *req = to_dwc3_request(request);
 891
 892         trace_dwc3_free_request(req);
 893         kfree(req);
 894 }
 895
 896 /**
 897  * dwc3_ep_prev_trb - returns the previous TRB in the ring
 898  * @dep: The endpoint with the TRB ring
 899  * @index: The index of the current TRB in the ring
 900  *
 901  * Returns the TRB prior to the one pointed to by the index. If the
 902  * index is 0, we will wrap backwards, skip the link TRB, and return
 903  * the one just before that.
 904  */
 905 static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
 906 {
 907         u8 tmp = index;
 908
 909         if (!tmp)
 910                 tmp = DWC3_TRB_NUM - 1;
 911
 912         return &dep->trb_pool[tmp - 1];
 913 }
 914
 915 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
 916 {
 917         struct dwc3_trb         *tmp;
 918         u8                      trbs_left;
 919
 920         /*
 921          * If enqueue & dequeue are equal than it is either full or empty.
 922          *
 923          * One way to know for sure is if the TRB right before us has HWO bit
 924          * set or not. If it has, then we're definitely full and can't fit any
 925          * more transfers in our ring.
 926          */
 927         if (dep->trb_enqueue == dep->trb_dequeue) {
 928                 tmp = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
 929                 if (tmp->ctrl & DWC3_TRB_CTRL_HWO)
 930                         return 0;
 931
 932                 return DWC3_TRB_NUM - 1;
 933         }
 934
 935         trbs_left = dep->trb_dequeue - dep->trb_enqueue;
 936         trbs_left &= (DWC3_TRB_NUM - 1);
 937
 938         if (dep->trb_dequeue < dep->trb_enqueue)
 939                 trbs_left--;
 940
 941         return trbs_left;
 942 }
 943
 944 static void __dwc3_prepare_one_trb(struct dwc3_ep *dep, struct dwc3_trb *trb,
 945                 dma_addr_t dma, unsigned length, unsigned chain, unsigned node,
 946                 unsigned stream_id, unsigned short_not_ok,
 947                 unsigned no_interrupt, unsigned is_last)
 948 {
 949         struct dwc3             *dwc = dep->dwc;
 950         struct usb_gadget       *gadget = &dwc->gadget;
 951         enum usb_device_speed   speed = gadget->speed;
 952
 953         trb->size = DWC3_TRB_SIZE_LENGTH(length);
 954         trb->bpl = lower_32_bits(dma);
 955         trb->bph = upper_32_bits(dma);
 956
 957         switch (usb_endpoint_type(dep->endpoint.desc)) {
 958         case USB_ENDPOINT_XFER_CONTROL:
 959                 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
 960                 break;
 961
 962         case USB_ENDPOINT_XFER_ISOC:
 963                 if (!node) {
 964                         trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
 965
 966                         /*
 967                          * USB Specification 2.0 Section 5.9.2 states that: "If
 968                          * there is only a single transaction in the microframe,
 969                          * only a DATA0 data packet PID is used.  If there are
 970                          * two transactions per microframe, DATA1 is used for
 971                          * the first transaction data packet and DATA0 is used
 972                          * for the second transaction data packet.  If there are
 973                          * three transactions per microframe, DATA2 is used for
 974                          * the first transaction data packet, DATA1 is used for
 975                          * the second, and DATA0 is used for the third."
 976                          *
 977                          * IOW, we should satisfy the following cases:
 978                          *
 979                          * 1) length <= maxpacket
 980                          *      - DATA0
 981                          *
 982                          * 2) maxpacket < length <= (2 * maxpacket)
 983                          *      - DATA1, DATA0
 984                          *
 985                          * 3) (2 * maxpacket) < length <= (3 * maxpacket)
 986                          *      - DATA2, DATA1, DATA0
 987                          */
 988                         if (speed == USB_SPEED_HIGH) {
 989                                 struct usb_ep *ep = &dep->endpoint;
 990                                 unsigned int mult = 2;
 991                                 unsigned int maxp = usb_endpoint_maxp(ep->desc);
 992
 993                                 if (length <= (2 * maxp))
 994                                         mult--;
 995
 996                                 if (length <= maxp)
 997                                         mult--;
 998
 999                                 trb->size |= DWC3_TRB_SIZE_PCM1(mult);
1000                         }
1001                 } else {
1002                         trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
1003                 }
1004
1005                 /* always enable Interrupt on Missed ISOC */
1006                 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1007                 break;
1008
1009         case USB_ENDPOINT_XFER_BULK:
1010         case USB_ENDPOINT_XFER_INT:
1011                 trb->ctrl = DWC3_TRBCTL_NORMAL;
1012                 break;
1013         default:
1014                 /*
1015                  * This is only possible with faulty memory because we
1016                  * checked it already :)
1017                  */
1018                 dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
1019                                 usb_endpoint_type(dep->endpoint.desc));
1020         }
1021
1022         /*
1023          * Enable Continue on Short Packet
1024          * when endpoint is not a stream capable
1025          */
1026         if (usb_endpoint_dir_out(dep->endpoint.desc)) {
1027                 if (!dep->stream_capable)
1028                         trb->ctrl |= DWC3_TRB_CTRL_CSP;
1029
1030                 if (short_not_ok)
1031                         trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1032         }
1033
1034         if ((!no_interrupt && !chain) ||
1035                         (dwc3_calc_trbs_left(dep) == 1))
1036                 trb->ctrl |= DWC3_TRB_CTRL_IOC;
1037
1038         if (chain)
1039                 trb->ctrl |= DWC3_TRB_CTRL_CHN;
1040         else if (dep->stream_capable && is_last)
1041                 trb->ctrl |= DWC3_TRB_CTRL_LST;
1042
1043         if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
1044                 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);
1045
1046         trb->ctrl |= DWC3_TRB_CTRL_HWO;
1047
1048         dwc3_ep_inc_enq(dep);
1049
1050         trace_dwc3_prepare_trb(dep, trb);
1051 }
1052
1053 /**
1054  * dwc3_prepare_one_trb - setup one TRB from one request
1055  * @dep: endpoint for which this request is prepared
1056  * @req: dwc3_request pointer
1057  * @trb_length: buffer size of the TRB
1058  * @chain: should this TRB be chained to the next?
1059  * @node: only for isochronous endpoints. First TRB needs different type.
1060  */
1061 static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
1062                 struct dwc3_request *req, unsigned int trb_length,
1063                 unsigned chain, unsigned node)
1064 {
1065         struct dwc3_trb         *trb;
1066         dma_addr_t              dma;
1067         unsigned                stream_id = req->request.stream_id;
1068         unsigned                short_not_ok = req->request.short_not_ok;
1069         unsigned                no_interrupt = req->request.no_interrupt;
1070         unsigned                is_last = req->request.is_last;
1071
1072         if (req->request.num_sgs > 0)
1073                 dma = sg_dma_address(req->start_sg);
1074         else
1075                 dma = req->request.dma;
1076
1077         trb = &dep->trb_pool[dep->trb_enqueue];
1078
1079         if (!req->trb) {
1080                 dwc3_gadget_move_started_request(req);
1081                 req->trb = trb;
1082                 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
1083         }
1084
1085         req->num_trbs++;
1086
1087         __dwc3_prepare_one_trb(dep, trb, dma, trb_length, chain, node,
1088                         stream_id, short_not_ok, no_interrupt, is_last);
1089 }
1090
1091 static void dwc3_prepare_one_trb_sg(struct dwc3_ep *dep,
1092                 struct dwc3_request *req)
1093 {
1094         struct scatterlist *sg = req->start_sg;
1095         struct scatterlist *s;
1096         int             i;
1097         unsigned int length = req->request.length;
1098         unsigned int remaining = req->request.num_mapped_sgs
1099                 - req->num_queued_sgs;
1100
1101         /*
1102          * If we resume preparing the request, then get the remaining length of
1103          * the request and resume where we left off.
1104          */
1105         for_each_sg(req->request.sg, s, req->num_queued_sgs, i)
1106                 length -= sg_dma_len(s);
1107
1108         for_each_sg(sg, s, remaining, i) {
1109                 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1110                 unsigned int rem = length % maxp;
1111                 unsigned int trb_length;
1112                 unsigned chain = true;
1113
1114                 trb_length = min_t(unsigned int, length, sg_dma_len(s));
1115
1116                 length -= trb_length;
1117
1118                 /*
1119                  * IOMMU driver is coalescing the list of sgs which shares a
1120                  * page boundary into one and giving it to USB driver. With
1121                  * this the number of sgs mapped is not equal to the number of
1122                  * sgs passed. So mark the chain bit to false if it isthe last
1123                  * mapped sg.
1124                  */
1125                 if ((i == remaining - 1) || !length)
1126                         chain = false;
1127
1128                 if (rem && usb_endpoint_dir_out(dep->endpoint.desc) && !chain) {
1129                         struct dwc3     *dwc = dep->dwc;
1130                         struct dwc3_trb *trb;
1131
1132                         req->needs_extra_trb = true;
1133
1134                         /* prepare normal TRB */
1135                         dwc3_prepare_one_trb(dep, req, trb_length, true, i);
1136
1137                         /* Now prepare one extra TRB to align transfer size */
1138                         trb = &dep->trb_pool[dep->trb_enqueue];
1139                         req->num_trbs++;
1140                         __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr,
1141                                         maxp - rem, false, 1,
1142                                         req->request.stream_id,
1143                                         req->request.short_not_ok,
1144                                         req->request.no_interrupt,
1145                                         req->request.is_last);
1146                 } else if (req->request.zero && req->request.length &&
1147                            !usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
1148                            !rem && !chain) {
1149                         struct dwc3     *dwc = dep->dwc;
1150                         struct dwc3_trb *trb;
1151
1152                         req->needs_extra_trb = true;
1153
1154                         /* Prepare normal TRB */
1155                         dwc3_prepare_one_trb(dep, req, trb_length, true, i);
1156
1157                         /* Prepare one extra TRB to handle ZLP */
1158                         trb = &dep->trb_pool[dep->trb_enqueue];
1159                         req->num_trbs++;
1160                         __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, 0,
1161                                                !req->direction, 1,
1162                                                req->request.stream_id,
1163                                                req->request.short_not_ok,
1164                                                req->request.no_interrupt,
1165                                                req->request.is_last);
1166
1167                         /* Prepare one more TRB to handle MPS alignment */
1168                         if (!req->direction) {
1169                                 trb = &dep->trb_pool[dep->trb_enqueue];
1170                                 req->num_trbs++;
1171                                 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, maxp,
1172                                                        false, 1, req->request.stream_id,
1173                                                        req->request.short_not_ok,
1174                                                        req->request.no_interrupt,
1175                                                        req->request.is_last);
1176                         }
1177                 } else {
1178                         dwc3_prepare_one_trb(dep, req, trb_length, chain, i);
1179                 }
1180
1181                 /*
1182                  * There can be a situation where all sgs in sglist are not
1183                  * queued because of insufficient trb number. To handle this
1184                  * case, update start_sg to next sg to be queued, so that
1185                  * we have free trbs we can continue queuing from where we
1186                  * previously stopped
1187                  */
1188                 if (chain)
1189                         req->start_sg = sg_next(s);
1190
1191                 req->num_queued_sgs++;
1192
1193                 /*
1194                  * The number of pending SG entries may not correspond to the
1195                  * number of mapped SG entries. If all the data are queued, then
1196                  * don't include unused SG entries.
1197                  */
1198                 if (length == 0) {
1199                         req->num_pending_sgs -= req->request.num_mapped_sgs - req->num_queued_sgs;
1200                         break;
1201                 }
1202
1203                 if (!dwc3_calc_trbs_left(dep))
1204                         break;
1205         }
1206 }
1207
1208 static void dwc3_prepare_one_trb_linear(struct dwc3_ep *dep,
1209                 struct dwc3_request *req)
1210 {
1211         unsigned int length = req->request.length;
1212         unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1213         unsigned int rem = length % maxp;
1214
1215         if ((!length || rem) && usb_endpoint_dir_out(dep->endpoint.desc)) {
1216                 struct dwc3     *dwc = dep->dwc;
1217                 struct dwc3_trb *trb;
1218
1219                 req->needs_extra_trb = true;
1220
1221                 /* prepare normal TRB */
1222                 dwc3_prepare_one_trb(dep, req, length, true, 0);
1223
1224                 /* Now prepare one extra TRB to align transfer size */
1225                 trb = &dep->trb_pool[dep->trb_enqueue];
1226                 req->num_trbs++;
1227                 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, maxp - rem,
1228                                 false, 1, req->request.stream_id,
1229                                 req->request.short_not_ok,
1230                                 req->request.no_interrupt,
1231                                 req->request.is_last);
1232         } else if (req->request.zero && req->request.length &&
1233                    !usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
1234                    (IS_ALIGNED(req->request.length, maxp))) {
1235                 struct dwc3     *dwc = dep->dwc;
1236                 struct dwc3_trb *trb;
1237
1238                 req->needs_extra_trb = true;
1239
1240                 /* prepare normal TRB */
1241                 dwc3_prepare_one_trb(dep, req, length, true, 0);
1242
1243                 /* Prepare one extra TRB to handle ZLP */
1244                 trb = &dep->trb_pool[dep->trb_enqueue];
1245                 req->num_trbs++;
1246                 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, 0,
1247                                 !req->direction, 1, req->request.stream_id,
1248                                 req->request.short_not_ok,
1249                                 req->request.no_interrupt,
1250                                 req->request.is_last);
1251
1252                 /* Prepare one more TRB to handle MPS alignment for OUT */
1253                 if (!req->direction) {
1254                         trb = &dep->trb_pool[dep->trb_enqueue];
1255                         req->num_trbs++;
1256                         __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, maxp,
1257                                                false, 1, req->request.stream_id,
1258                                                req->request.short_not_ok,
1259                                                req->request.no_interrupt,
1260                                                req->request.is_last);
1261                 }
1262         } else {
1263                 dwc3_prepare_one_trb(dep, req, length, false, 0);
1264         }
1265 }
1266
1267 /*
1268  * dwc3_prepare_trbs - setup TRBs from requests
1269  * @dep: endpoint for which requests are being prepared
1270  *
1271  * The function goes through the requests list and sets up TRBs for the
1272  * transfers. The function returns once there are no more TRBs available or
1273  * it runs out of requests.
1274  */
1275 static void dwc3_prepare_trbs(struct dwc3_ep *dep)
1276 {
1277         struct dwc3_request     *req, *n;
1278
1279         BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
1280
1281         /*
1282          * We can get in a situation where there's a request in the started list
1283          * but there weren't enough TRBs to fully kick it in the first time
1284          * around, so it has been waiting for more TRBs to be freed up.
1285          *
1286          * In that case, we should check if we have a request with pending_sgs
1287          * in the started list and prepare TRBs for that request first,
1288          * otherwise we will prepare TRBs completely out of order and that will
1289          * break things.
1290          */
1291         list_for_each_entry(req, &dep->started_list, list) {
1292                 if (req->num_pending_sgs > 0)
1293                         dwc3_prepare_one_trb_sg(dep, req);
1294
1295                 if (!dwc3_calc_trbs_left(dep))
1296                         return;
1297
1298                 /*
1299                  * Don't prepare beyond a transfer. In DWC_usb32, its transfer
1300                  * burst capability may try to read and use TRBs beyond the
1301                  * active transfer instead of stopping.
1302                  */
1303                 if (dep->stream_capable && req->request.is_last)
1304                         return;
1305         }
1306
1307         list_for_each_entry_safe(req, n, &dep->pending_list, list) {
1308                 struct dwc3     *dwc = dep->dwc;
1309                 int             ret;
1310
1311                 ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request,
1312                                                     dep->direction);
1313                 if (ret)
1314                         return;
1315
1316                 req->sg                 = req->request.sg;
1317                 req->start_sg           = req->sg;
1318                 req->num_queued_sgs     = 0;
1319                 req->num_pending_sgs    = req->request.num_mapped_sgs;
1320
1321                 if (req->num_pending_sgs > 0)
1322                         dwc3_prepare_one_trb_sg(dep, req);
1323                 else
1324                         dwc3_prepare_one_trb_linear(dep, req);
1325
1326                 if (!dwc3_calc_trbs_left(dep))
1327                         return;
1328
1329                 /*
1330                  * Don't prepare beyond a transfer. In DWC_usb32, its transfer
1331                  * burst capability may try to read and use TRBs beyond the
1332                  * active transfer instead of stopping.
1333                  */
1334                 if (dep->stream_capable && req->request.is_last)
1335                         return;
1336         }
1337 }
1338
1339 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep);
1340
1341 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
1342 {
1343         struct dwc3_gadget_ep_cmd_params params;
1344         struct dwc3_request             *req;
1345         int                             starting;
1346         int                             ret;
1347         u32                             cmd;
1348
1349         if (!dwc3_calc_trbs_left(dep))
1350                 return 0;
1351
1352         starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED);
1353
1354         dwc3_prepare_trbs(dep);
1355         req = next_request(&dep->started_list);
1356         if (!req) {
1357                 dep->flags |= DWC3_EP_PENDING_REQUEST;
1358                 return 0;
1359         }
1360
1361         memset(&params, 0, sizeof(params));
1362
1363         if (starting) {
1364                 params.param0 = upper_32_bits(req->trb_dma);
1365                 params.param1 = lower_32_bits(req->trb_dma);
1366                 cmd = DWC3_DEPCMD_STARTTRANSFER;
1367
1368                 if (dep->stream_capable)
1369                         cmd |= DWC3_DEPCMD_PARAM(req->request.stream_id);
1370
1371                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
1372                         cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);
1373         } else {
1374                 cmd = DWC3_DEPCMD_UPDATETRANSFER |
1375                         DWC3_DEPCMD_PARAM(dep->resource_index);
1376         }
1377
1378         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
1379         if (ret < 0) {
1380                 struct dwc3_request *tmp;
1381
1382                 if (ret == -EAGAIN)
1383                         return ret;
1384
1385                 dwc3_stop_active_transfer(dep, true, true);
1386
1387                 list_for_each_entry_safe(req, tmp, &dep->started_list, list)
1388                         dwc3_gadget_move_cancelled_request(req);
1389
1390                 /* If ep isn't started, then there's no end transfer pending */
1391                 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
1392                         dwc3_gadget_ep_cleanup_cancelled_requests(dep);
1393
1394                 return ret;
1395         }
1396
1397         if (dep->stream_capable && req->request.is_last)
1398                 dep->flags |= DWC3_EP_WAIT_TRANSFER_COMPLETE;
1399
1400         return 0;
1401 }
1402
1403 static int __dwc3_gadget_get_frame(struct dwc3 *dwc)
1404 {
1405         u32                     reg;
1406
1407         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1408         return DWC3_DSTS_SOFFN(reg);
1409 }
1410
1411 /**
1412  * dwc3_gadget_start_isoc_quirk - workaround invalid frame number
1413  * @dep: isoc endpoint
1414  *
1415  * This function tests for the correct combination of BIT[15:14] from the 16-bit
1416  * microframe number reported by the XferNotReady event for the future frame
1417  * number to start the isoc transfer.
1418  *
1419  * In DWC_usb31 version 1.70a-ea06 and prior, for highspeed and fullspeed
1420  * isochronous IN, BIT[15:14] of the 16-bit microframe number reported by the
1421  * XferNotReady event are invalid. The driver uses this number to schedule the
1422  * isochronous transfer and passes it to the START TRANSFER command. Because
1423  * this number is invalid, the command may fail. If BIT[15:14] matches the
1424  * internal 16-bit microframe, the START TRANSFER command will pass and the
1425  * transfer will start at the scheduled time, if it is off by 1, the command
1426  * will still pass, but the transfer will start 2 seconds in the future. For all
1427  * other conditions, the START TRANSFER command will fail with bus-expiry.
1428  *
1429  * In order to workaround this issue, we can test for the correct combination of
1430  * BIT[15:14] by sending START TRANSFER commands with different values of
1431  * BIT[15:14]: 'b00, 'b01, 'b10, and 'b11. Each combination is 2^14 uframe apart
1432  * (or 2 seconds). 4 seconds into the future will result in a bus-expiry status.
1433  * As the result, within the 4 possible combinations for BIT[15:14], there will
1434  * be 2 successful and 2 failure START COMMAND status. One of the 2 successful
1435  * command status will result in a 2-second delay start. The smaller BIT[15:14]
1436  * value is the correct combination.
1437  *
1438  * Since there are only 4 outcomes and the results are ordered, we can simply
1439  * test 2 START TRANSFER commands with BIT[15:14] combinations 'b00 and 'b01 to
1440  * deduce the smaller successful combination.
1441  *
1442  * Let test0 = test status for combination 'b00 and test1 = test status for 'b01
1443  * of BIT[15:14]. The correct combination is as follow:
1444  *
1445  * if test0 fails and test1 passes, BIT[15:14] is 'b01
1446  * if test0 fails and test1 fails, BIT[15:14] is 'b10
1447  * if test0 passes and test1 fails, BIT[15:14] is 'b11
1448  * if test0 passes and test1 passes, BIT[15:14] is 'b00
1449  *
1450  * Synopsys STAR 9001202023: Wrong microframe number for isochronous IN
1451  * endpoints.
1452  */
1453 static int dwc3_gadget_start_isoc_quirk(struct dwc3_ep *dep)
1454 {
1455         int cmd_status = 0;
1456         bool test0;
1457         bool test1;
1458
1459         while (dep->combo_num < 2) {
1460                 struct dwc3_gadget_ep_cmd_params params;
1461                 u32 test_frame_number;
1462                 u32 cmd;
1463
1464                 /*
1465                  * Check if we can start isoc transfer on the next interval or
1466                  * 4 uframes in the future with BIT[15:14] as dep->combo_num
1467                  */
1468                 test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK;
1469                 test_frame_number |= dep->combo_num << 14;
1470                 test_frame_number += max_t(u32, 4, dep->interval);
1471
1472                 params.param0 = upper_32_bits(dep->dwc->bounce_addr);
1473                 params.param1 = lower_32_bits(dep->dwc->bounce_addr);
1474
1475                 cmd = DWC3_DEPCMD_STARTTRANSFER;
1476                 cmd |= DWC3_DEPCMD_PARAM(test_frame_number);
1477                 cmd_status = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
1478
1479                 /* Redo if some other failure beside bus-expiry is received */
1480                 if (cmd_status && cmd_status != -EAGAIN) {
1481                         dep->start_cmd_status = 0;
1482                         dep->combo_num = 0;
1483                         return 0;
1484                 }
1485
1486                 /* Store the first test status */
1487                 if (dep->combo_num == 0)
1488                         dep->start_cmd_status = cmd_status;
1489
1490                 dep->combo_num++;
1491
1492                 /*
1493                  * End the transfer if the START_TRANSFER command is successful
1494                  * to wait for the next XferNotReady to test the command again
1495                  */
1496                 if (cmd_status == 0) {
1497                         dwc3_stop_active_transfer(dep, true, true);
1498                         return 0;
1499                 }
1500         }
1501
1502         /* test0 and test1 are both completed at this point */
1503         test0 = (dep->start_cmd_status == 0);
1504         test1 = (cmd_status == 0);
1505
1506         if (!test0 && test1)
1507                 dep->combo_num = 1;
1508         else if (!test0 && !test1)
1509                 dep->combo_num = 2;
1510         else if (test0 && !test1)
1511                 dep->combo_num = 3;
1512         else if (test0 && test1)
1513                 dep->combo_num = 0;
1514
1515         dep->frame_number &= DWC3_FRNUMBER_MASK;
1516         dep->frame_number |= dep->combo_num << 14;
1517         dep->frame_number += max_t(u32, 4, dep->interval);
1518
1519         /* Reinitialize test variables */
1520         dep->start_cmd_status = 0;
1521         dep->combo_num = 0;
1522
1523         return __dwc3_gadget_kick_transfer(dep);
1524 }
1525
1526 static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep)
1527 {
1528         const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
1529         struct dwc3 *dwc = dep->dwc;
1530         int ret;
1531         int i;
1532
1533         if (list_empty(&dep->pending_list) &&
1534             list_empty(&dep->started_list)) {
1535                 dep->flags |= DWC3_EP_PENDING_REQUEST;
1536                 return -EAGAIN;
1537         }
1538
1539         if (!dwc->dis_start_transfer_quirk &&
1540             (DWC3_VER_IS_PRIOR(DWC31, 170A) ||
1541              DWC3_VER_TYPE_IS_WITHIN(DWC31, 170A, EA01, EA06))) {
1542                 if (dwc->gadget.speed <= USB_SPEED_HIGH && dep->direction)
1543                         return dwc3_gadget_start_isoc_quirk(dep);
1544         }
1545
1546         if (desc->bInterval <= 14 &&
1547             dwc->gadget.speed >= USB_SPEED_HIGH) {
1548                 u32 frame = __dwc3_gadget_get_frame(dwc);
1549                 bool rollover = frame <
1550                                 (dep->frame_number & DWC3_FRNUMBER_MASK);
1551
1552                 /*
1553                  * frame_number is set from XferNotReady and may be already
1554                  * out of date. DSTS only provides the lower 14 bit of the
1555                  * current frame number. So add the upper two bits of
1556                  * frame_number and handle a possible rollover.
1557                  * This will provide the correct frame_number unless more than
1558                  * rollover has happened since XferNotReady.
1559                  */
1560
1561                 dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) |
1562                                      frame;
1563                 if (rollover)
1564                         dep->frame_number += BIT(14);
1565         }
1566
1567         for (i = 0; i < DWC3_ISOC_MAX_RETRIES; i++) {
1568                 dep->frame_number = DWC3_ALIGN_FRAME(dep, i + 1);
1569
1570                 ret = __dwc3_gadget_kick_transfer(dep);
1571                 if (ret != -EAGAIN)
1572                         break;
1573         }
1574
1575         /*
1576          * After a number of unsuccessful start attempts due to bus-expiry
1577          * status, issue END_TRANSFER command and retry on the next XferNotReady
1578          * event.
1579          */
1580         if (ret == -EAGAIN) {
1581                 struct dwc3_gadget_ep_cmd_params params;
1582                 u32 cmd;
1583
1584                 cmd = DWC3_DEPCMD_ENDTRANSFER |
1585                         DWC3_DEPCMD_CMDIOC |
1586                         DWC3_DEPCMD_PARAM(dep->resource_index);
1587
1588                 dep->resource_index = 0;
1589                 memset(&params, 0, sizeof(params));
1590
1591                 ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
1592                 if (!ret)
1593                         dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
1594         }
1595
1596         return ret;
1597 }
1598
1599 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1600 {
1601         struct dwc3             *dwc = dep->dwc;
1602
1603         if (!dep->endpoint.desc) {
1604                 dev_err(dwc->dev, "%s: can't queue to disabled endpoint\n",
1605                                 dep->name);
1606                 return -ESHUTDOWN;
1607         }
1608
1609         if (WARN(req->dep != dep, "request %pK belongs to '%s'\n",
1610                                 &req->request, req->dep->name))
1611                 return -EINVAL;
1612
1613         if (WARN(req->status < DWC3_REQUEST_STATUS_COMPLETED,
1614                                 "%s: request %pK already in flight\n",
1615                                 dep->name, &req->request))
1616                 return -EINVAL;
1617
1618         pm_runtime_get(dwc->dev);
1619
1620         req->request.actual     = 0;
1621         req->request.status     = -EINPROGRESS;
1622
1623         trace_dwc3_ep_queue(req);
1624
1625         list_add_tail(&req->list, &dep->pending_list);
1626         req->status = DWC3_REQUEST_STATUS_QUEUED;
1627
1628         if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)
1629                 return 0;
1630
1631         /* Start the transfer only after the END_TRANSFER is completed */
1632         if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) {
1633                 dep->flags |= DWC3_EP_DELAY_START;
1634                 return 0;
1635         }
1636
1637         /*
1638          * NOTICE: Isochronous endpoints should NEVER be prestarted. We must
1639          * wait for a XferNotReady event so we will know what's the current
1640          * (micro-)frame number.
1641          *
1642          * Without this trick, we are very, very likely gonna get Bus Expiry
1643          * errors which will force us issue EndTransfer command.
1644          */
1645         if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1646                 if (!(dep->flags & DWC3_EP_PENDING_REQUEST) &&
1647                                 !(dep->flags & DWC3_EP_TRANSFER_STARTED))
1648                         return 0;
1649
1650                 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) {
1651                         if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) {
1652                                 return __dwc3_gadget_start_isoc(dep);
1653                         }
1654                 }
1655         }
1656
1657         return __dwc3_gadget_kick_transfer(dep);
1658 }
1659
1660 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1661         gfp_t gfp_flags)
1662 {
1663         struct dwc3_request             *req = to_dwc3_request(request);
1664         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1665         struct dwc3                     *dwc = dep->dwc;
1666
1667         unsigned long                   flags;
1668
1669         int                             ret;
1670
1671         spin_lock_irqsave(&dwc->lock, flags);
1672         ret = __dwc3_gadget_ep_queue(dep, req);
1673         spin_unlock_irqrestore(&dwc->lock, flags);
1674
1675         return ret;
1676 }
1677
1678 static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep *dep, struct dwc3_request *req)
1679 {
1680         int i;
1681
1682         /* If req->trb is not set, then the request has not started */
1683         if (!req->trb)
1684                 return;
1685
1686         /*
1687          * If request was already started, this means we had to
1688          * stop the transfer. With that we also need to ignore
1689          * all TRBs used by the request, however TRBs can only
1690          * be modified after completion of END_TRANSFER
1691          * command. So what we do here is that we wait for
1692          * END_TRANSFER completion and only after that, we jump
1693          * over TRBs by clearing HWO and incrementing dequeue
1694          * pointer.
1695          */
1696         for (i = 0; i < req->num_trbs; i++) {
1697                 struct dwc3_trb *trb;
1698
1699                 trb = &dep->trb_pool[dep->trb_dequeue];
1700                 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1701                 dwc3_ep_inc_deq(dep);
1702         }
1703
1704         req->num_trbs = 0;
1705 }
1706
1707 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep)
1708 {
1709         struct dwc3_request             *req;
1710         struct dwc3_request             *tmp;
1711
1712         list_for_each_entry_safe(req, tmp, &dep->cancelled_list, list) {
1713                 dwc3_gadget_ep_skip_trbs(dep, req);
1714                 dwc3_gadget_giveback(dep, req, -ECONNRESET);
1715         }
1716 }
1717
1718 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
1719                 struct usb_request *request)
1720 {
1721         struct dwc3_request             *req = to_dwc3_request(request);
1722         struct dwc3_request             *r = NULL;
1723
1724         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1725         struct dwc3                     *dwc = dep->dwc;
1726
1727         unsigned long                   flags;
1728         int                             ret = 0;
1729
1730         trace_dwc3_ep_dequeue(req);
1731
1732         spin_lock_irqsave(&dwc->lock, flags);
1733
1734         list_for_each_entry(r, &dep->cancelled_list, list) {
1735                 if (r == req)
1736                         goto out;
1737         }
1738
1739         list_for_each_entry(r, &dep->pending_list, list) {
1740                 if (r == req) {
1741                         dwc3_gadget_giveback(dep, req, -ECONNRESET);
1742                         goto out;
1743                 }
1744         }
1745
1746         list_for_each_entry(r, &dep->started_list, list) {
1747                 if (r == req) {
1748                         struct dwc3_request *t;
1749
1750                         /* wait until it is processed */
1751                         dwc3_stop_active_transfer(dep, true, true);
1752
1753                         /*
1754                          * Remove any started request if the transfer is
1755                          * cancelled.
1756                          */
1757                         list_for_each_entry_safe(r, t, &dep->started_list, list)
1758                                 dwc3_gadget_move_cancelled_request(r);
1759
1760                         goto out;
1761                 }
1762         }
1763
1764         dev_err(dwc->dev, "request %pK was not queued to %s\n",
1765                 request, ep->name);
1766         ret = -EINVAL;
1767 out:
1768         spin_unlock_irqrestore(&dwc->lock, flags);
1769
1770         return ret;
1771 }
1772
1773 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
1774 {
1775         struct dwc3_gadget_ep_cmd_params        params;
1776         struct dwc3                             *dwc = dep->dwc;
1777         struct dwc3_request                     *req;
1778         struct dwc3_request                     *tmp;
1779         int                                     ret;
1780
1781         if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1782                 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
1783                 return -EINVAL;
1784         }
1785
1786         memset(&params, 0x00, sizeof(params));
1787
1788         if (value) {
1789                 struct dwc3_trb *trb;
1790
1791                 unsigned transfer_in_flight;
1792                 unsigned started;
1793
1794                 if (dep->number > 1)
1795                         trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
1796                 else
1797                         trb = &dwc->ep0_trb[dep->trb_enqueue];
1798
1799                 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO;
1800                 started = !list_empty(&dep->started_list);
1801
1802                 if (!protocol && ((dep->direction && transfer_in_flight) ||
1803                                 (!dep->direction && started))) {
1804                         return -EAGAIN;
1805                 }
1806
1807                 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
1808                                 &params);
1809                 if (ret)
1810                         dev_err(dwc->dev, "failed to set STALL on %s\n",
1811                                         dep->name);
1812                 else
1813                         dep->flags |= DWC3_EP_STALL;
1814         } else {
1815                 /*
1816                  * Don't issue CLEAR_STALL command to control endpoints. The
1817                  * controller automatically clears the STALL when it receives
1818                  * the SETUP token.
1819                  */
1820                 if (dep->number <= 1) {
1821                         dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
1822                         return 0;
1823                 }
1824
1825                 ret = dwc3_send_clear_stall_ep_cmd(dep);
1826                 if (ret) {
1827                         dev_err(dwc->dev, "failed to clear STALL on %s\n",
1828                                         dep->name);
1829                         return ret;
1830                 }
1831
1832                 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
1833
1834                 dwc3_stop_active_transfer(dep, true, true);
1835
1836                 list_for_each_entry_safe(req, tmp, &dep->started_list, list)
1837                         dwc3_gadget_move_cancelled_request(req);
1838
1839                 list_for_each_entry_safe(req, tmp, &dep->pending_list, list)
1840                         dwc3_gadget_move_cancelled_request(req);
1841
1842                 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING)) {
1843                         dep->flags &= ~DWC3_EP_DELAY_START;
1844                         dwc3_gadget_ep_cleanup_cancelled_requests(dep);
1845                 }
1846         }
1847
1848         return ret;
1849 }
1850
1851 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
1852 {
1853         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1854         struct dwc3                     *dwc = dep->dwc;
1855
1856         unsigned long                   flags;
1857
1858         int                             ret;
1859
1860         spin_lock_irqsave(&dwc->lock, flags);
1861         ret = __dwc3_gadget_ep_set_halt(dep, value, false);
1862         spin_unlock_irqrestore(&dwc->lock, flags);
1863
1864         return ret;
1865 }
1866
1867 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
1868 {
1869         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1870         struct dwc3                     *dwc = dep->dwc;
1871         unsigned long                   flags;
1872         int                             ret;
1873
1874         spin_lock_irqsave(&dwc->lock, flags);
1875         dep->flags |= DWC3_EP_WEDGE;
1876
1877         if (dep->number == 0 || dep->number == 1)
1878                 ret = __dwc3_gadget_ep0_set_halt(ep, 1);
1879         else
1880                 ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
1881         spin_unlock_irqrestore(&dwc->lock, flags);
1882
1883         return ret;
1884 }
1885
1886 /* -------------------------------------------------------------------------- */
1887
1888 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
1889         .bLength        = USB_DT_ENDPOINT_SIZE,
1890         .bDescriptorType = USB_DT_ENDPOINT,
1891         .bmAttributes   = USB_ENDPOINT_XFER_CONTROL,
1892 };
1893
1894 static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
1895         .enable         = dwc3_gadget_ep0_enable,
1896         .disable        = dwc3_gadget_ep0_disable,
1897         .alloc_request  = dwc3_gadget_ep_alloc_request,
1898         .free_request   = dwc3_gadget_ep_free_request,
1899         .queue          = dwc3_gadget_ep0_queue,
1900         .dequeue        = dwc3_gadget_ep_dequeue,
1901         .set_halt       = dwc3_gadget_ep0_set_halt,
1902         .set_wedge      = dwc3_gadget_ep_set_wedge,
1903 };
1904
1905 static const struct usb_ep_ops dwc3_gadget_ep_ops = {
1906         .enable         = dwc3_gadget_ep_enable,
1907         .disable        = dwc3_gadget_ep_disable,
1908         .alloc_request  = dwc3_gadget_ep_alloc_request,
1909         .free_request   = dwc3_gadget_ep_free_request,
1910         .queue          = dwc3_gadget_ep_queue,
1911         .dequeue        = dwc3_gadget_ep_dequeue,
1912         .set_halt       = dwc3_gadget_ep_set_halt,
1913         .set_wedge      = dwc3_gadget_ep_set_wedge,
1914 };
1915
1916 /* -------------------------------------------------------------------------- */
1917
1918 static int dwc3_gadget_get_frame(struct usb_gadget *g)
1919 {
1920         struct dwc3             *dwc = gadget_to_dwc(g);
1921
1922         return __dwc3_gadget_get_frame(dwc);
1923 }
1924
1925 static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
1926 {
1927         int                     retries;
1928
1929         int                     ret;
1930         u32                     reg;
1931
1932         u8                      link_state;
1933
1934         /*
1935          * According to the Databook Remote wakeup request should
1936          * be issued only when the device is in early suspend state.
1937          *
1938          * We can check that via USB Link State bits in DSTS register.
1939          */
1940         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1941
1942         link_state = DWC3_DSTS_USBLNKST(reg);
1943
1944         switch (link_state) {
1945         case DWC3_LINK_STATE_RESET:
1946         case DWC3_LINK_STATE_RX_DET:    /* in HS, means Early Suspend */
1947         case DWC3_LINK_STATE_U3:        /* in HS, means SUSPEND */
1948         case DWC3_LINK_STATE_RESUME:
1949                 break;
1950         default:
1951                 return -EINVAL;
1952         }
1953
1954         ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
1955         if (ret < 0) {
1956                 dev_err(dwc->dev, "failed to put link in Recovery\n");
1957                 return ret;
1958         }
1959
1960         /* Recent versions do this automatically */
1961         if (DWC3_VER_IS_PRIOR(DWC3, 194A)) {
1962                 /* write zeroes to Link Change Request */
1963                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1964                 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
1965                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1966         }
1967
1968         /* poll until Link State changes to ON */
1969         retries = 20000;
1970
1971         while (retries--) {
1972                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1973
1974                 /* in HS, means ON */
1975                 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
1976                         break;
1977         }
1978
1979         if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
1980                 dev_err(dwc->dev, "failed to send remote wakeup\n");
1981                 return -EINVAL;
1982         }
1983
1984         return 0;
1985 }
1986
1987 static int dwc3_gadget_wakeup(struct usb_gadget *g)
1988 {
1989         struct dwc3             *dwc = gadget_to_dwc(g);
1990         unsigned long           flags;
1991         int                     ret;
1992
1993         spin_lock_irqsave(&dwc->lock, flags);
1994         ret = __dwc3_gadget_wakeup(dwc);
1995         spin_unlock_irqrestore(&dwc->lock, flags);
1996
1997         return ret;
1998 }
1999
2000 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
2001                 int is_selfpowered)
2002 {
2003         struct dwc3             *dwc = gadget_to_dwc(g);
2004         unsigned long           flags;
2005
2006         spin_lock_irqsave(&dwc->lock, flags);
2007         g->is_selfpowered = !!is_selfpowered;
2008         spin_unlock_irqrestore(&dwc->lock, flags);
2009
2010         return 0;
2011 }
2012
2013 static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
2014 {
2015         u32                     reg;
2016         u32                     timeout = 500;
2017
2018         if (pm_runtime_suspended(dwc->dev))
2019                 return 0;
2020
2021         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2022         if (is_on) {
2023                 if (DWC3_VER_IS_WITHIN(DWC3, ANY, 187A)) {
2024                         reg &= ~DWC3_DCTL_TRGTULST_MASK;
2025                         reg |= DWC3_DCTL_TRGTULST_RX_DET;
2026                 }
2027
2028                 if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
2029                         reg &= ~DWC3_DCTL_KEEP_CONNECT;
2030                 reg |= DWC3_DCTL_RUN_STOP;
2031
2032                 if (dwc->has_hibernation)
2033                         reg |= DWC3_DCTL_KEEP_CONNECT;
2034
2035                 dwc->pullups_connected = true;
2036         } else {
2037                 reg &= ~DWC3_DCTL_RUN_STOP;
2038
2039                 if (dwc->has_hibernation && !suspend)
2040                         reg &= ~DWC3_DCTL_KEEP_CONNECT;
2041
2042                 dwc->pullups_connected = false;
2043         }
2044
2045         dwc3_gadget_dctl_write_safe(dwc, reg);
2046
2047         do {
2048                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2049                 reg &= DWC3_DSTS_DEVCTRLHLT;
2050         } while (--timeout && !(!is_on ^ !reg));
2051
2052         if (!timeout)
2053                 return -ETIMEDOUT;
2054
2055         return 0;
2056 }
2057
2058 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
2059 {
2060         struct dwc3             *dwc = gadget_to_dwc(g);
2061         unsigned long           flags;
2062         int                     ret;
2063
2064         is_on = !!is_on;
2065
2066         /*
2067          * Per databook, when we want to stop the gadget, if a control transfer
2068          * is still in process, complete it and get the core into setup phase.
2069          */
2070         if (!is_on && dwc->ep0state != EP0_SETUP_PHASE) {
2071                 reinit_completion(&dwc->ep0_in_setup);
2072
2073                 ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
2074                                 msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
2075                 if (ret == 0) {
2076                         dev_err(dwc->dev, "timed out waiting for SETUP phase\n");
2077                         return -ETIMEDOUT;
2078                 }
2079         }
2080
2081         spin_lock_irqsave(&dwc->lock, flags);
2082         ret = dwc3_gadget_run_stop(dwc, is_on, false);
2083         spin_unlock_irqrestore(&dwc->lock, flags);
2084
2085         return ret;
2086 }
2087
2088 static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
2089 {
2090         u32                     reg;
2091
2092         /* Enable all but Start and End of Frame IRQs */
2093         reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
2094                         DWC3_DEVTEN_EVNTOVERFLOWEN |
2095                         DWC3_DEVTEN_CMDCMPLTEN |
2096                         DWC3_DEVTEN_ERRTICERREN |
2097                         DWC3_DEVTEN_WKUPEVTEN |
2098                         DWC3_DEVTEN_CONNECTDONEEN |
2099                         DWC3_DEVTEN_USBRSTEN |
2100                         DWC3_DEVTEN_DISCONNEVTEN);
2101
2102         if (DWC3_VER_IS_PRIOR(DWC3, 250A))
2103                 reg |= DWC3_DEVTEN_ULSTCNGEN;
2104
2105         dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
2106 }
2107
2108 static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
2109 {
2110         /* mask all interrupts */
2111         dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
2112 }
2113
2114 static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
2115 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
2116
2117 /**
2118  * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG
2119  * @dwc: pointer to our context structure
2120  *
2121  * The following looks like complex but it's actually very simple. In order to
2122  * calculate the number of packets we can burst at once on OUT transfers, we're
2123  * gonna use RxFIFO size.
2124  *
2125  * To calculate RxFIFO size we need two numbers:
2126  * MDWIDTH = size, in bits, of the internal memory bus
2127  * RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits)
2128  *
2129  * Given these two numbers, the formula is simple:
2130  *
2131  * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
2132  *
2133  * 24 bytes is for 3x SETUP packets
2134  * 16 bytes is a clock domain crossing tolerance
2135  *
2136  * Given RxFIFO Size, NUMP = RxFIFOSize / 1024;
2137  */
2138 static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
2139 {
2140         u32 ram2_depth;
2141         u32 mdwidth;
2142         u32 nump;
2143         u32 reg;
2144
2145         ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
2146         mdwidth = DWC3_GHWPARAMS0_MDWIDTH(dwc->hwparams.hwparams0);
2147         if (DWC3_IP_IS(DWC32))
2148                 mdwidth += DWC3_GHWPARAMS6_MDWIDTH(dwc->hwparams.hwparams6);
2149
2150         nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
2151         nump = min_t(u32, nump, 16);
2152
2153         /* update NumP */
2154         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2155         reg &= ~DWC3_DCFG_NUMP_MASK;
2156         reg |= nump << DWC3_DCFG_NUMP_SHIFT;
2157         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2158 }
2159
2160 static int __dwc3_gadget_start(struct dwc3 *dwc)
2161 {
2162         struct dwc3_ep          *dep;
2163         int                     ret = 0;
2164         u32                     reg;
2165
2166         /*
2167          * Use IMOD if enabled via dwc->imod_interval. Otherwise, if
2168          * the core supports IMOD, disable it.
2169          */
2170         if (dwc->imod_interval) {
2171                 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
2172                 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
2173         } else if (dwc3_has_imod(dwc)) {
2174                 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0);
2175         }
2176
2177         /*
2178          * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP
2179          * field instead of letting dwc3 itself calculate that automatically.
2180          *
2181          * This way, we maximize the chances that we'll be able to get several
2182          * bursts of data without going through any sort of endpoint throttling.
2183          */
2184         reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
2185         if (DWC3_IP_IS(DWC3))
2186                 reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
2187         else
2188                 reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
2189
2190         dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
2191
2192         dwc3_gadget_setup_nump(dwc);
2193
2194         /* Start with SuperSpeed Default */
2195         dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2196
2197         dep = dwc->eps[0];
2198         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2199         if (ret) {
2200                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2201                 goto err0;
2202         }
2203
2204         dep = dwc->eps[1];
2205         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2206         if (ret) {
2207                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2208                 goto err1;
2209         }
2210
2211         /* begin to receive SETUP packets */
2212         dwc->ep0state = EP0_SETUP_PHASE;
2213         dwc->link_state = DWC3_LINK_STATE_SS_DIS;
2214         dwc3_ep0_out_start(dwc);
2215
2216         dwc3_gadget_enable_irq(dwc);
2217
2218         return 0;
2219
2220 err1:
2221         __dwc3_gadget_ep_disable(dwc->eps[0]);
2222
2223 err0:
2224         return ret;
2225 }
2226
2227 static int dwc3_gadget_start(struct usb_gadget *g,
2228                 struct usb_gadget_driver *driver)
2229 {
2230         struct dwc3             *dwc = gadget_to_dwc(g);
2231         unsigned long           flags;
2232         int                     ret = 0;
2233         int                     irq;
2234
2235         irq = dwc->irq_gadget;
2236         ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
2237                         IRQF_SHARED, "dwc3", dwc->ev_buf);
2238         if (ret) {
2239                 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
2240                                 irq, ret);
2241                 goto err0;
2242         }
2243
2244         spin_lock_irqsave(&dwc->lock, flags);
2245         if (dwc->gadget_driver) {
2246                 dev_err(dwc->dev, "%s is already bound to %s\n",
2247                                 dwc->gadget.name,
2248                                 dwc->gadget_driver->driver.name);
2249                 ret = -EBUSY;
2250                 goto err1;
2251         }
2252
2253         dwc->gadget_driver      = driver;
2254
2255         if (pm_runtime_active(dwc->dev))
2256                 __dwc3_gadget_start(dwc);
2257
2258         spin_unlock_irqrestore(&dwc->lock, flags);
2259
2260         return 0;
2261
2262 err1:
2263         spin_unlock_irqrestore(&dwc->lock, flags);
2264         free_irq(irq, dwc);
2265
2266 err0:
2267         return ret;
2268 }
2269
2270 static void __dwc3_gadget_stop(struct dwc3 *dwc)
2271 {
2272         dwc3_gadget_disable_irq(dwc);
2273         __dwc3_gadget_ep_disable(dwc->eps[0]);
2274         __dwc3_gadget_ep_disable(dwc->eps[1]);
2275 }
2276
2277 static int dwc3_gadget_stop(struct usb_gadget *g)
2278 {
2279         struct dwc3             *dwc = gadget_to_dwc(g);
2280         unsigned long           flags;
2281
2282         spin_lock_irqsave(&dwc->lock, flags);
2283
2284         if (pm_runtime_suspended(dwc->dev))
2285                 goto out;
2286
2287         __dwc3_gadget_stop(dwc);
2288
2289 out:
2290         dwc->gadget_driver      = NULL;
2291         spin_unlock_irqrestore(&dwc->lock, flags);
2292
2293         free_irq(dwc->irq_gadget, dwc->ev_buf);
2294
2295         return 0;
2296 }
2297
2298 static void dwc3_gadget_config_params(struct usb_gadget *g,
2299                                       struct usb_dcd_config_params *params)
2300 {
2301         struct dwc3             *dwc = gadget_to_dwc(g);
2302
2303         params->besl_baseline = USB_DEFAULT_BESL_UNSPECIFIED;
2304         params->besl_deep = USB_DEFAULT_BESL_UNSPECIFIED;
2305
2306         /* Recommended BESL */
2307         if (!dwc->dis_enblslpm_quirk) {
2308                 /*
2309                  * If the recommended BESL baseline is 0 or if the BESL deep is
2310                  * less than 2, Microsoft's Windows 10 host usb stack will issue
2311                  * a usb reset immediately after it receives the extended BOS
2312                  * descriptor and the enumeration will fail. To maintain
2313                  * compatibility with the Windows' usb stack, let's set the
2314                  * recommended BESL baseline to 1 and clamp the BESL deep to be
2315                  * within 2 to 15.
2316                  */
2317                 params->besl_baseline = 1;
2318                 if (dwc->is_utmi_l1_suspend)
2319                         params->besl_deep =
2320                                 clamp_t(u8, dwc->hird_threshold, 2, 15);
2321         }
2322
2323         /* U1 Device exit Latency */
2324         if (dwc->dis_u1_entry_quirk)
2325                 params->bU1devExitLat = 0;
2326         else
2327                 params->bU1devExitLat = DWC3_DEFAULT_U1_DEV_EXIT_LAT;
2328
2329         /* U2 Device exit Latency */
2330         if (dwc->dis_u2_entry_quirk)
2331                 params->bU2DevExitLat = 0;
2332         else
2333                 params->bU2DevExitLat =
2334                                 cpu_to_le16(DWC3_DEFAULT_U2_DEV_EXIT_LAT);
2335 }
2336
2337 static void dwc3_gadget_set_speed(struct usb_gadget *g,
2338                                   enum usb_device_speed speed)
2339 {
2340         struct dwc3             *dwc = gadget_to_dwc(g);
2341         unsigned long           flags;
2342         u32                     reg;
2343
2344         spin_lock_irqsave(&dwc->lock, flags);
2345         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2346         reg &= ~(DWC3_DCFG_SPEED_MASK);
2347
2348         /*
2349          * WORKAROUND: DWC3 revision < 2.20a have an issue
2350          * which would cause metastability state on Run/Stop
2351          * bit if we try to force the IP to USB2-only mode.
2352          *
2353          * Because of that, we cannot configure the IP to any
2354          * speed other than the SuperSpeed
2355          *
2356          * Refers to:
2357          *
2358          * STAR#9000525659: Clock Domain Crossing on DCTL in
2359          * USB 2.0 Mode
2360          */
2361         if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
2362             !dwc->dis_metastability_quirk) {
2363                 reg |= DWC3_DCFG_SUPERSPEED;
2364         } else {
2365                 switch (speed) {
2366                 case USB_SPEED_LOW:
2367                         reg |= DWC3_DCFG_LOWSPEED;
2368                         break;
2369                 case USB_SPEED_FULL:
2370                         reg |= DWC3_DCFG_FULLSPEED;
2371                         break;
2372                 case USB_SPEED_HIGH:
2373                         reg |= DWC3_DCFG_HIGHSPEED;
2374                         break;
2375                 case USB_SPEED_SUPER:
2376                         reg |= DWC3_DCFG_SUPERSPEED;
2377                         break;
2378                 case USB_SPEED_SUPER_PLUS:
2379                         if (DWC3_IP_IS(DWC3))
2380                                 reg |= DWC3_DCFG_SUPERSPEED;
2381                         else
2382                                 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2383                         break;
2384                 default:
2385                         dev_err(dwc->dev, "invalid speed (%d)\n", speed);
2386
2387                         if (DWC3_IP_IS(DWC3))
2388                                 reg |= DWC3_DCFG_SUPERSPEED;
2389                         else
2390                                 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2391                 }
2392         }
2393         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2394
2395         spin_unlock_irqrestore(&dwc->lock, flags);
2396 }
2397
2398 static const struct usb_gadget_ops dwc3_gadget_ops = {
2399         .get_frame              = dwc3_gadget_get_frame,
2400         .wakeup                 = dwc3_gadget_wakeup,
2401         .set_selfpowered        = dwc3_gadget_set_selfpowered,
2402         .pullup                 = dwc3_gadget_pullup,
2403         .udc_start              = dwc3_gadget_start,
2404         .udc_stop               = dwc3_gadget_stop,
2405         .udc_set_speed          = dwc3_gadget_set_speed,
2406         .get_config_params      = dwc3_gadget_config_params,
2407 };
2408
2409 /* -------------------------------------------------------------------------- */
2410
2411 static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep)
2412 {
2413         struct dwc3 *dwc = dep->dwc;
2414
2415         usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
2416         dep->endpoint.maxburst = 1;
2417         dep->endpoint.ops = &dwc3_gadget_ep0_ops;
2418         if (!dep->direction)
2419                 dwc->gadget.ep0 = &dep->endpoint;
2420
2421         dep->endpoint.caps.type_control = true;
2422
2423         return 0;
2424 }
2425
2426 static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep)
2427 {
2428         struct dwc3 *dwc = dep->dwc;
2429         int mdwidth;
2430         int size;
2431
2432         mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
2433         if (DWC3_IP_IS(DWC32))
2434                 mdwidth += DWC3_GHWPARAMS6_MDWIDTH(dwc->hwparams.hwparams6);
2435
2436         /* MDWIDTH is represented in bits, we need it in bytes */
2437         mdwidth /= 8;
2438
2439         size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1));
2440         if (DWC3_IP_IS(DWC3))
2441                 size = DWC3_GTXFIFOSIZ_TXFDEP(size);
2442         else
2443                 size = DWC31_GTXFIFOSIZ_TXFDEP(size);
2444
2445         /* FIFO Depth is in MDWDITH bytes. Multiply */
2446         size *= mdwidth;
2447
2448         /*
2449          * To meet performance requirement, a minimum TxFIFO size of 3x
2450          * MaxPacketSize is recommended for endpoints that support burst and a
2451          * minimum TxFIFO size of 2x MaxPacketSize for endpoints that don't
2452          * support burst. Use those numbers and we can calculate the max packet
2453          * limit as below.
2454          */
2455         if (dwc->maximum_speed >= USB_SPEED_SUPER)
2456                 size /= 3;
2457         else
2458                 size /= 2;
2459
2460         usb_ep_set_maxpacket_limit(&dep->endpoint, size);
2461
2462         dep->endpoint.max_streams = 15;
2463         dep->endpoint.ops = &dwc3_gadget_ep_ops;
2464         list_add_tail(&dep->endpoint.ep_list,
2465                         &dwc->gadget.ep_list);
2466         dep->endpoint.caps.type_iso = true;
2467         dep->endpoint.caps.type_bulk = true;
2468         dep->endpoint.caps.type_int = true;
2469
2470         return dwc3_alloc_trb_pool(dep);
2471 }
2472
2473 static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep)
2474 {
2475         struct dwc3 *dwc = dep->dwc;
2476         int mdwidth;
2477         int size;
2478
2479         mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
2480         if (DWC3_IP_IS(DWC32))
2481                 mdwidth += DWC3_GHWPARAMS6_MDWIDTH(dwc->hwparams.hwparams6);
2482
2483         /* MDWIDTH is represented in bits, convert to bytes */
2484         mdwidth /= 8;
2485
2486         /* All OUT endpoints share a single RxFIFO space */
2487         size = dwc3_readl(dwc->regs, DWC3_GRXFIFOSIZ(0));
2488         if (DWC3_IP_IS(DWC3))
2489                 size = DWC3_GRXFIFOSIZ_RXFDEP(size);
2490         else
2491                 size = DWC31_GRXFIFOSIZ_RXFDEP(size);
2492
2493         /* FIFO depth is in MDWDITH bytes */
2494         size *= mdwidth;
2495
2496         /*
2497          * To meet performance requirement, a minimum recommended RxFIFO size
2498          * is defined as follow:
2499          * RxFIFO size >= (3 x MaxPacketSize) +
2500          * (3 x 8 bytes setup packets size) + (16 bytes clock crossing margin)
2501          *
2502          * Then calculate the max packet limit as below.
2503          */
2504         size -= (3 * 8) + 16;
2505         if (size < 0)
2506                 size = 0;
2507         else
2508                 size /= 3;
2509
2510         usb_ep_set_maxpacket_limit(&dep->endpoint, size);
2511         dep->endpoint.max_streams = 15;
2512         dep->endpoint.ops = &dwc3_gadget_ep_ops;
2513         list_add_tail(&dep->endpoint.ep_list,
2514                         &dwc->gadget.ep_list);
2515         dep->endpoint.caps.type_iso = true;
2516         dep->endpoint.caps.type_bulk = true;
2517         dep->endpoint.caps.type_int = true;
2518
2519         return dwc3_alloc_trb_pool(dep);
2520 }
2521
2522 static int dwc3_gadget_init_endpoint(struct dwc3 *dwc, u8 epnum)
2523 {
2524         struct dwc3_ep                  *dep;
2525         bool                            direction = epnum & 1;
2526         int                             ret;
2527         u8                              num = epnum >> 1;
2528
2529         dep = kzalloc(sizeof(*dep), GFP_KERNEL);
2530         if (!dep)
2531                 return -ENOMEM;
2532
2533         dep->dwc = dwc;
2534         dep->number = epnum;
2535         dep->direction = direction;
2536         dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
2537         dwc->eps[epnum] = dep;
2538         dep->combo_num = 0;
2539         dep->start_cmd_status = 0;
2540
2541         snprintf(dep->name, sizeof(dep->name), "ep%u%s", num,
2542                         direction ? "in" : "out");
2543
2544         dep->endpoint.name = dep->name;
2545
2546         if (!(dep->number > 1)) {
2547                 dep->endpoint.desc = &dwc3_gadget_ep0_desc;
2548                 dep->endpoint.comp_desc = NULL;
2549         }
2550
2551         if (num == 0)
2552                 ret = dwc3_gadget_init_control_endpoint(dep);
2553         else if (direction)
2554                 ret = dwc3_gadget_init_in_endpoint(dep);
2555         else
2556                 ret = dwc3_gadget_init_out_endpoint(dep);
2557
2558         if (ret)
2559                 return ret;
2560
2561         dep->endpoint.caps.dir_in = direction;
2562         dep->endpoint.caps.dir_out = !direction;
2563
2564         INIT_LIST_HEAD(&dep->pending_list);
2565         INIT_LIST_HEAD(&dep->started_list);
2566         INIT_LIST_HEAD(&dep->cancelled_list);
2567
2568         return 0;
2569 }
2570
2571 static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total)
2572 {
2573         u8                              epnum;
2574
2575         INIT_LIST_HEAD(&dwc->gadget.ep_list);
2576
2577         for (epnum = 0; epnum < total; epnum++) {
2578                 int                     ret;
2579
2580                 ret = dwc3_gadget_init_endpoint(dwc, epnum);
2581                 if (ret)
2582                         return ret;
2583         }
2584
2585         return 0;
2586 }
2587
2588 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
2589 {
2590         struct dwc3_ep                  *dep;
2591         u8                              epnum;
2592
2593         for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2594                 dep = dwc->eps[epnum];
2595                 if (!dep)
2596                         continue;
2597                 /*
2598                  * Physical endpoints 0 and 1 are special; they form the
2599                  * bi-directional USB endpoint 0.
2600                  *
2601                  * For those two physical endpoints, we don't allocate a TRB
2602                  * pool nor do we add them the endpoints list. Due to that, we
2603                  * shouldn't do these two operations otherwise we would end up
2604                  * with all sorts of bugs when removing dwc3.ko.
2605                  */
2606                 if (epnum != 0 && epnum != 1) {
2607                         dwc3_free_trb_pool(dep);
2608                         list_del(&dep->endpoint.ep_list);
2609                 }
2610
2611                 kfree(dep);
2612         }
2613 }
2614
2615 /* -------------------------------------------------------------------------- */
2616
2617 static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep,
2618                 struct dwc3_request *req, struct dwc3_trb *trb,
2619                 const struct dwc3_event_depevt *event, int status, int chain)
2620 {
2621         unsigned int            count;
2622
2623         dwc3_ep_inc_deq(dep);
2624
2625         trace_dwc3_complete_trb(dep, trb);
2626         req->num_trbs--;
2627
2628         /*
2629          * If we're in the middle of series of chained TRBs and we
2630          * receive a short transfer along the way, DWC3 will skip
2631          * through all TRBs including the last TRB in the chain (the
2632          * where CHN bit is zero. DWC3 will also avoid clearing HWO
2633          * bit and SW has to do it manually.
2634          *
2635          * We're going to do that here to avoid problems of HW trying
2636          * to use bogus TRBs for transfers.
2637          */
2638         if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO))
2639                 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2640
2641         /*
2642          * For isochronous transfers, the first TRB in a service interval must
2643          * have the Isoc-First type. Track and report its interval frame number.
2644          */
2645         if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
2646             (trb->ctrl & DWC3_TRBCTL_ISOCHRONOUS_FIRST)) {
2647                 unsigned int frame_number;
2648
2649                 frame_number = DWC3_TRB_CTRL_GET_SID_SOFN(trb->ctrl);
2650                 frame_number &= ~(dep->interval - 1);
2651                 req->request.frame_number = frame_number;
2652         }
2653
2654         /*
2655          * If we're dealing with unaligned size OUT transfer, we will be left
2656          * with one TRB pending in the ring. We need to manually clear HWO bit
2657          * from that TRB.
2658          */
2659
2660         if (req->needs_extra_trb && !(trb->ctrl & DWC3_TRB_CTRL_CHN)) {
2661                 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2662                 return 1;
2663         }
2664
2665         count = trb->size & DWC3_TRB_SIZE_MASK;
2666         req->remaining += count;
2667
2668         if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
2669                 return 1;
2670
2671         if (event->status & DEPEVT_STATUS_SHORT && !chain)
2672                 return 1;
2673
2674         if ((trb->ctrl & DWC3_TRB_CTRL_IOC) ||
2675             (trb->ctrl & DWC3_TRB_CTRL_LST))
2676                 return 1;
2677
2678         return 0;
2679 }
2680
2681 static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep,
2682                 struct dwc3_request *req, const struct dwc3_event_depevt *event,
2683                 int status)
2684 {
2685         struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
2686         struct scatterlist *sg = req->sg;
2687         struct scatterlist *s;
2688         unsigned int pending = req->num_pending_sgs;
2689         unsigned int i;
2690         int ret = 0;
2691
2692         for_each_sg(sg, s, pending, i) {
2693                 trb = &dep->trb_pool[dep->trb_dequeue];
2694
2695                 req->sg = sg_next(s);
2696                 req->num_pending_sgs--;
2697
2698                 ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req,
2699                                 trb, event, status, true);
2700                 if (ret)
2701                         break;
2702         }
2703
2704         return ret;
2705 }
2706
2707 static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep,
2708                 struct dwc3_request *req, const struct dwc3_event_depevt *event,
2709                 int status)
2710 {
2711         struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
2712
2713         return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb,
2714                         event, status, false);
2715 }
2716
2717 static bool dwc3_gadget_ep_request_completed(struct dwc3_request *req)
2718 {
2719         return req->num_pending_sgs == 0;
2720 }
2721
2722 static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep,
2723                 const struct dwc3_event_depevt *event,
2724                 struct dwc3_request *req, int status)
2725 {
2726         int ret;
2727
2728         if (req->num_pending_sgs)
2729                 ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event,
2730                                 status);
2731         else
2732                 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
2733                                 status);
2734
2735         if (req->needs_extra_trb) {
2736                 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
2737
2738                 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
2739                                 status);
2740
2741                 /* Reclaim MPS padding TRB for ZLP */
2742                 if (!req->direction && req->request.zero && req->request.length &&
2743                     !usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
2744                     (IS_ALIGNED(req->request.length, maxp)))
2745                         ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event, status);
2746
2747                 req->needs_extra_trb = false;
2748         }
2749
2750         req->request.actual = req->request.length - req->remaining;
2751
2752         if (!dwc3_gadget_ep_request_completed(req))
2753                 goto out;
2754
2755         dwc3_gadget_giveback(dep, req, status);
2756
2757 out:
2758         return ret;
2759 }
2760
2761 static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep,
2762                 const struct dwc3_event_depevt *event, int status)
2763 {
2764         struct dwc3_request     *req;
2765         struct dwc3_request     *tmp;
2766
2767         list_for_each_entry_safe(req, tmp, &dep->started_list, list) {
2768                 int ret;
2769
2770                 ret = dwc3_gadget_ep_cleanup_completed_request(dep, event,
2771                                 req, status);
2772                 if (ret)
2773                         break;
2774         }
2775 }
2776
2777 static bool dwc3_gadget_ep_should_continue(struct dwc3_ep *dep)
2778 {
2779         struct dwc3_request     *req;
2780
2781         if (!list_empty(&dep->pending_list))
2782                 return true;
2783
2784         /*
2785          * We only need to check the first entry of the started list. We can
2786          * assume the completed requests are removed from the started list.
2787          */
2788         req = next_request(&dep->started_list);
2789         if (!req)
2790                 return false;
2791
2792         return !dwc3_gadget_ep_request_completed(req);
2793 }
2794
2795 static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep,
2796                 const struct dwc3_event_depevt *event)
2797 {
2798         dep->frame_number = event->parameters;
2799 }
2800
2801 static bool dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep *dep,
2802                 const struct dwc3_event_depevt *event, int status)
2803 {
2804         struct dwc3             *dwc = dep->dwc;
2805         bool                    no_started_trb = true;
2806
2807         dwc3_gadget_ep_cleanup_completed_requests(dep, event, status);
2808
2809         if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
2810                 goto out;
2811
2812         if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
2813                 list_empty(&dep->started_list) &&
2814                 (list_empty(&dep->pending_list) || status == -EXDEV))
2815                 dwc3_stop_active_transfer(dep, true, true);
2816         else if (dwc3_gadget_ep_should_continue(dep))
2817                 if (__dwc3_gadget_kick_transfer(dep) == 0)
2818                         no_started_trb = false;
2819
2820 out:
2821         /*
2822          * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
2823          * See dwc3_gadget_linksts_change_interrupt() for 1st half.
2824          */
2825         if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
2826                 u32             reg;
2827                 int             i;
2828
2829                 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
2830                         dep = dwc->eps[i];
2831
2832                         if (!(dep->flags & DWC3_EP_ENABLED))
2833                                 continue;
2834
2835                         if (!list_empty(&dep->started_list))
2836                                 return no_started_trb;
2837                 }
2838
2839                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2840                 reg |= dwc->u1u2;
2841                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2842
2843                 dwc->u1u2 = 0;
2844         }
2845
2846         return no_started_trb;
2847 }
2848
2849 static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
2850                 const struct dwc3_event_depevt *event)
2851 {
2852         int status = 0;
2853
2854         if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
2855                 dwc3_gadget_endpoint_frame_from_event(dep, event);
2856
2857         if (event->status & DEPEVT_STATUS_BUSERR)
2858                 status = -ECONNRESET;
2859
2860         if (event->status & DEPEVT_STATUS_MISSED_ISOC)
2861                 status = -EXDEV;
2862
2863         dwc3_gadget_endpoint_trbs_complete(dep, event, status);
2864 }
2865
2866 static void dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep *dep,
2867                 const struct dwc3_event_depevt *event)
2868 {
2869         int status = 0;
2870
2871         dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
2872
2873         if (event->status & DEPEVT_STATUS_BUSERR)
2874                 status = -ECONNRESET;
2875
2876         if (dwc3_gadget_endpoint_trbs_complete(dep, event, status))
2877                 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
2878 }
2879
2880 static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep,
2881                 const struct dwc3_event_depevt *event)
2882 {
2883         dwc3_gadget_endpoint_frame_from_event(dep, event);
2884
2885         /*
2886          * The XferNotReady event is generated only once before the endpoint
2887          * starts. It will be generated again when END_TRANSFER command is
2888          * issued. For some controller versions, the XferNotReady event may be
2889          * generated while the END_TRANSFER command is still in process. Ignore
2890          * it and wait for the next XferNotReady event after the command is
2891          * completed.
2892          */
2893         if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
2894                 return;
2895
2896         (void) __dwc3_gadget_start_isoc(dep);
2897 }
2898
2899 static void dwc3_gadget_endpoint_stream_event(struct dwc3_ep *dep,
2900                 const struct dwc3_event_depevt *event)
2901 {
2902         struct dwc3 *dwc = dep->dwc;
2903
2904         if (event->status == DEPEVT_STREAMEVT_FOUND) {
2905                 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
2906                 goto out;
2907         }
2908
2909         /* Note: NoStream rejection event param value is 0 and not 0xFFFF */
2910         switch (event->parameters) {
2911         case DEPEVT_STREAM_PRIME:
2912                 /*
2913                  * If the host can properly transition the endpoint state from
2914                  * idle to prime after a NoStream rejection, there's no need to
2915                  * force restarting the endpoint to reinitiate the stream. To
2916                  * simplify the check, assume the host follows the USB spec if
2917                  * it primed the endpoint more than once.
2918                  */
2919                 if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) {
2920                         if (dep->flags & DWC3_EP_FIRST_STREAM_PRIMED)
2921                                 dep->flags &= ~DWC3_EP_FORCE_RESTART_STREAM;
2922                         else
2923                                 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
2924                 }
2925
2926                 break;
2927         case DEPEVT_STREAM_NOSTREAM:
2928                 if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) ||
2929                     !(dep->flags & DWC3_EP_FORCE_RESTART_STREAM) ||
2930                     !(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE))
2931                         break;
2932
2933                 /*
2934                  * If the host rejects a stream due to no active stream, by the
2935                  * USB and xHCI spec, the endpoint will be put back to idle
2936                  * state. When the host is ready (buffer added/updated), it will
2937                  * prime the endpoint to inform the usb device controller. This
2938                  * triggers the device controller to issue ERDY to restart the
2939                  * stream. However, some hosts don't follow this and keep the
2940                  * endpoint in the idle state. No prime will come despite host
2941                  * streams are updated, and the device controller will not be
2942                  * triggered to generate ERDY to move the next stream data. To
2943                  * workaround this and maintain compatibility with various
2944                  * hosts, force to reinitate the stream until the host is ready
2945                  * instead of waiting for the host to prime the endpoint.
2946                  */
2947                 if (DWC3_VER_IS_WITHIN(DWC32, 100A, ANY)) {
2948                         unsigned int cmd = DWC3_DGCMD_SET_ENDPOINT_PRIME;
2949
2950                         dwc3_send_gadget_generic_command(dwc, cmd, dep->number);
2951                 } else {
2952                         dep->flags |= DWC3_EP_DELAY_START;
2953                         dwc3_stop_active_transfer(dep, true, true);
2954                         return;
2955                 }
2956                 break;
2957         }
2958
2959 out:
2960         dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM;
2961 }
2962
2963 static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
2964                 const struct dwc3_event_depevt *event)
2965 {
2966         struct dwc3_ep          *dep;
2967         u8                      epnum = event->endpoint_number;
2968         u8                      cmd;
2969
2970         dep = dwc->eps[epnum];
2971
2972         if (!(dep->flags & DWC3_EP_ENABLED)) {
2973                 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED))
2974                         return;
2975
2976                 /* Handle only EPCMDCMPLT when EP disabled */
2977                 if (event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT)
2978                         return;
2979         }
2980
2981         if (epnum == 0 || epnum == 1) {
2982                 dwc3_ep0_interrupt(dwc, event);
2983                 return;
2984         }
2985
2986         switch (event->endpoint_event) {
2987         case DWC3_DEPEVT_XFERINPROGRESS:
2988                 dwc3_gadget_endpoint_transfer_in_progress(dep, event);
2989                 break;
2990         case DWC3_DEPEVT_XFERNOTREADY:
2991                 dwc3_gadget_endpoint_transfer_not_ready(dep, event);
2992                 break;
2993         case DWC3_DEPEVT_EPCMDCMPLT:
2994                 cmd = DEPEVT_PARAMETER_CMD(event->parameters);
2995
2996                 if (cmd == DWC3_DEPCMD_ENDTRANSFER) {
2997                         dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
2998                         dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
2999                         dwc3_gadget_ep_cleanup_cancelled_requests(dep);
3000                         if ((dep->flags & DWC3_EP_DELAY_START) &&
3001                             !usb_endpoint_xfer_isoc(dep->endpoint.desc))
3002                                 __dwc3_gadget_kick_transfer(dep);
3003
3004                         dep->flags &= ~DWC3_EP_DELAY_START;
3005                 }
3006                 break;
3007         case DWC3_DEPEVT_XFERCOMPLETE:
3008                 dwc3_gadget_endpoint_transfer_complete(dep, event);
3009                 break;
3010         case DWC3_DEPEVT_STREAMEVT:
3011                 dwc3_gadget_endpoint_stream_event(dep, event);
3012                 break;
3013         case DWC3_DEPEVT_RXTXFIFOEVT:
3014                 break;
3015         }
3016 }
3017
3018 static void dwc3_disconnect_gadget(struct dwc3 *dwc)
3019 {
3020         if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
3021                 spin_unlock(&dwc->lock);
3022                 dwc->gadget_driver->disconnect(&dwc->gadget);
3023                 spin_lock(&dwc->lock);
3024         }
3025 }
3026
3027 static void dwc3_suspend_gadget(struct dwc3 *dwc)
3028 {
3029         if (dwc->gadget_driver && dwc->gadget_driver->suspend) {
3030                 spin_unlock(&dwc->lock);
3031                 dwc->gadget_driver->suspend(&dwc->gadget);
3032                 spin_lock(&dwc->lock);
3033         }
3034 }
3035
3036 static void dwc3_resume_gadget(struct dwc3 *dwc)
3037 {
3038         if (dwc->gadget_driver && dwc->gadget_driver->resume) {
3039                 spin_unlock(&dwc->lock);
3040                 dwc->gadget_driver->resume(&dwc->gadget);
3041                 spin_lock(&dwc->lock);
3042         }
3043 }
3044
3045 static void dwc3_reset_gadget(struct dwc3 *dwc)
3046 {
3047         if (!dwc->gadget_driver)
3048                 return;
3049
3050         if (dwc->gadget.speed != USB_SPEED_UNKNOWN) {
3051                 spin_unlock(&dwc->lock);
3052                 usb_gadget_udc_reset(&dwc->gadget, dwc->gadget_driver);
3053                 spin_lock(&dwc->lock);
3054         }
3055 }
3056
3057 static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
3058         bool interrupt)
3059 {
3060         struct dwc3_gadget_ep_cmd_params params;
3061         u32 cmd;
3062         int ret;
3063
3064         if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) ||
3065             (dep->flags & DWC3_EP_END_TRANSFER_PENDING))
3066                 return;
3067
3068         /*
3069          * NOTICE: We are violating what the Databook says about the
3070          * EndTransfer command. Ideally we would _always_ wait for the
3071          * EndTransfer Command Completion IRQ, but that's causing too
3072          * much trouble synchronizing between us and gadget driver.
3073          *
3074          * We have discussed this with the IP Provider and it was
3075          * suggested to giveback all requests here.
3076          *
3077          * Note also that a similar handling was tested by Synopsys
3078          * (thanks a lot Paul) and nothing bad has come out of it.
3079          * In short, what we're doing is issuing EndTransfer with
3080          * CMDIOC bit set and delay kicking transfer until the
3081          * EndTransfer command had completed.
3082          *
3083          * As of IP version 3.10a of the DWC_usb3 IP, the controller
3084          * supports a mode to work around the above limitation. The
3085          * software can poll the CMDACT bit in the DEPCMD register
3086          * after issuing a EndTransfer command. This mode is enabled
3087          * by writing GUCTL2[14]. This polling is already done in the
3088          * dwc3_send_gadget_ep_cmd() function so if the mode is
3089          * enabled, the EndTransfer command will have completed upon
3090          * returning from this function.
3091          *
3092          * This mode is NOT available on the DWC_usb31 IP.
3093          */
3094
3095         cmd = DWC3_DEPCMD_ENDTRANSFER;
3096         cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
3097         cmd |= interrupt ? DWC3_DEPCMD_CMDIOC : 0;
3098         cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
3099         memset(&params, 0, sizeof(params));
3100         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
3101         WARN_ON_ONCE(ret);
3102         dep->resource_index = 0;
3103
3104         /*
3105          * The END_TRANSFER command will cause the controller to generate a
3106          * NoStream Event, and it's not due to the host DP NoStream rejection.
3107          * Ignore the next NoStream event.
3108          */
3109         if (dep->stream_capable)
3110                 dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM;
3111
3112         if (!interrupt)
3113                 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3114         else
3115                 dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
3116 }
3117
3118 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
3119 {
3120         u32 epnum;
3121
3122         for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
3123                 struct dwc3_ep *dep;
3124                 int ret;
3125
3126                 dep = dwc->eps[epnum];
3127                 if (!dep)
3128                         continue;
3129
3130                 if (!(dep->flags & DWC3_EP_STALL))
3131                         continue;
3132
3133                 dep->flags &= ~DWC3_EP_STALL;
3134
3135                 ret = dwc3_send_clear_stall_ep_cmd(dep);
3136                 WARN_ON_ONCE(ret);
3137         }
3138 }
3139
3140 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
3141 {
3142         int                     reg;
3143
3144         dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RX_DET);
3145
3146         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3147         reg &= ~DWC3_DCTL_INITU1ENA;
3148         reg &= ~DWC3_DCTL_INITU2ENA;
3149         dwc3_gadget_dctl_write_safe(dwc, reg);
3150
3151         dwc3_disconnect_gadget(dwc);
3152
3153         dwc->gadget.speed = USB_SPEED_UNKNOWN;
3154         dwc->setup_packet_pending = false;
3155         usb_gadget_set_state(&dwc->gadget, USB_STATE_NOTATTACHED);
3156
3157         dwc->connected = false;
3158 }
3159
3160 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
3161 {
3162         u32                     reg;
3163
3164         dwc->connected = true;
3165
3166         /*
3167          * WORKAROUND: DWC3 revisions <1.88a have an issue which
3168          * would cause a missing Disconnect Event if there's a
3169          * pending Setup Packet in the FIFO.
3170          *
3171          * There's no suggested workaround on the official Bug
3172          * report, which states that "unless the driver/application
3173          * is doing any special handling of a disconnect event,
3174          * there is no functional issue".
3175          *
3176          * Unfortunately, it turns out that we _do_ some special
3177          * handling of a disconnect event, namely complete all
3178          * pending transfers, notify gadget driver of the
3179          * disconnection, and so on.
3180          *
3181          * Our suggested workaround is to follow the Disconnect
3182          * Event steps here, instead, based on a setup_packet_pending
3183          * flag. Such flag gets set whenever we have a SETUP_PENDING
3184          * status for EP0 TRBs and gets cleared on XferComplete for the
3185          * same endpoint.
3186          *
3187          * Refers to:
3188          *
3189          * STAR#9000466709: RTL: Device : Disconnect event not
3190          * generated if setup packet pending in FIFO
3191          */
3192         if (DWC3_VER_IS_PRIOR(DWC3, 188A)) {
3193                 if (dwc->setup_packet_pending)
3194                         dwc3_gadget_disconnect_interrupt(dwc);
3195         }
3196
3197         dwc3_reset_gadget(dwc);
3198
3199         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3200         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
3201         dwc3_gadget_dctl_write_safe(dwc, reg);
3202         dwc->test_mode = false;
3203         dwc3_clear_stall_all_ep(dwc);
3204
3205         /* Reset device address to zero */
3206         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
3207         reg &= ~(DWC3_DCFG_DEVADDR_MASK);
3208         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
3209 }
3210
3211 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
3212 {
3213         struct dwc3_ep          *dep;
3214         int                     ret;
3215         u32                     reg;
3216         u8                      speed;
3217
3218         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
3219         speed = reg & DWC3_DSTS_CONNECTSPD;
3220         dwc->speed = speed;
3221
3222         /*
3223          * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
3224          * each time on Connect Done.
3225          *
3226          * Currently we always use the reset value. If any platform
3227          * wants to set this to a different value, we need to add a
3228          * setting and update GCTL.RAMCLKSEL here.
3229          */
3230
3231         switch (speed) {
3232         case DWC3_DSTS_SUPERSPEED_PLUS:
3233                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
3234                 dwc->gadget.ep0->maxpacket = 512;
3235                 dwc->gadget.speed = USB_SPEED_SUPER_PLUS;
3236                 break;
3237         case DWC3_DSTS_SUPERSPEED:
3238                 /*
3239                  * WORKAROUND: DWC3 revisions <1.90a have an issue which
3240                  * would cause a missing USB3 Reset event.
3241                  *
3242                  * In such situations, we should force a USB3 Reset
3243                  * event by calling our dwc3_gadget_reset_interrupt()
3244                  * routine.
3245                  *
3246                  * Refers to:
3247                  *
3248                  * STAR#9000483510: RTL: SS : USB3 reset event may
3249                  * not be generated always when the link enters poll
3250                  */
3251                 if (DWC3_VER_IS_PRIOR(DWC3, 190A))
3252                         dwc3_gadget_reset_interrupt(dwc);
3253
3254                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
3255                 dwc->gadget.ep0->maxpacket = 512;
3256                 dwc->gadget.speed = USB_SPEED_SUPER;
3257                 break;
3258         case DWC3_DSTS_HIGHSPEED:
3259                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
3260                 dwc->gadget.ep0->maxpacket = 64;
3261                 dwc->gadget.speed = USB_SPEED_HIGH;
3262                 break;
3263         case DWC3_DSTS_FULLSPEED:
3264                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
3265                 dwc->gadget.ep0->maxpacket = 64;
3266                 dwc->gadget.speed = USB_SPEED_FULL;
3267                 break;
3268         case DWC3_DSTS_LOWSPEED:
3269                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
3270                 dwc->gadget.ep0->maxpacket = 8;
3271                 dwc->gadget.speed = USB_SPEED_LOW;
3272                 break;
3273         }
3274
3275         dwc->eps[1]->endpoint.maxpacket = dwc->gadget.ep0->maxpacket;
3276
3277         /* Enable USB2 LPM Capability */
3278
3279         if (!DWC3_VER_IS_WITHIN(DWC3, ANY, 194A) &&
3280             (speed != DWC3_DSTS_SUPERSPEED) &&
3281             (speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
3282                 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
3283                 reg |= DWC3_DCFG_LPM_CAP;
3284                 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
3285
3286                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3287                 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
3288
3289                 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold |
3290                                             (dwc->is_utmi_l1_suspend << 4));
3291
3292                 /*
3293                  * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
3294                  * DCFG.LPMCap is set, core responses with an ACK and the
3295                  * BESL value in the LPM token is less than or equal to LPM
3296                  * NYET threshold.
3297                  */
3298                 WARN_ONCE(DWC3_VER_IS_PRIOR(DWC3, 240A) && dwc->has_lpm_erratum,
3299                                 "LPM Erratum not available on dwc3 revisions < 2.40a\n");
3300
3301                 if (dwc->has_lpm_erratum && !DWC3_VER_IS_PRIOR(DWC3, 240A))
3302                         reg |= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold);
3303
3304                 dwc3_gadget_dctl_write_safe(dwc, reg);
3305         } else {
3306                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3307                 reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
3308                 dwc3_gadget_dctl_write_safe(dwc, reg);
3309         }
3310
3311         dep = dwc->eps[0];
3312         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
3313         if (ret) {
3314                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
3315                 return;
3316         }
3317
3318         dep = dwc->eps[1];
3319         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
3320         if (ret) {
3321                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
3322                 return;
3323         }
3324
3325         /*
3326          * Configure PHY via GUSB3PIPECTLn if required.
3327          *
3328          * Update GTXFIFOSIZn
3329          *
3330          * In both cases reset values should be sufficient.
3331          */
3332 }
3333
3334 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
3335 {
3336         /*
3337          * TODO take core out of low power mode when that's
3338          * implemented.
3339          */
3340
3341         if (dwc->gadget_driver && dwc->gadget_driver->resume) {
3342                 spin_unlock(&dwc->lock);
3343                 dwc->gadget_driver->resume(&dwc->gadget);
3344                 spin_lock(&dwc->lock);
3345         }
3346 }
3347
3348 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
3349                 unsigned int evtinfo)
3350 {
3351         enum dwc3_link_state    next = evtinfo & DWC3_LINK_STATE_MASK;
3352         unsigned int            pwropt;
3353
3354         /*
3355          * WORKAROUND: DWC3 < 2.50a have an issue when configured without
3356          * Hibernation mode enabled which would show up when device detects
3357          * host-initiated U3 exit.
3358          *
3359          * In that case, device will generate a Link State Change Interrupt
3360          * from U3 to RESUME which is only necessary if Hibernation is
3361          * configured in.
3362          *
3363          * There are no functional changes due to such spurious event and we
3364          * just need to ignore it.
3365          *
3366          * Refers to:
3367          *
3368          * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
3369          * operational mode
3370          */
3371         pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
3372         if (DWC3_VER_IS_PRIOR(DWC3, 250A) &&
3373                         (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
3374                 if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
3375                                 (next == DWC3_LINK_STATE_RESUME)) {
3376                         return;
3377                 }
3378         }
3379
3380         /*
3381          * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
3382          * on the link partner, the USB session might do multiple entry/exit
3383          * of low power states before a transfer takes place.
3384          *
3385          * Due to this problem, we might experience lower throughput. The
3386          * suggested workaround is to disable DCTL[12:9] bits if we're
3387          * transitioning from U1/U2 to U0 and enable those bits again
3388          * after a transfer completes and there are no pending transfers
3389          * on any of the enabled endpoints.
3390          *
3391          * This is the first half of that workaround.
3392          *
3393          * Refers to:
3394          *
3395          * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
3396          * core send LGO_Ux entering U0
3397          */
3398         if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
3399                 if (next == DWC3_LINK_STATE_U0) {
3400                         u32     u1u2;
3401                         u32     reg;
3402
3403                         switch (dwc->link_state) {
3404                         case DWC3_LINK_STATE_U1:
3405                         case DWC3_LINK_STATE_U2:
3406                                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3407                                 u1u2 = reg & (DWC3_DCTL_INITU2ENA
3408                                                 | DWC3_DCTL_ACCEPTU2ENA
3409                                                 | DWC3_DCTL_INITU1ENA
3410                                                 | DWC3_DCTL_ACCEPTU1ENA);
3411
3412                                 if (!dwc->u1u2)
3413                                         dwc->u1u2 = reg & u1u2;
3414
3415                                 reg &= ~u1u2;
3416
3417                                 dwc3_gadget_dctl_write_safe(dwc, reg);
3418                                 break;
3419                         default:
3420                                 /* do nothing */
3421                                 break;
3422                         }
3423                 }
3424         }
3425
3426         switch (next) {
3427         case DWC3_LINK_STATE_U1:
3428                 if (dwc->speed == USB_SPEED_SUPER)
3429                         dwc3_suspend_gadget(dwc);
3430                 break;
3431         case DWC3_LINK_STATE_U2:
3432         case DWC3_LINK_STATE_U3:
3433                 dwc3_suspend_gadget(dwc);
3434                 break;
3435         case DWC3_LINK_STATE_RESUME:
3436                 dwc3_resume_gadget(dwc);
3437                 break;
3438         default:
3439                 /* do nothing */
3440                 break;
3441         }
3442
3443         dwc->link_state = next;
3444 }
3445
3446 static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc,
3447                                           unsigned int evtinfo)
3448 {
3449         enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
3450
3451         if (dwc->link_state != next && next == DWC3_LINK_STATE_U3)
3452                 dwc3_suspend_gadget(dwc);
3453
3454         dwc->link_state = next;
3455 }
3456
3457 static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc,
3458                 unsigned int evtinfo)
3459 {
3460         unsigned int is_ss = evtinfo & BIT(4);
3461
3462         /*
3463          * WORKAROUND: DWC3 revison 2.20a with hibernation support
3464          * have a known issue which can cause USB CV TD.9.23 to fail
3465          * randomly.
3466          *
3467          * Because of this issue, core could generate bogus hibernation
3468          * events which SW needs to ignore.
3469          *
3470          * Refers to:
3471          *
3472          * STAR#9000546576: Device Mode Hibernation: Issue in USB 2.0
3473          * Device Fallback from SuperSpeed
3474          */
3475         if (is_ss ^ (dwc->speed == USB_SPEED_SUPER))
3476                 return;
3477
3478         /* enter hibernation here */
3479 }
3480
3481 static void dwc3_gadget_interrupt(struct dwc3 *dwc,
3482                 const struct dwc3_event_devt *event)
3483 {
3484         switch (event->type) {
3485         case DWC3_DEVICE_EVENT_DISCONNECT:
3486                 dwc3_gadget_disconnect_interrupt(dwc);
3487                 break;
3488         case DWC3_DEVICE_EVENT_RESET:
3489                 dwc3_gadget_reset_interrupt(dwc);
3490                 break;
3491         case DWC3_DEVICE_EVENT_CONNECT_DONE:
3492                 dwc3_gadget_conndone_interrupt(dwc);
3493                 break;
3494         case DWC3_DEVICE_EVENT_WAKEUP:
3495                 dwc3_gadget_wakeup_interrupt(dwc);
3496                 break;
3497         case DWC3_DEVICE_EVENT_HIBER_REQ:
3498                 if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation,
3499                                         "unexpected hibernation event\n"))
3500                         break;
3501
3502                 dwc3_gadget_hibernation_interrupt(dwc, event->event_info);
3503                 break;
3504         case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
3505                 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
3506                 break;
3507         case DWC3_DEVICE_EVENT_EOPF:
3508                 /* It changed to be suspend event for version 2.30a and above */
3509                 if (!DWC3_VER_IS_PRIOR(DWC3, 230A)) {
3510                         /*
3511                          * Ignore suspend event until the gadget enters into
3512                          * USB_STATE_CONFIGURED state.
3513                          */
3514                         if (dwc->gadget.state >= USB_STATE_CONFIGURED)
3515                                 dwc3_gadget_suspend_interrupt(dwc,
3516                                                 event->event_info);
3517                 }
3518                 break;
3519         case DWC3_DEVICE_EVENT_SOF:
3520         case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
3521         case DWC3_DEVICE_EVENT_CMD_CMPL:
3522         case DWC3_DEVICE_EVENT_OVERFLOW:
3523                 break;
3524         default:
3525                 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
3526         }
3527 }
3528
3529 static void dwc3_process_event_entry(struct dwc3 *dwc,
3530                 const union dwc3_event *event)
3531 {
3532         trace_dwc3_event(event->raw, dwc);
3533
3534         if (!event->type.is_devspec)
3535                 dwc3_endpoint_interrupt(dwc, &event->depevt);
3536         else if (event->type.type == DWC3_EVENT_TYPE_DEV)
3537                 dwc3_gadget_interrupt(dwc, &event->devt);
3538         else
3539                 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
3540 }
3541
3542 static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
3543 {
3544         struct dwc3 *dwc = evt->dwc;
3545         irqreturn_t ret = IRQ_NONE;
3546         int left;
3547         u32 reg;
3548
3549         left = evt->count;
3550
3551         if (!(evt->flags & DWC3_EVENT_PENDING))
3552                 return IRQ_NONE;
3553
3554         while (left > 0) {
3555                 union dwc3_event event;
3556
3557                 event.raw = *(u32 *) (evt->cache + evt->lpos);
3558
3559                 dwc3_process_event_entry(dwc, &event);
3560
3561                 /*
3562                  * FIXME we wrap around correctly to the next entry as
3563                  * almost all entries are 4 bytes in size. There is one
3564                  * entry which has 12 bytes which is a regular entry
3565                  * followed by 8 bytes data. ATM I don't know how
3566                  * things are organized if we get next to the a
3567                  * boundary so I worry about that once we try to handle
3568                  * that.
3569                  */
3570                 evt->lpos = (evt->lpos + 4) % evt->length;
3571                 left -= 4;
3572         }
3573
3574         evt->count = 0;
3575         evt->flags &= ~DWC3_EVENT_PENDING;
3576         ret = IRQ_HANDLED;
3577
3578         /* Unmask interrupt */
3579         reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3580         reg &= ~DWC3_GEVNTSIZ_INTMASK;
3581         dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
3582
3583         if (dwc->imod_interval) {
3584                 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
3585                 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
3586         }
3587
3588         return ret;
3589 }
3590
3591 static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
3592 {
3593         struct dwc3_event_buffer *evt = _evt;
3594         struct dwc3 *dwc = evt->dwc;
3595         unsigned long flags;
3596         irqreturn_t ret = IRQ_NONE;
3597
3598         spin_lock_irqsave(&dwc->lock, flags);
3599         ret = dwc3_process_event_buf(evt);
3600         spin_unlock_irqrestore(&dwc->lock, flags);
3601
3602         return ret;
3603 }
3604
3605 static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
3606 {
3607         struct dwc3 *dwc = evt->dwc;
3608         u32 amount;
3609         u32 count;
3610         u32 reg;
3611
3612         if (pm_runtime_suspended(dwc->dev)) {
3613                 pm_runtime_get(dwc->dev);
3614                 disable_irq_nosync(dwc->irq_gadget);
3615                 dwc->pending_events = true;
3616                 return IRQ_HANDLED;
3617         }
3618
3619         /*
3620          * With PCIe legacy interrupt, test shows that top-half irq handler can
3621          * be called again after HW interrupt deassertion. Check if bottom-half
3622          * irq event handler completes before caching new event to prevent
3623          * losing events.
3624          */
3625         if (evt->flags & DWC3_EVENT_PENDING)
3626                 return IRQ_HANDLED;
3627
3628         count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
3629         count &= DWC3_GEVNTCOUNT_MASK;
3630         if (!count)
3631                 return IRQ_NONE;
3632
3633         evt->count = count;
3634         evt->flags |= DWC3_EVENT_PENDING;
3635
3636         /* Mask interrupt */
3637         reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3638         reg |= DWC3_GEVNTSIZ_INTMASK;
3639         dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
3640
3641         amount = min(count, evt->length - evt->lpos);
3642         memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);
3643
3644         if (amount < count)
3645                 memcpy(evt->cache, evt->buf, count - amount);
3646
3647         dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count);
3648
3649         return IRQ_WAKE_THREAD;
3650 }
3651
3652 static irqreturn_t dwc3_interrupt(int irq, void *_evt)
3653 {
3654         struct dwc3_event_buffer        *evt = _evt;
3655
3656         return dwc3_check_event_buf(evt);
3657 }
3658
3659 static int dwc3_gadget_get_irq(struct dwc3 *dwc)
3660 {
3661         struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
3662         int irq;
3663
3664         irq = platform_get_irq_byname_optional(dwc3_pdev, "peripheral");
3665         if (irq > 0)
3666                 goto out;
3667
3668         if (irq == -EPROBE_DEFER)
3669                 goto out;
3670
3671         irq = platform_get_irq_byname_optional(dwc3_pdev, "dwc_usb3");
3672         if (irq > 0)
3673                 goto out;
3674
3675         if (irq == -EPROBE_DEFER)
3676                 goto out;
3677
3678         irq = platform_get_irq(dwc3_pdev, 0);
3679         if (irq > 0)
3680                 goto out;
3681
3682         if (!irq)
3683                 irq = -EINVAL;
3684
3685 out:
3686         return irq;
3687 }
3688
3689 /**
3690  * dwc3_gadget_init - initializes gadget related registers
3691  * @dwc: pointer to our controller context structure
3692  *
3693  * Returns 0 on success otherwise negative errno.
3694  */
3695 int dwc3_gadget_init(struct dwc3 *dwc)
3696 {
3697         int ret;
3698         int irq;
3699
3700         irq = dwc3_gadget_get_irq(dwc);
3701         if (irq < 0) {
3702                 ret = irq;
3703                 goto err0;
3704         }
3705
3706         dwc->irq_gadget = irq;
3707
3708         dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev,
3709                                           sizeof(*dwc->ep0_trb) * 2,
3710                                           &dwc->ep0_trb_addr, GFP_KERNEL);
3711         if (!dwc->ep0_trb) {
3712                 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
3713                 ret = -ENOMEM;
3714                 goto err0;
3715         }
3716
3717         dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL);
3718         if (!dwc->setup_buf) {
3719                 ret = -ENOMEM;
3720                 goto err1;
3721         }
3722
3723         dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE,
3724                         &dwc->bounce_addr, GFP_KERNEL);
3725         if (!dwc->bounce) {
3726                 ret = -ENOMEM;
3727                 goto err2;
3728         }
3729
3730         init_completion(&dwc->ep0_in_setup);
3731
3732         dwc->gadget.ops                 = &dwc3_gadget_ops;
3733         dwc->gadget.speed               = USB_SPEED_UNKNOWN;
3734         dwc->gadget.sg_supported        = true;
3735         dwc->gadget.name                = "dwc3-gadget";
3736         dwc->gadget.lpm_capable         = true;
3737
3738         /*
3739          * FIXME We might be setting max_speed to <SUPER, however versions
3740          * <2.20a of dwc3 have an issue with metastability (documented
3741          * elsewhere in this driver) which tells us we can't set max speed to
3742          * anything lower than SUPER.
3743          *
3744          * Because gadget.max_speed is only used by composite.c and function
3745          * drivers (i.e. it won't go into dwc3's registers) we are allowing this
3746          * to happen so we avoid sending SuperSpeed Capability descriptor
3747          * together with our BOS descriptor as that could confuse host into
3748          * thinking we can handle super speed.
3749          *
3750          * Note that, in fact, we won't even support GetBOS requests when speed
3751          * is less than super speed because we don't have means, yet, to tell
3752          * composite.c that we are USB 2.0 + LPM ECN.
3753          */
3754         if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
3755             !dwc->dis_metastability_quirk)
3756                 dev_info(dwc->dev, "changing max_speed on rev %08x\n",
3757                                 dwc->revision);
3758
3759         dwc->gadget.max_speed           = dwc->maximum_speed;
3760
3761         /*
3762          * REVISIT: Here we should clear all pending IRQs to be
3763          * sure we're starting from a well known location.
3764          */
3765
3766         ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps);
3767         if (ret)
3768                 goto err3;
3769
3770         ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
3771         if (ret) {
3772                 dev_err(dwc->dev, "failed to register udc\n");
3773                 goto err4;
3774         }
3775
3776         dwc3_gadget_set_speed(&dwc->gadget, dwc->maximum_speed);
3777
3778         return 0;
3779
3780 err4:
3781         dwc3_gadget_free_endpoints(dwc);
3782
3783 err3:
3784         dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
3785                         dwc->bounce_addr);
3786
3787 err2:
3788         kfree(dwc->setup_buf);
3789
3790 err1:
3791         dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
3792                         dwc->ep0_trb, dwc->ep0_trb_addr);
3793
3794 err0:
3795         return ret;
3796 }
3797
3798 /* -------------------------------------------------------------------------- */
3799
3800 void dwc3_gadget_exit(struct dwc3 *dwc)
3801 {
3802         usb_del_gadget_udc(&dwc->gadget);
3803         dwc3_gadget_free_endpoints(dwc);
3804         dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
3805                           dwc->bounce_addr);
3806         kfree(dwc->setup_buf);
3807         dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
3808                           dwc->ep0_trb, dwc->ep0_trb_addr);
3809 }
3810
3811 int dwc3_gadget_suspend(struct dwc3 *dwc)
3812 {
3813         if (!dwc->gadget_driver)
3814                 return 0;
3815
3816         dwc3_gadget_run_stop(dwc, false, false);
3817         dwc3_disconnect_gadget(dwc);
3818         __dwc3_gadget_stop(dwc);
3819
3820         return 0;
3821 }
3822
3823 int dwc3_gadget_resume(struct dwc3 *dwc)
3824 {
3825         int                     ret;
3826
3827         if (!dwc->gadget_driver)
3828                 return 0;
3829
3830         ret = __dwc3_gadget_start(dwc);
3831         if (ret < 0)
3832                 goto err0;
3833
3834         ret = dwc3_gadget_run_stop(dwc, true, false);
3835         if (ret < 0)
3836                 goto err1;
3837
3838         return 0;
3839
3840 err1:
3841         __dwc3_gadget_stop(dwc);
3842
3843 err0:
3844         return ret;
3845 }
3846
3847 void dwc3_gadget_process_pending_events(struct dwc3 *dwc)
3848 {
3849         if (dwc->pending_events) {
3850                 dwc3_interrupt(dwc->irq_gadget, dwc->ev_buf);
3851                 dwc->pending_events = false;
3852                 enable_irq(dwc->irq_gadget);
3853         }
3854 }