drivers/usb/host/xhci-dbgcap.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * xhci-dbgcap.c - xHCI debug capability support
   4  *
   5  * Copyright (C) 2017 Intel Corporation
   6  *
   7  * Author: Lu Baolu <[email protected]>
   8  */
   9 #include <linux/bug.h>
  10 #include <linux/device.h>
  11 #include <linux/dma-mapping.h>
  12 #include <linux/errno.h>
  13 #include <linux/kstrtox.h>
  14 #include <linux/list.h>
  15 #include <linux/nls.h>
  16 #include <linux/pm_runtime.h>
  17 #include <linux/slab.h>
  18 #include <linux/spinlock.h>
  19 #include <linux/string.h>
  20 #include <linux/sysfs.h>
  21 #include <linux/types.h>
  22 #include <linux/workqueue.h>
  23
  24 #include <linux/io-64-nonatomic-lo-hi.h>
  25
  26 #include <asm/byteorder.h>
  27
  28 #include "xhci.h"
  29 #include "xhci-trace.h"
  30 #include "xhci-dbgcap.h"
  31
  32 static void dbc_free_ctx(struct device *dev, struct xhci_container_ctx *ctx)
  33 {
  34         if (!ctx)
  35                 return;
  36         dma_free_coherent(dev, ctx->size, ctx->bytes, ctx->dma);
  37         kfree(ctx);
  38 }
  39
  40 /* we use only one segment for DbC rings */
  41 static void dbc_ring_free(struct device *dev, struct xhci_ring *ring)
  42 {
  43         if (!ring)
  44                 return;
  45
  46         if (ring->first_seg) {
  47                 dma_free_coherent(dev, TRB_SEGMENT_SIZE,
  48                                   ring->first_seg->trbs,
  49                                   ring->first_seg->dma);
  50                 kfree(ring->first_seg);
  51         }
  52         kfree(ring);
  53 }
  54
  55 static u32 xhci_dbc_populate_strings(struct dbc_str_descs *strings)
  56 {
  57         struct usb_string_descriptor    *s_desc;
  58         u32                             string_length;
  59
  60         /* Serial string: */
  61         s_desc = (struct usb_string_descriptor *)strings->serial;
  62         utf8s_to_utf16s(DBC_STRING_SERIAL, strlen(DBC_STRING_SERIAL),
  63                         UTF16_LITTLE_ENDIAN, (wchar_t *)s_desc->wData,
  64                         DBC_MAX_STRING_LENGTH);
  65
  66         s_desc->bLength         = (strlen(DBC_STRING_SERIAL) + 1) * 2;
  67         s_desc->bDescriptorType = USB_DT_STRING;
  68         string_length           = s_desc->bLength;
  69         string_length           <<= 8;
  70
  71         /* Product string: */
  72         s_desc = (struct usb_string_descriptor *)strings->product;
  73         utf8s_to_utf16s(DBC_STRING_PRODUCT, strlen(DBC_STRING_PRODUCT),
  74                         UTF16_LITTLE_ENDIAN, (wchar_t *)s_desc->wData,
  75                         DBC_MAX_STRING_LENGTH);
  76
  77         s_desc->bLength         = (strlen(DBC_STRING_PRODUCT) + 1) * 2;
  78         s_desc->bDescriptorType = USB_DT_STRING;
  79         string_length           += s_desc->bLength;
  80         string_length           <<= 8;
  81
  82         /* Manufacture string: */
  83         s_desc = (struct usb_string_descriptor *)strings->manufacturer;
  84         utf8s_to_utf16s(DBC_STRING_MANUFACTURER,
  85                         strlen(DBC_STRING_MANUFACTURER),
  86                         UTF16_LITTLE_ENDIAN, (wchar_t *)s_desc->wData,
  87                         DBC_MAX_STRING_LENGTH);
  88
  89         s_desc->bLength         = (strlen(DBC_STRING_MANUFACTURER) + 1) * 2;
  90         s_desc->bDescriptorType = USB_DT_STRING;
  91         string_length           += s_desc->bLength;
  92         string_length           <<= 8;
  93
  94         /* String0: */
  95         strings->string0[0]     = 4;
  96         strings->string0[1]     = USB_DT_STRING;
  97         strings->string0[2]     = 0x09;
  98         strings->string0[3]     = 0x04;
  99         string_length           += 4;
 100
 101         return string_length;
 102 }
 103
 104 static void xhci_dbc_init_contexts(struct xhci_dbc *dbc, u32 string_length)
 105 {
 106         struct dbc_info_context *info;
 107         struct xhci_ep_ctx      *ep_ctx;
 108         u32                     dev_info;
 109         dma_addr_t              deq, dma;
 110         unsigned int            max_burst;
 111
 112         if (!dbc)
 113                 return;
 114
 115         /* Populate info Context: */
 116         info                    = (struct dbc_info_context *)dbc->ctx->bytes;
 117         dma                     = dbc->string_dma;
 118         info->string0           = cpu_to_le64(dma);
 119         info->manufacturer      = cpu_to_le64(dma + DBC_MAX_STRING_LENGTH);
 120         info->product           = cpu_to_le64(dma + DBC_MAX_STRING_LENGTH * 2);
 121         info->serial            = cpu_to_le64(dma + DBC_MAX_STRING_LENGTH * 3);
 122         info->length            = cpu_to_le32(string_length);
 123
 124         /* Populate bulk out endpoint context: */
 125         ep_ctx                  = dbc_bulkout_ctx(dbc);
 126         max_burst               = DBC_CTRL_MAXBURST(readl(&dbc->regs->control));
 127         deq                     = dbc_bulkout_enq(dbc);
 128         ep_ctx->ep_info         = 0;
 129         ep_ctx->ep_info2        = dbc_epctx_info2(BULK_OUT_EP, 1024, max_burst);
 130         ep_ctx->deq             = cpu_to_le64(deq | dbc->ring_out->cycle_state);
 131
 132         /* Populate bulk in endpoint context: */
 133         ep_ctx                  = dbc_bulkin_ctx(dbc);
 134         deq                     = dbc_bulkin_enq(dbc);
 135         ep_ctx->ep_info         = 0;
 136         ep_ctx->ep_info2        = dbc_epctx_info2(BULK_IN_EP, 1024, max_burst);
 137         ep_ctx->deq             = cpu_to_le64(deq | dbc->ring_in->cycle_state);
 138
 139         /* Set DbC context and info registers: */
 140         lo_hi_writeq(dbc->ctx->dma, &dbc->regs->dccp);
 141
 142         dev_info = (dbc->idVendor << 16) | dbc->bInterfaceProtocol;
 143         writel(dev_info, &dbc->regs->devinfo1);
 144
 145         dev_info = (dbc->bcdDevice << 16) | dbc->idProduct;
 146         writel(dev_info, &dbc->regs->devinfo2);
 147 }
 148
 149 static void xhci_dbc_giveback(struct dbc_request *req, int status)
 150         __releases(&dbc->lock)
 151         __acquires(&dbc->lock)
 152 {
 153         struct xhci_dbc         *dbc = req->dbc;
 154         struct device           *dev = dbc->dev;
 155
 156         list_del_init(&req->list_pending);
 157         req->trb_dma = 0;
 158         req->trb = NULL;
 159
 160         if (req->status == -EINPROGRESS)
 161                 req->status = status;
 162
 163         trace_xhci_dbc_giveback_request(req);
 164
 165         dma_unmap_single(dev,
 166                          req->dma,
 167                          req->length,
 168                          dbc_ep_dma_direction(req));
 169
 170         /* Give back the transfer request: */
 171         spin_unlock(&dbc->lock);
 172         req->complete(dbc, req);
 173         spin_lock(&dbc->lock);
 174 }
 175
 176 static void trb_to_noop(union xhci_trb *trb)
 177 {
 178         trb->generic.field[0]   = 0;
 179         trb->generic.field[1]   = 0;
 180         trb->generic.field[2]   = 0;
 181         trb->generic.field[3]   &= cpu_to_le32(TRB_CYCLE);
 182         trb->generic.field[3]   |= cpu_to_le32(TRB_TYPE(TRB_TR_NOOP));
 183 }
 184
 185 static void xhci_dbc_flush_single_request(struct dbc_request *req)
 186 {
 187         trb_to_noop(req->trb);
 188         xhci_dbc_giveback(req, -ESHUTDOWN);
 189 }
 190
 191 static void xhci_dbc_flush_endpoint_requests(struct dbc_ep *dep)
 192 {
 193         struct dbc_request      *req, *tmp;
 194
 195         list_for_each_entry_safe(req, tmp, &dep->list_pending, list_pending)
 196                 xhci_dbc_flush_single_request(req);
 197 }
 198
 199 static void xhci_dbc_flush_requests(struct xhci_dbc *dbc)
 200 {
 201         xhci_dbc_flush_endpoint_requests(&dbc->eps[BULK_OUT]);
 202         xhci_dbc_flush_endpoint_requests(&dbc->eps[BULK_IN]);
 203 }
 204
 205 struct dbc_request *
 206 dbc_alloc_request(struct xhci_dbc *dbc, unsigned int direction, gfp_t flags)
 207 {
 208         struct dbc_request      *req;
 209
 210         if (direction != BULK_IN &&
 211             direction != BULK_OUT)
 212                 return NULL;
 213
 214         if (!dbc)
 215                 return NULL;
 216
 217         req = kzalloc(sizeof(*req), flags);
 218         if (!req)
 219                 return NULL;
 220
 221         req->dbc = dbc;
 222         INIT_LIST_HEAD(&req->list_pending);
 223         INIT_LIST_HEAD(&req->list_pool);
 224         req->direction = direction;
 225
 226         trace_xhci_dbc_alloc_request(req);
 227
 228         return req;
 229 }
 230
 231 void
 232 dbc_free_request(struct dbc_request *req)
 233 {
 234         trace_xhci_dbc_free_request(req);
 235
 236         kfree(req);
 237 }
 238
 239 static void
 240 xhci_dbc_queue_trb(struct xhci_ring *ring, u32 field1,
 241                    u32 field2, u32 field3, u32 field4)
 242 {
 243         union xhci_trb          *trb, *next;
 244
 245         trb = ring->enqueue;
 246         trb->generic.field[0]   = cpu_to_le32(field1);
 247         trb->generic.field[1]   = cpu_to_le32(field2);
 248         trb->generic.field[2]   = cpu_to_le32(field3);
 249         trb->generic.field[3]   = cpu_to_le32(field4);
 250
 251         trace_xhci_dbc_gadget_ep_queue(ring, &trb->generic);
 252
 253         ring->num_trbs_free--;
 254         next = ++(ring->enqueue);
 255         if (TRB_TYPE_LINK_LE32(next->link.control)) {
 256                 next->link.control ^= cpu_to_le32(TRB_CYCLE);
 257                 ring->enqueue = ring->enq_seg->trbs;
 258                 ring->cycle_state ^= 1;
 259         }
 260 }
 261
 262 static int xhci_dbc_queue_bulk_tx(struct dbc_ep *dep,
 263                                   struct dbc_request *req)
 264 {
 265         u64                     addr;
 266         union xhci_trb          *trb;
 267         unsigned int            num_trbs;
 268         struct xhci_dbc         *dbc = req->dbc;
 269         struct xhci_ring        *ring = dep->ring;
 270         u32                     length, control, cycle;
 271
 272         num_trbs = count_trbs(req->dma, req->length);
 273         WARN_ON(num_trbs != 1);
 274         if (ring->num_trbs_free < num_trbs)
 275                 return -EBUSY;
 276
 277         addr    = req->dma;
 278         trb     = ring->enqueue;
 279         cycle   = ring->cycle_state;
 280         length  = TRB_LEN(req->length);
 281         control = TRB_TYPE(TRB_NORMAL) | TRB_IOC;
 282
 283         if (cycle)
 284                 control &= cpu_to_le32(~TRB_CYCLE);
 285         else
 286                 control |= cpu_to_le32(TRB_CYCLE);
 287
 288         req->trb = ring->enqueue;
 289         req->trb_dma = xhci_trb_virt_to_dma(ring->enq_seg, ring->enqueue);
 290         xhci_dbc_queue_trb(ring,
 291                            lower_32_bits(addr),
 292                            upper_32_bits(addr),
 293                            length, control);
 294
 295         /*
 296          * Add a barrier between writes of trb fields and flipping
 297          * the cycle bit:
 298          */
 299         wmb();
 300
 301         if (cycle)
 302                 trb->generic.field[3] |= cpu_to_le32(TRB_CYCLE);
 303         else
 304                 trb->generic.field[3] &= cpu_to_le32(~TRB_CYCLE);
 305
 306         writel(DBC_DOOR_BELL_TARGET(dep->direction), &dbc->regs->doorbell);
 307
 308         return 0;
 309 }
 310
 311 static int
 312 dbc_ep_do_queue(struct dbc_request *req)
 313 {
 314         int                     ret;
 315         struct xhci_dbc         *dbc = req->dbc;
 316         struct device           *dev = dbc->dev;
 317         struct dbc_ep           *dep = &dbc->eps[req->direction];
 318
 319         if (!req->length || !req->buf)
 320                 return -EINVAL;
 321
 322         req->actual             = 0;
 323         req->status             = -EINPROGRESS;
 324
 325         req->dma = dma_map_single(dev,
 326                                   req->buf,
 327                                   req->length,
 328                                   dbc_ep_dma_direction(dep));
 329         if (dma_mapping_error(dev, req->dma)) {
 330                 dev_err(dbc->dev, "failed to map buffer\n");
 331                 return -EFAULT;
 332         }
 333
 334         ret = xhci_dbc_queue_bulk_tx(dep, req);
 335         if (ret) {
 336                 dev_err(dbc->dev, "failed to queue trbs\n");
 337                 dma_unmap_single(dev,
 338                                  req->dma,
 339                                  req->length,
 340                                  dbc_ep_dma_direction(dep));
 341                 return -EFAULT;
 342         }
 343
 344         list_add_tail(&req->list_pending, &dep->list_pending);
 345
 346         return 0;
 347 }
 348
 349 int dbc_ep_queue(struct dbc_request *req)
 350 {
 351         unsigned long           flags;
 352         struct xhci_dbc         *dbc = req->dbc;
 353         int                     ret = -ESHUTDOWN;
 354
 355         if (!dbc)
 356                 return -ENODEV;
 357
 358         if (req->direction != BULK_IN &&
 359             req->direction != BULK_OUT)
 360                 return -EINVAL;
 361
 362         spin_lock_irqsave(&dbc->lock, flags);
 363         if (dbc->state == DS_CONFIGURED)
 364                 ret = dbc_ep_do_queue(req);
 365         spin_unlock_irqrestore(&dbc->lock, flags);
 366
 367         mod_delayed_work(system_wq, &dbc->event_work, 0);
 368
 369         trace_xhci_dbc_queue_request(req);
 370
 371         return ret;
 372 }
 373
 374 static inline void xhci_dbc_do_eps_init(struct xhci_dbc *dbc, bool direction)
 375 {
 376         struct dbc_ep           *dep;
 377
 378         dep                     = &dbc->eps[direction];
 379         dep->dbc                = dbc;
 380         dep->direction          = direction;
 381         dep->ring               = direction ? dbc->ring_in : dbc->ring_out;
 382
 383         INIT_LIST_HEAD(&dep->list_pending);
 384 }
 385
 386 static void xhci_dbc_eps_init(struct xhci_dbc *dbc)
 387 {
 388         xhci_dbc_do_eps_init(dbc, BULK_OUT);
 389         xhci_dbc_do_eps_init(dbc, BULK_IN);
 390 }
 391
 392 static void xhci_dbc_eps_exit(struct xhci_dbc *dbc)
 393 {
 394         memset(dbc->eps, 0, sizeof_field(struct xhci_dbc, eps));
 395 }
 396
 397 static int dbc_erst_alloc(struct device *dev, struct xhci_ring *evt_ring,
 398                     struct xhci_erst *erst, gfp_t flags)
 399 {
 400         erst->entries = dma_alloc_coherent(dev, sizeof(*erst->entries),
 401                                            &erst->erst_dma_addr, flags);
 402         if (!erst->entries)
 403                 return -ENOMEM;
 404
 405         erst->num_entries = 1;
 406         erst->entries[0].seg_addr = cpu_to_le64(evt_ring->first_seg->dma);
 407         erst->entries[0].seg_size = cpu_to_le32(TRBS_PER_SEGMENT);
 408         erst->entries[0].rsvd = 0;
 409         return 0;
 410 }
 411
 412 static void dbc_erst_free(struct device *dev, struct xhci_erst *erst)
 413 {
 414         dma_free_coherent(dev, sizeof(*erst->entries), erst->entries,
 415                           erst->erst_dma_addr);
 416         erst->entries = NULL;
 417 }
 418
 419 static struct xhci_container_ctx *
 420 dbc_alloc_ctx(struct device *dev, gfp_t flags)
 421 {
 422         struct xhci_container_ctx *ctx;
 423
 424         ctx = kzalloc(sizeof(*ctx), flags);
 425         if (!ctx)
 426                 return NULL;
 427
 428         /* xhci 7.6.9, all three contexts; info, ep-out and ep-in. Each 64 bytes*/
 429         ctx->size = 3 * DBC_CONTEXT_SIZE;
 430         ctx->bytes = dma_alloc_coherent(dev, ctx->size, &ctx->dma, flags);
 431         if (!ctx->bytes) {
 432                 kfree(ctx);
 433                 return NULL;
 434         }
 435         return ctx;
 436 }
 437
 438 static struct xhci_ring *
 439 xhci_dbc_ring_alloc(struct device *dev, enum xhci_ring_type type, gfp_t flags)
 440 {
 441         struct xhci_ring *ring;
 442         struct xhci_segment *seg;
 443         dma_addr_t dma;
 444
 445         ring = kzalloc(sizeof(*ring), flags);
 446         if (!ring)
 447                 return NULL;
 448
 449         ring->num_segs = 1;
 450         ring->type = type;
 451
 452         seg = kzalloc(sizeof(*seg), flags);
 453         if (!seg)
 454                 goto seg_fail;
 455
 456         ring->first_seg = seg;
 457         ring->last_seg = seg;
 458         seg->next = seg;
 459
 460         seg->trbs = dma_alloc_coherent(dev, TRB_SEGMENT_SIZE, &dma, flags);
 461         if (!seg->trbs)
 462                 goto dma_fail;
 463
 464         seg->dma = dma;
 465
 466         /* Only event ring does not use link TRB */
 467         if (type != TYPE_EVENT) {
 468                 union xhci_trb *trb = &seg->trbs[TRBS_PER_SEGMENT - 1];
 469
 470                 trb->link.segment_ptr = cpu_to_le64(dma);
 471                 trb->link.control = cpu_to_le32(LINK_TOGGLE | TRB_TYPE(TRB_LINK));
 472         }
 473         INIT_LIST_HEAD(&ring->td_list);
 474         xhci_initialize_ring_info(ring, 1);
 475         return ring;
 476 dma_fail:
 477         kfree(seg);
 478 seg_fail:
 479         kfree(ring);
 480         return NULL;
 481 }
 482
 483 static int xhci_dbc_mem_init(struct xhci_dbc *dbc, gfp_t flags)
 484 {
 485         int                     ret;
 486         dma_addr_t              deq;
 487         u32                     string_length;
 488         struct device           *dev = dbc->dev;
 489
 490         /* Allocate various rings for events and transfers: */
 491         dbc->ring_evt = xhci_dbc_ring_alloc(dev, TYPE_EVENT, flags);
 492         if (!dbc->ring_evt)
 493                 goto evt_fail;
 494
 495         dbc->ring_in = xhci_dbc_ring_alloc(dev, TYPE_BULK, flags);
 496         if (!dbc->ring_in)
 497                 goto in_fail;
 498
 499         dbc->ring_out = xhci_dbc_ring_alloc(dev, TYPE_BULK, flags);
 500         if (!dbc->ring_out)
 501                 goto out_fail;
 502
 503         /* Allocate and populate ERST: */
 504         ret = dbc_erst_alloc(dev, dbc->ring_evt, &dbc->erst, flags);
 505         if (ret)
 506                 goto erst_fail;
 507
 508         /* Allocate context data structure: */
 509         dbc->ctx = dbc_alloc_ctx(dev, flags); /* was sysdev, and is still */
 510         if (!dbc->ctx)
 511                 goto ctx_fail;
 512
 513         /* Allocate the string table: */
 514         dbc->string_size = sizeof(*dbc->string);
 515         dbc->string = dma_alloc_coherent(dev, dbc->string_size,
 516                                          &dbc->string_dma, flags);
 517         if (!dbc->string)
 518                 goto string_fail;
 519
 520         /* Setup ERST register: */
 521         writel(dbc->erst.num_entries, &dbc->regs->ersts);
 522
 523         lo_hi_writeq(dbc->erst.erst_dma_addr, &dbc->regs->erstba);
 524         deq = xhci_trb_virt_to_dma(dbc->ring_evt->deq_seg,
 525                                    dbc->ring_evt->dequeue);
 526         lo_hi_writeq(deq, &dbc->regs->erdp);
 527
 528         /* Setup strings and contexts: */
 529         string_length = xhci_dbc_populate_strings(dbc->string);
 530         xhci_dbc_init_contexts(dbc, string_length);
 531
 532         xhci_dbc_eps_init(dbc);
 533         dbc->state = DS_INITIALIZED;
 534
 535         return 0;
 536
 537 string_fail:
 538         dbc_free_ctx(dev, dbc->ctx);
 539         dbc->ctx = NULL;
 540 ctx_fail:
 541         dbc_erst_free(dev, &dbc->erst);
 542 erst_fail:
 543         dbc_ring_free(dev, dbc->ring_out);
 544         dbc->ring_out = NULL;
 545 out_fail:
 546         dbc_ring_free(dev, dbc->ring_in);
 547         dbc->ring_in = NULL;
 548 in_fail:
 549         dbc_ring_free(dev, dbc->ring_evt);
 550         dbc->ring_evt = NULL;
 551 evt_fail:
 552         return -ENOMEM;
 553 }
 554
 555 static void xhci_dbc_mem_cleanup(struct xhci_dbc *dbc)
 556 {
 557         if (!dbc)
 558                 return;
 559
 560         xhci_dbc_eps_exit(dbc);
 561
 562         dma_free_coherent(dbc->dev, dbc->string_size, dbc->string, dbc->string_dma);
 563         dbc->string = NULL;
 564
 565         dbc_free_ctx(dbc->dev, dbc->ctx);
 566         dbc->ctx = NULL;
 567
 568         dbc_erst_free(dbc->dev, &dbc->erst);
 569         dbc_ring_free(dbc->dev, dbc->ring_out);
 570         dbc_ring_free(dbc->dev, dbc->ring_in);
 571         dbc_ring_free(dbc->dev, dbc->ring_evt);
 572         dbc->ring_in = NULL;
 573         dbc->ring_out = NULL;
 574         dbc->ring_evt = NULL;
 575 }
 576
 577 static int xhci_do_dbc_start(struct xhci_dbc *dbc)
 578 {
 579         int                     ret;
 580         u32                     ctrl;
 581
 582         if (dbc->state != DS_DISABLED)
 583                 return -EINVAL;
 584
 585         writel(0, &dbc->regs->control);
 586         ret = xhci_handshake(&dbc->regs->control,
 587                              DBC_CTRL_DBC_ENABLE,
 588                              0, 1000);
 589         if (ret)
 590                 return ret;
 591
 592         ret = xhci_dbc_mem_init(dbc, GFP_ATOMIC);
 593         if (ret)
 594                 return ret;
 595
 596         ctrl = readl(&dbc->regs->control);
 597         writel(ctrl | DBC_CTRL_DBC_ENABLE | DBC_CTRL_PORT_ENABLE,
 598                &dbc->regs->control);
 599         ret = xhci_handshake(&dbc->regs->control,
 600                              DBC_CTRL_DBC_ENABLE,
 601                              DBC_CTRL_DBC_ENABLE, 1000);
 602         if (ret)
 603                 return ret;
 604
 605         dbc->state = DS_ENABLED;
 606
 607         return 0;
 608 }
 609
 610 static int xhci_do_dbc_stop(struct xhci_dbc *dbc)
 611 {
 612         if (dbc->state == DS_DISABLED)
 613                 return -EINVAL;
 614
 615         writel(0, &dbc->regs->control);
 616         dbc->state = DS_DISABLED;
 617
 618         return 0;
 619 }
 620
 621 static int xhci_dbc_start(struct xhci_dbc *dbc)
 622 {
 623         int                     ret;
 624         unsigned long           flags;
 625
 626         WARN_ON(!dbc);
 627
 628         pm_runtime_get_sync(dbc->dev); /* note this was self.controller */
 629
 630         spin_lock_irqsave(&dbc->lock, flags);
 631         ret = xhci_do_dbc_start(dbc);
 632         spin_unlock_irqrestore(&dbc->lock, flags);
 633
 634         if (ret) {
 635                 pm_runtime_put(dbc->dev); /* note this was self.controller */
 636                 return ret;
 637         }
 638
 639         return mod_delayed_work(system_wq, &dbc->event_work,
 640                                 msecs_to_jiffies(dbc->poll_interval));
 641 }
 642
 643 static void xhci_dbc_stop(struct xhci_dbc *dbc)
 644 {
 645         int ret;
 646         unsigned long           flags;
 647
 648         WARN_ON(!dbc);
 649
 650         switch (dbc->state) {
 651         case DS_DISABLED:
 652                 return;
 653         case DS_CONFIGURED:
 654                 if (dbc->driver->disconnect)
 655                         dbc->driver->disconnect(dbc);
 656                 break;
 657         default:
 658                 break;
 659         }
 660
 661         cancel_delayed_work_sync(&dbc->event_work);
 662
 663         spin_lock_irqsave(&dbc->lock, flags);
 664         ret = xhci_do_dbc_stop(dbc);
 665         spin_unlock_irqrestore(&dbc->lock, flags);
 666         if (ret)
 667                 return;
 668
 669         xhci_dbc_mem_cleanup(dbc);
 670         pm_runtime_put_sync(dbc->dev); /* note, was self.controller */
 671 }
 672
 673 static void
 674 handle_ep_halt_changes(struct xhci_dbc *dbc, struct dbc_ep *dep, bool halted)
 675 {
 676         if (halted) {
 677                 dev_info(dbc->dev, "DbC Endpoint halted\n");
 678                 dep->halted = 1;
 679
 680         } else if (dep->halted) {
 681                 dev_info(dbc->dev, "DbC Endpoint halt cleared\n");
 682                 dep->halted = 0;
 683
 684                 if (!list_empty(&dep->list_pending))
 685                         writel(DBC_DOOR_BELL_TARGET(dep->direction),
 686                                &dbc->regs->doorbell);
 687         }
 688 }
 689
 690 static void
 691 dbc_handle_port_status(struct xhci_dbc *dbc, union xhci_trb *event)
 692 {
 693         u32                     portsc;
 694
 695         portsc = readl(&dbc->regs->portsc);
 696         if (portsc & DBC_PORTSC_CONN_CHANGE)
 697                 dev_info(dbc->dev, "DbC port connect change\n");
 698
 699         if (portsc & DBC_PORTSC_RESET_CHANGE)
 700                 dev_info(dbc->dev, "DbC port reset change\n");
 701
 702         if (portsc & DBC_PORTSC_LINK_CHANGE)
 703                 dev_info(dbc->dev, "DbC port link status change\n");
 704
 705         if (portsc & DBC_PORTSC_CONFIG_CHANGE)
 706                 dev_info(dbc->dev, "DbC config error change\n");
 707
 708         /* Port reset change bit will be cleared in other place: */
 709         writel(portsc & ~DBC_PORTSC_RESET_CHANGE, &dbc->regs->portsc);
 710 }
 711
 712 static void dbc_handle_xfer_event(struct xhci_dbc *dbc, union xhci_trb *event)
 713 {
 714         struct dbc_ep           *dep;
 715         struct xhci_ring        *ring;
 716         int                     ep_id;
 717         int                     status;
 718         struct xhci_ep_ctx      *ep_ctx;
 719         u32                     comp_code;
 720         size_t                  remain_length;
 721         struct dbc_request      *req = NULL, *r;
 722
 723         comp_code       = GET_COMP_CODE(le32_to_cpu(event->generic.field[2]));
 724         remain_length   = EVENT_TRB_LEN(le32_to_cpu(event->generic.field[2]));
 725         ep_id           = TRB_TO_EP_ID(le32_to_cpu(event->generic.field[3]));
 726         dep             = (ep_id == EPID_OUT) ?
 727                                 get_out_ep(dbc) : get_in_ep(dbc);
 728         ep_ctx          = (ep_id == EPID_OUT) ?
 729                                 dbc_bulkout_ctx(dbc) : dbc_bulkin_ctx(dbc);
 730         ring            = dep->ring;
 731
 732         /* Match the pending request: */
 733         list_for_each_entry(r, &dep->list_pending, list_pending) {
 734                 if (r->trb_dma == event->trans_event.buffer) {
 735                         req = r;
 736                         break;
 737                 }
 738                 if (r->status == -COMP_STALL_ERROR) {
 739                         dev_warn(dbc->dev, "Give back stale stalled req\n");
 740                         ring->num_trbs_free++;
 741                         xhci_dbc_giveback(r, 0);
 742                 }
 743         }
 744
 745         if (!req) {
 746                 dev_warn(dbc->dev, "no matched request\n");
 747                 return;
 748         }
 749
 750         trace_xhci_dbc_handle_transfer(ring, &req->trb->generic);
 751
 752         switch (comp_code) {
 753         case COMP_SUCCESS:
 754                 remain_length = 0;
 755                 fallthrough;
 756         case COMP_SHORT_PACKET:
 757                 status = 0;
 758                 break;
 759         case COMP_TRB_ERROR:
 760         case COMP_BABBLE_DETECTED_ERROR:
 761         case COMP_USB_TRANSACTION_ERROR:
 762                 dev_warn(dbc->dev, "tx error %d detected\n", comp_code);
 763                 status = -comp_code;
 764                 break;
 765         case COMP_STALL_ERROR:
 766                 dev_warn(dbc->dev, "Stall error at bulk TRB %llx, remaining %zu, ep deq %llx\n",
 767                          event->trans_event.buffer, remain_length, ep_ctx->deq);
 768                 status = 0;
 769                 dep->halted = 1;
 770
 771                 /*
 772                  * xHC DbC may trigger a STALL bulk xfer event when host sends a
 773                  * ClearFeature(ENDPOINT_HALT) request even if there wasn't an
 774                  * active bulk transfer.
 775                  *
 776                  * Don't give back this transfer request as hardware will later
 777                  * start processing TRBs starting from this 'STALLED' TRB,
 778                  * causing TRBs and requests to be out of sync.
 779                  *
 780                  * If STALL event shows some bytes were transferred then assume
 781                  * it's an actual transfer issue and give back the request.
 782                  * In this case mark the TRB as No-Op to avoid hw from using the
 783                  * TRB again.
 784                  */
 785
 786                 if ((ep_ctx->deq & ~TRB_CYCLE) == event->trans_event.buffer) {
 787                         dev_dbg(dbc->dev, "Ep stopped on Stalled TRB\n");
 788                         if (remain_length == req->length) {
 789                                 dev_dbg(dbc->dev, "Spurious stall event, keep req\n");
 790                                 req->status = -COMP_STALL_ERROR;
 791                                 req->actual = 0;
 792                                 return;
 793                         }
 794                         dev_dbg(dbc->dev, "Give back stalled req, but turn TRB to No-op\n");
 795                         trb_to_noop(req->trb);
 796                 }
 797                 break;
 798
 799         default:
 800                 dev_err(dbc->dev, "unknown tx error %d\n", comp_code);
 801                 status = -comp_code;
 802                 break;
 803         }
 804
 805         ring->num_trbs_free++;
 806         req->actual = req->length - remain_length;
 807         xhci_dbc_giveback(req, status);
 808 }
 809
 810 static void inc_evt_deq(struct xhci_ring *ring)
 811 {
 812         /* If on the last TRB of the segment go back to the beginning */
 813         if (ring->dequeue == &ring->deq_seg->trbs[TRBS_PER_SEGMENT - 1]) {
 814                 ring->cycle_state ^= 1;
 815                 ring->dequeue = ring->deq_seg->trbs;
 816                 return;
 817         }
 818         ring->dequeue++;
 819 }
 820
 821 static enum evtreturn xhci_dbc_do_handle_events(struct xhci_dbc *dbc)
 822 {
 823         dma_addr_t              deq;
 824         union xhci_trb          *evt;
 825         u32                     ctrl, portsc;
 826         bool                    update_erdp = false;
 827
 828         /* DbC state machine: */
 829         switch (dbc->state) {
 830         case DS_DISABLED:
 831         case DS_INITIALIZED:
 832
 833                 return EVT_ERR;
 834         case DS_ENABLED:
 835                 portsc = readl(&dbc->regs->portsc);
 836                 if (portsc & DBC_PORTSC_CONN_STATUS) {
 837                         dbc->state = DS_CONNECTED;
 838                         dev_info(dbc->dev, "DbC connected\n");
 839                 }
 840
 841                 return EVT_DONE;
 842         case DS_CONNECTED:
 843                 ctrl = readl(&dbc->regs->control);
 844                 if (ctrl & DBC_CTRL_DBC_RUN) {
 845                         dbc->state = DS_CONFIGURED;
 846                         dev_info(dbc->dev, "DbC configured\n");
 847                         portsc = readl(&dbc->regs->portsc);
 848                         writel(portsc, &dbc->regs->portsc);
 849                         return EVT_GSER;
 850                 }
 851
 852                 return EVT_DONE;
 853         case DS_CONFIGURED:
 854                 /* Handle cable unplug event: */
 855                 portsc = readl(&dbc->regs->portsc);
 856                 if (!(portsc & DBC_PORTSC_PORT_ENABLED) &&
 857                     !(portsc & DBC_PORTSC_CONN_STATUS)) {
 858                         dev_info(dbc->dev, "DbC cable unplugged\n");
 859                         dbc->state = DS_ENABLED;
 860                         xhci_dbc_flush_requests(dbc);
 861
 862                         return EVT_DISC;
 863                 }
 864
 865                 /* Handle debug port reset event: */
 866                 if (portsc & DBC_PORTSC_RESET_CHANGE) {
 867                         dev_info(dbc->dev, "DbC port reset\n");
 868                         writel(portsc, &dbc->regs->portsc);
 869                         dbc->state = DS_ENABLED;
 870                         xhci_dbc_flush_requests(dbc);
 871
 872                         return EVT_DISC;
 873                 }
 874
 875                 /* Check and handle changes in endpoint halt status */
 876                 ctrl = readl(&dbc->regs->control);
 877                 handle_ep_halt_changes(dbc, get_in_ep(dbc), ctrl & DBC_CTRL_HALT_IN_TR);
 878                 handle_ep_halt_changes(dbc, get_out_ep(dbc), ctrl & DBC_CTRL_HALT_OUT_TR);
 879
 880                 /* Clear DbC run change bit: */
 881                 if (ctrl & DBC_CTRL_DBC_RUN_CHANGE) {
 882                         writel(ctrl, &dbc->regs->control);
 883                         ctrl = readl(&dbc->regs->control);
 884                 }
 885                 break;
 886         default:
 887                 dev_err(dbc->dev, "Unknown DbC state %d\n", dbc->state);
 888                 break;
 889         }
 890
 891         /* Handle the events in the event ring: */
 892         evt = dbc->ring_evt->dequeue;
 893         while ((le32_to_cpu(evt->event_cmd.flags) & TRB_CYCLE) ==
 894                         dbc->ring_evt->cycle_state) {
 895                 /*
 896                  * Add a barrier between reading the cycle flag and any
 897                  * reads of the event's flags/data below:
 898                  */
 899                 rmb();
 900
 901                 trace_xhci_dbc_handle_event(dbc->ring_evt, &evt->generic);
 902
 903                 switch (le32_to_cpu(evt->event_cmd.flags) & TRB_TYPE_BITMASK) {
 904                 case TRB_TYPE(TRB_PORT_STATUS):
 905                         dbc_handle_port_status(dbc, evt);
 906                         break;
 907                 case TRB_TYPE(TRB_TRANSFER):
 908                         dbc_handle_xfer_event(dbc, evt);
 909                         break;
 910                 default:
 911                         break;
 912                 }
 913
 914                 inc_evt_deq(dbc->ring_evt);
 915
 916                 evt = dbc->ring_evt->dequeue;
 917                 update_erdp = true;
 918         }
 919
 920         /* Update event ring dequeue pointer: */
 921         if (update_erdp) {
 922                 deq = xhci_trb_virt_to_dma(dbc->ring_evt->deq_seg,
 923                                            dbc->ring_evt->dequeue);
 924                 lo_hi_writeq(deq, &dbc->regs->erdp);
 925         }
 926
 927         return EVT_DONE;
 928 }
 929
 930 static void xhci_dbc_handle_events(struct work_struct *work)
 931 {
 932         enum evtreturn          evtr;
 933         struct xhci_dbc         *dbc;
 934         unsigned long           flags;
 935         unsigned int            poll_interval;
 936
 937         dbc = container_of(to_delayed_work(work), struct xhci_dbc, event_work);
 938         poll_interval = dbc->poll_interval;
 939
 940         spin_lock_irqsave(&dbc->lock, flags);
 941         evtr = xhci_dbc_do_handle_events(dbc);
 942         spin_unlock_irqrestore(&dbc->lock, flags);
 943
 944         switch (evtr) {
 945         case EVT_GSER:
 946                 if (dbc->driver->configure)
 947                         dbc->driver->configure(dbc);
 948                 break;
 949         case EVT_DISC:
 950                 if (dbc->driver->disconnect)
 951                         dbc->driver->disconnect(dbc);
 952                 break;
 953         case EVT_DONE:
 954                 /* set fast poll rate if there are pending data transfers */
 955                 if (!list_empty(&dbc->eps[BULK_OUT].list_pending) ||
 956                     !list_empty(&dbc->eps[BULK_IN].list_pending))
 957                         poll_interval = 1;
 958                 break;
 959         default:
 960                 dev_info(dbc->dev, "stop handling dbc events\n");
 961                 return;
 962         }
 963
 964         mod_delayed_work(system_wq, &dbc->event_work,
 965                          msecs_to_jiffies(poll_interval));
 966 }
 967
 968 static const char * const dbc_state_strings[DS_MAX] = {
 969         [DS_DISABLED] = "disabled",
 970         [DS_INITIALIZED] = "initialized",
 971         [DS_ENABLED] = "enabled",
 972         [DS_CONNECTED] = "connected",
 973         [DS_CONFIGURED] = "configured",
 974 };
 975
 976 static ssize_t dbc_show(struct device *dev,
 977                         struct device_attribute *attr,
 978                         char *buf)
 979 {
 980         struct xhci_dbc         *dbc;
 981         struct xhci_hcd         *xhci;
 982
 983         xhci = hcd_to_xhci(dev_get_drvdata(dev));
 984         dbc = xhci->dbc;
 985
 986         if (dbc->state >= ARRAY_SIZE(dbc_state_strings))
 987                 return sysfs_emit(buf, "unknown\n");
 988
 989         return sysfs_emit(buf, "%s\n", dbc_state_strings[dbc->state]);
 990 }
 991
 992 static ssize_t dbc_store(struct device *dev,
 993                          struct device_attribute *attr,
 994                          const char *buf, size_t count)
 995 {
 996         struct xhci_hcd         *xhci;
 997         struct xhci_dbc         *dbc;
 998
 999         xhci = hcd_to_xhci(dev_get_drvdata(dev));
1000         dbc = xhci->dbc;
1001
1002         if (sysfs_streq(buf, "enable"))
1003                 xhci_dbc_start(dbc);
1004         else if (sysfs_streq(buf, "disable"))
1005                 xhci_dbc_stop(dbc);
1006         else
1007                 return -EINVAL;
1008
1009         return count;
1010 }
1011
1012 static ssize_t dbc_idVendor_show(struct device *dev,
1013                             struct device_attribute *attr,
1014                             char *buf)
1015 {
1016         struct xhci_dbc         *dbc;
1017         struct xhci_hcd         *xhci;
1018
1019         xhci = hcd_to_xhci(dev_get_drvdata(dev));
1020         dbc = xhci->dbc;
1021
1022         return sysfs_emit(buf, "%04x\n", dbc->idVendor);
1023 }
1024
1025 static ssize_t dbc_idVendor_store(struct device *dev,
1026                              struct device_attribute *attr,
1027                              const char *buf, size_t size)
1028 {
1029         struct xhci_dbc         *dbc;
1030         struct xhci_hcd         *xhci;
1031         void __iomem            *ptr;
1032         u16                     value;
1033         u32                     dev_info;
1034         int ret;
1035
1036         ret = kstrtou16(buf, 0, &value);
1037         if (ret)
1038                 return ret;
1039
1040         xhci = hcd_to_xhci(dev_get_drvdata(dev));
1041         dbc = xhci->dbc;
1042         if (dbc->state != DS_DISABLED)
1043                 return -EBUSY;
1044
1045         dbc->idVendor = value;
1046         ptr = &dbc->regs->devinfo1;
1047         dev_info = readl(ptr);
1048         dev_info = (dev_info & ~(0xffffu << 16)) | (value << 16);
1049         writel(dev_info, ptr);
1050
1051         return size;
1052 }
1053
1054 static ssize_t dbc_idProduct_show(struct device *dev,
1055                             struct device_attribute *attr,
1056                             char *buf)
1057 {
1058         struct xhci_dbc         *dbc;
1059         struct xhci_hcd         *xhci;
1060
1061         xhci = hcd_to_xhci(dev_get_drvdata(dev));
1062         dbc = xhci->dbc;
1063
1064         return sysfs_emit(buf, "%04x\n", dbc->idProduct);
1065 }
1066
1067 static ssize_t dbc_idProduct_store(struct device *dev,
1068                              struct device_attribute *attr,
1069                              const char *buf, size_t size)
1070 {
1071         struct xhci_dbc         *dbc;
1072         struct xhci_hcd         *xhci;
1073         void __iomem            *ptr;
1074         u32                     dev_info;
1075         u16                     value;
1076         int ret;
1077
1078         ret = kstrtou16(buf, 0, &value);
1079         if (ret)
1080                 return ret;
1081
1082         xhci = hcd_to_xhci(dev_get_drvdata(dev));
1083         dbc = xhci->dbc;
1084         if (dbc->state != DS_DISABLED)
1085                 return -EBUSY;
1086
1087         dbc->idProduct = value;
1088         ptr = &dbc->regs->devinfo2;
1089         dev_info = readl(ptr);
1090         dev_info = (dev_info & ~(0xffffu)) | value;
1091         writel(dev_info, ptr);
1092         return size;
1093 }
1094
1095 static ssize_t dbc_bcdDevice_show(struct device *dev,
1096                                    struct device_attribute *attr,
1097                                    char *buf)
1098 {
1099         struct xhci_dbc *dbc;
1100         struct xhci_hcd *xhci;
1101
1102         xhci = hcd_to_xhci(dev_get_drvdata(dev));
1103         dbc = xhci->dbc;
1104
1105         return sysfs_emit(buf, "%04x\n", dbc->bcdDevice);
1106 }
1107
1108 static ssize_t dbc_bcdDevice_store(struct device *dev,
1109                                     struct device_attribute *attr,
1110                                     const char *buf, size_t size)
1111 {
1112         struct xhci_dbc *dbc;
1113         struct xhci_hcd *xhci;
1114         void __iomem *ptr;
1115         u32 dev_info;
1116         u16 value;
1117         int ret;
1118
1119         ret = kstrtou16(buf, 0, &value);
1120         if (ret)
1121                 return ret;
1122
1123         xhci = hcd_to_xhci(dev_get_drvdata(dev));
1124         dbc = xhci->dbc;
1125         if (dbc->state != DS_DISABLED)
1126                 return -EBUSY;
1127
1128         dbc->bcdDevice = value;
1129         ptr = &dbc->regs->devinfo2;
1130         dev_info = readl(ptr);
1131         dev_info = (dev_info & ~(0xffffu << 16)) | (value << 16);
1132         writel(dev_info, ptr);
1133
1134         return size;
1135 }
1136
1137 static ssize_t dbc_bInterfaceProtocol_show(struct device *dev,
1138                                  struct device_attribute *attr,
1139                                  char *buf)
1140 {
1141         struct xhci_dbc *dbc;
1142         struct xhci_hcd *xhci;
1143
1144         xhci = hcd_to_xhci(dev_get_drvdata(dev));
1145         dbc = xhci->dbc;
1146
1147         return sysfs_emit(buf, "%02x\n", dbc->bInterfaceProtocol);
1148 }
1149
1150 static ssize_t dbc_bInterfaceProtocol_store(struct device *dev,
1151                                   struct device_attribute *attr,
1152                                   const char *buf, size_t size)
1153 {
1154         struct xhci_dbc *dbc;
1155         struct xhci_hcd *xhci;
1156         void __iomem *ptr;
1157         u32 dev_info;
1158         u8 value;
1159         int ret;
1160
1161         /* bInterfaceProtocol is 8 bit, but... */
1162         ret = kstrtou8(buf, 0, &value);
1163         if (ret)
1164                 return ret;
1165
1166         /* ...xhci only supports values 0 and 1 */
1167         if (value > 1)
1168                 return -EINVAL;
1169
1170         xhci = hcd_to_xhci(dev_get_drvdata(dev));
1171         dbc = xhci->dbc;
1172         if (dbc->state != DS_DISABLED)
1173                 return -EBUSY;
1174
1175         dbc->bInterfaceProtocol = value;
1176         ptr = &dbc->regs->devinfo1;
1177         dev_info = readl(ptr);
1178         dev_info = (dev_info & ~(0xffu)) | value;
1179         writel(dev_info, ptr);
1180
1181         return size;
1182 }
1183
1184 static ssize_t dbc_poll_interval_ms_show(struct device *dev,
1185                                          struct device_attribute *attr,
1186                                          char *buf)
1187 {
1188         struct xhci_dbc *dbc;
1189         struct xhci_hcd *xhci;
1190
1191         xhci = hcd_to_xhci(dev_get_drvdata(dev));
1192         dbc = xhci->dbc;
1193
1194         return sysfs_emit(buf, "%u\n", dbc->poll_interval);
1195 }
1196
1197 static ssize_t dbc_poll_interval_ms_store(struct device *dev,
1198                                           struct device_attribute *attr,
1199                                           const char *buf, size_t size)
1200 {
1201         struct xhci_dbc *dbc;
1202         struct xhci_hcd *xhci;
1203         u32 value;
1204         int ret;
1205
1206         ret = kstrtou32(buf, 0, &value);
1207         if (ret || value > DBC_POLL_INTERVAL_MAX)
1208                 return -EINVAL;
1209
1210         xhci = hcd_to_xhci(dev_get_drvdata(dev));
1211         dbc = xhci->dbc;
1212
1213         dbc->poll_interval = value;
1214
1215         mod_delayed_work(system_wq, &dbc->event_work, 0);
1216
1217         return size;
1218 }
1219
1220 static DEVICE_ATTR_RW(dbc);
1221 static DEVICE_ATTR_RW(dbc_idVendor);
1222 static DEVICE_ATTR_RW(dbc_idProduct);
1223 static DEVICE_ATTR_RW(dbc_bcdDevice);
1224 static DEVICE_ATTR_RW(dbc_bInterfaceProtocol);
1225 static DEVICE_ATTR_RW(dbc_poll_interval_ms);
1226
1227 static struct attribute *dbc_dev_attrs[] = {
1228         &dev_attr_dbc.attr,
1229         &dev_attr_dbc_idVendor.attr,
1230         &dev_attr_dbc_idProduct.attr,
1231         &dev_attr_dbc_bcdDevice.attr,
1232         &dev_attr_dbc_bInterfaceProtocol.attr,
1233         &dev_attr_dbc_poll_interval_ms.attr,
1234         NULL
1235 };
1236 ATTRIBUTE_GROUPS(dbc_dev);
1237
1238 struct xhci_dbc *
1239 xhci_alloc_dbc(struct device *dev, void __iomem *base, const struct dbc_driver *driver)
1240 {
1241         struct xhci_dbc         *dbc;
1242         int                     ret;
1243
1244         dbc = kzalloc(sizeof(*dbc), GFP_KERNEL);
1245         if (!dbc)
1246                 return NULL;
1247
1248         dbc->regs = base;
1249         dbc->dev = dev;
1250         dbc->driver = driver;
1251         dbc->idProduct = DBC_PRODUCT_ID;
1252         dbc->idVendor = DBC_VENDOR_ID;
1253         dbc->bcdDevice = DBC_DEVICE_REV;
1254         dbc->bInterfaceProtocol = DBC_PROTOCOL;
1255         dbc->poll_interval = DBC_POLL_INTERVAL_DEFAULT;
1256
1257         if (readl(&dbc->regs->control) & DBC_CTRL_DBC_ENABLE)
1258                 goto err;
1259
1260         INIT_DELAYED_WORK(&dbc->event_work, xhci_dbc_handle_events);
1261         spin_lock_init(&dbc->lock);
1262
1263         ret = sysfs_create_groups(&dev->kobj, dbc_dev_groups);
1264         if (ret)
1265                 goto err;
1266
1267         return dbc;
1268 err:
1269         kfree(dbc);
1270         return NULL;
1271 }
1272
1273 /* undo what xhci_alloc_dbc() did */
1274 void xhci_dbc_remove(struct xhci_dbc *dbc)
1275 {
1276         if (!dbc)
1277                 return;
1278         /* stop hw, stop wq and call dbc->ops->stop() */
1279         xhci_dbc_stop(dbc);
1280
1281         /* remove sysfs files */
1282         sysfs_remove_groups(&dbc->dev->kobj, dbc_dev_groups);
1283
1284         kfree(dbc);
1285 }
1286
1287
1288 int xhci_create_dbc_dev(struct xhci_hcd *xhci)
1289 {
1290         struct device           *dev;
1291         void __iomem            *base;
1292         int                     ret;
1293         int                     dbc_cap_offs;
1294
1295         /* create all parameters needed resembling a dbc device */
1296         dev = xhci_to_hcd(xhci)->self.controller;
1297         base = &xhci->cap_regs->hc_capbase;
1298
1299         dbc_cap_offs = xhci_find_next_ext_cap(base, 0, XHCI_EXT_CAPS_DEBUG);
1300         if (!dbc_cap_offs)
1301                 return -ENODEV;
1302
1303         /* already allocated and in use */
1304         if (xhci->dbc)
1305                 return -EBUSY;
1306
1307         ret = xhci_dbc_tty_probe(dev, base + dbc_cap_offs, xhci);
1308
1309         return ret;
1310 }
1311
1312 void xhci_remove_dbc_dev(struct xhci_hcd *xhci)
1313 {
1314         unsigned long           flags;
1315
1316         if (!xhci->dbc)
1317                 return;
1318
1319         xhci_dbc_tty_remove(xhci->dbc);
1320         spin_lock_irqsave(&xhci->lock, flags);
1321         xhci->dbc = NULL;
1322         spin_unlock_irqrestore(&xhci->lock, flags);
1323 }
1324
1325 #ifdef CONFIG_PM
1326 int xhci_dbc_suspend(struct xhci_hcd *xhci)
1327 {
1328         struct xhci_dbc         *dbc = xhci->dbc;
1329
1330         if (!dbc)
1331                 return 0;
1332
1333         if (dbc->state == DS_CONFIGURED)
1334                 dbc->resume_required = 1;
1335
1336         xhci_dbc_stop(dbc);
1337
1338         return 0;
1339 }
1340
1341 int xhci_dbc_resume(struct xhci_hcd *xhci)
1342 {
1343         int                     ret = 0;
1344         struct xhci_dbc         *dbc = xhci->dbc;
1345
1346         if (!dbc)
1347                 return 0;
1348
1349         if (dbc->resume_required) {
1350                 dbc->resume_required = 0;
1351                 xhci_dbc_start(dbc);
1352         }
1353
1354         return ret;
1355 }
1356 #endif /* CONFIG_PM */
1357
1358 int xhci_dbc_init(void)
1359 {
1360         return dbc_tty_init();
1361 }
1362
1363 void xhci_dbc_exit(void)
1364 {
1365         dbc_tty_exit();
1366 }