drivers/dma/dw/core.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Core driver for the Synopsys DesignWare DMA Controller
   4  *
   5  * Copyright (C) 2007-2008 Atmel Corporation
   6  * Copyright (C) 2010-2011 ST Microelectronics
   7  * Copyright (C) 2013 Intel Corporation
   8  */
   9
  10 #include <linux/bitops.h>
  11 #include <linux/delay.h>
  12 #include <linux/dmaengine.h>
  13 #include <linux/dma-mapping.h>
  14 #include <linux/dmapool.h>
  15 #include <linux/err.h>
  16 #include <linux/init.h>
  17 #include <linux/interrupt.h>
  18 #include <linux/io.h>
  19 #include <linux/mm.h>
  20 #include <linux/module.h>
  21 #include <linux/slab.h>
  22 #include <linux/pm_runtime.h>
  23
  24 #include "../dmaengine.h"
  25 #include "internal.h"
  26
  27 /*
  28  * This supports the Synopsys "DesignWare AHB Central DMA Controller",
  29  * (DW_ahb_dmac) which is used with various AMBA 2.0 systems (not all
  30  * of which use ARM any more).  See the "Databook" from Synopsys for
  31  * information beyond what licensees probably provide.
  32  *
  33  * The driver has been tested with the Atmel AT32AP7000, which does not
  34  * support descriptor writeback.
  35  */
  36
  37 /* The set of bus widths supported by the DMA controller */
  38 #define DW_DMA_BUSWIDTHS                          \
  39         BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED)       | \
  40         BIT(DMA_SLAVE_BUSWIDTH_1_BYTE)          | \
  41         BIT(DMA_SLAVE_BUSWIDTH_2_BYTES)         | \
  42         BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)
  43
  44 /*----------------------------------------------------------------------*/
  45
  46 static struct device *chan2dev(struct dma_chan *chan)
  47 {
  48         return &chan->dev->device;
  49 }
  50
  51 static struct dw_desc *dwc_first_active(struct dw_dma_chan *dwc)
  52 {
  53         return to_dw_desc(dwc->active_list.next);
  54 }
  55
  56 static dma_cookie_t dwc_tx_submit(struct dma_async_tx_descriptor *tx)
  57 {
  58         struct dw_desc          *desc = txd_to_dw_desc(tx);
  59         struct dw_dma_chan      *dwc = to_dw_dma_chan(tx->chan);
  60         dma_cookie_t            cookie;
  61         unsigned long           flags;
  62
  63         spin_lock_irqsave(&dwc->lock, flags);
  64         cookie = dma_cookie_assign(tx);
  65
  66         /*
  67          * REVISIT: We should attempt to chain as many descriptors as
  68          * possible, perhaps even appending to those already submitted
  69          * for DMA. But this is hard to do in a race-free manner.
  70          */
  71
  72         list_add_tail(&desc->desc_node, &dwc->queue);
  73         spin_unlock_irqrestore(&dwc->lock, flags);
  74         dev_vdbg(chan2dev(tx->chan), "%s: queued %u\n",
  75                  __func__, desc->txd.cookie);
  76
  77         return cookie;
  78 }
  79
  80 static struct dw_desc *dwc_desc_get(struct dw_dma_chan *dwc)
  81 {
  82         struct dw_dma *dw = to_dw_dma(dwc->chan.device);
  83         struct dw_desc *desc;
  84         dma_addr_t phys;
  85
  86         desc = dma_pool_zalloc(dw->desc_pool, GFP_ATOMIC, &phys);
  87         if (!desc)
  88                 return NULL;
  89
  90         dwc->descs_allocated++;
  91         INIT_LIST_HEAD(&desc->tx_list);
  92         dma_async_tx_descriptor_init(&desc->txd, &dwc->chan);
  93         desc->txd.tx_submit = dwc_tx_submit;
  94         desc->txd.flags = DMA_CTRL_ACK;
  95         desc->txd.phys = phys;
  96         return desc;
  97 }
  98
  99 static void dwc_desc_put(struct dw_dma_chan *dwc, struct dw_desc *desc)
 100 {
 101         struct dw_dma *dw = to_dw_dma(dwc->chan.device);
 102         struct dw_desc *child, *_next;
 103
 104         if (unlikely(!desc))
 105                 return;
 106
 107         list_for_each_entry_safe(child, _next, &desc->tx_list, desc_node) {
 108                 list_del(&child->desc_node);
 109                 dma_pool_free(dw->desc_pool, child, child->txd.phys);
 110                 dwc->descs_allocated--;
 111         }
 112
 113         dma_pool_free(dw->desc_pool, desc, desc->txd.phys);
 114         dwc->descs_allocated--;
 115 }
 116
 117 static void dwc_initialize(struct dw_dma_chan *dwc)
 118 {
 119         struct dw_dma *dw = to_dw_dma(dwc->chan.device);
 120
 121         if (test_bit(DW_DMA_IS_INITIALIZED, &dwc->flags))
 122                 return;
 123
 124         dw->initialize_chan(dwc);
 125
 126         /* Enable interrupts */
 127         channel_set_bit(dw, MASK.XFER, dwc->mask);
 128         channel_set_bit(dw, MASK.ERROR, dwc->mask);
 129
 130         set_bit(DW_DMA_IS_INITIALIZED, &dwc->flags);
 131 }
 132
 133 /*----------------------------------------------------------------------*/
 134
 135 static inline void dwc_dump_chan_regs(struct dw_dma_chan *dwc)
 136 {
 137         dev_err(chan2dev(&dwc->chan),
 138                 "  SAR: 0x%x DAR: 0x%x LLP: 0x%x CTL: 0x%x:%08x\n",
 139                 channel_readl(dwc, SAR),
 140                 channel_readl(dwc, DAR),
 141                 channel_readl(dwc, LLP),
 142                 channel_readl(dwc, CTL_HI),
 143                 channel_readl(dwc, CTL_LO));
 144 }
 145
 146 static inline void dwc_chan_disable(struct dw_dma *dw, struct dw_dma_chan *dwc)
 147 {
 148         channel_clear_bit(dw, CH_EN, dwc->mask);
 149         while (dma_readl(dw, CH_EN) & dwc->mask)
 150                 cpu_relax();
 151 }
 152
 153 /*----------------------------------------------------------------------*/
 154
 155 /* Perform single block transfer */
 156 static inline void dwc_do_single_block(struct dw_dma_chan *dwc,
 157                                        struct dw_desc *desc)
 158 {
 159         struct dw_dma   *dw = to_dw_dma(dwc->chan.device);
 160         u32             ctllo;
 161
 162         /*
 163          * Software emulation of LLP mode relies on interrupts to continue
 164          * multi block transfer.
 165          */
 166         ctllo = lli_read(desc, ctllo) | DWC_CTLL_INT_EN;
 167
 168         channel_writel(dwc, SAR, lli_read(desc, sar));
 169         channel_writel(dwc, DAR, lli_read(desc, dar));
 170         channel_writel(dwc, CTL_LO, ctllo);
 171         channel_writel(dwc, CTL_HI, lli_read(desc, ctlhi));
 172         channel_set_bit(dw, CH_EN, dwc->mask);
 173
 174         /* Move pointer to next descriptor */
 175         dwc->tx_node_active = dwc->tx_node_active->next;
 176 }
 177
 178 /* Called with dwc->lock held and bh disabled */
 179 static void dwc_dostart(struct dw_dma_chan *dwc, struct dw_desc *first)
 180 {
 181         struct dw_dma   *dw = to_dw_dma(dwc->chan.device);
 182         u8              lms = DWC_LLP_LMS(dwc->dws.m_master);
 183         unsigned long   was_soft_llp;
 184
 185         /* ASSERT:  channel is idle */
 186         if (dma_readl(dw, CH_EN) & dwc->mask) {
 187                 dev_err(chan2dev(&dwc->chan),
 188                         "%s: BUG: Attempted to start non-idle channel\n",
 189                         __func__);
 190                 dwc_dump_chan_regs(dwc);
 191
 192                 /* The tasklet will hopefully advance the queue... */
 193                 return;
 194         }
 195
 196         if (dwc->nollp) {
 197                 was_soft_llp = test_and_set_bit(DW_DMA_IS_SOFT_LLP,
 198                                                 &dwc->flags);
 199                 if (was_soft_llp) {
 200                         dev_err(chan2dev(&dwc->chan),
 201                                 "BUG: Attempted to start new LLP transfer inside ongoing one\n");
 202                         return;
 203                 }
 204
 205                 dwc_initialize(dwc);
 206
 207                 first->residue = first->total_len;
 208                 dwc->tx_node_active = &first->tx_list;
 209
 210                 /* Submit first block */
 211                 dwc_do_single_block(dwc, first);
 212
 213                 return;
 214         }
 215
 216         dwc_initialize(dwc);
 217
 218         channel_writel(dwc, LLP, first->txd.phys | lms);
 219         channel_writel(dwc, CTL_LO, DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);
 220         channel_writel(dwc, CTL_HI, 0);
 221         channel_set_bit(dw, CH_EN, dwc->mask);
 222 }
 223
 224 static void dwc_dostart_first_queued(struct dw_dma_chan *dwc)
 225 {
 226         struct dw_desc *desc;
 227
 228         if (list_empty(&dwc->queue))
 229                 return;
 230
 231         list_move(dwc->queue.next, &dwc->active_list);
 232         desc = dwc_first_active(dwc);
 233         dev_vdbg(chan2dev(&dwc->chan), "%s: started %u\n", __func__, desc->txd.cookie);
 234         dwc_dostart(dwc, desc);
 235 }
 236
 237 /*----------------------------------------------------------------------*/
 238
 239 static void
 240 dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc,
 241                 bool callback_required)
 242 {
 243         struct dma_async_tx_descriptor  *txd = &desc->txd;
 244         struct dw_desc                  *child;
 245         unsigned long                   flags;
 246         struct dmaengine_desc_callback  cb;
 247
 248         dev_vdbg(chan2dev(&dwc->chan), "descriptor %u complete\n", txd->cookie);
 249
 250         spin_lock_irqsave(&dwc->lock, flags);
 251         dma_cookie_complete(txd);
 252         if (callback_required)
 253                 dmaengine_desc_get_callback(txd, &cb);
 254         else
 255                 memset(&cb, 0, sizeof(cb));
 256
 257         /* async_tx_ack */
 258         list_for_each_entry(child, &desc->tx_list, desc_node)
 259                 async_tx_ack(&child->txd);
 260         async_tx_ack(&desc->txd);
 261         dwc_desc_put(dwc, desc);
 262         spin_unlock_irqrestore(&dwc->lock, flags);
 263
 264         dmaengine_desc_callback_invoke(&cb, NULL);
 265 }
 266
 267 static void dwc_complete_all(struct dw_dma *dw, struct dw_dma_chan *dwc)
 268 {
 269         struct dw_desc *desc, *_desc;
 270         LIST_HEAD(list);
 271         unsigned long flags;
 272
 273         spin_lock_irqsave(&dwc->lock, flags);
 274         if (dma_readl(dw, CH_EN) & dwc->mask) {
 275                 dev_err(chan2dev(&dwc->chan),
 276                         "BUG: XFER bit set, but channel not idle!\n");
 277
 278                 /* Try to continue after resetting the channel... */
 279                 dwc_chan_disable(dw, dwc);
 280         }
 281
 282         /*
 283          * Submit queued descriptors ASAP, i.e. before we go through
 284          * the completed ones.
 285          */
 286         list_splice_init(&dwc->active_list, &list);
 287         dwc_dostart_first_queued(dwc);
 288
 289         spin_unlock_irqrestore(&dwc->lock, flags);
 290
 291         list_for_each_entry_safe(desc, _desc, &list, desc_node)
 292                 dwc_descriptor_complete(dwc, desc, true);
 293 }
 294
 295 /* Returns how many bytes were already received from source */
 296 static inline u32 dwc_get_sent(struct dw_dma_chan *dwc)
 297 {
 298         struct dw_dma *dw = to_dw_dma(dwc->chan.device);
 299         u32 ctlhi = channel_readl(dwc, CTL_HI);
 300         u32 ctllo = channel_readl(dwc, CTL_LO);
 301
 302         return dw->block2bytes(dwc, ctlhi, ctllo >> 4 & 7);
 303 }
 304
 305 static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)
 306 {
 307         dma_addr_t llp;
 308         struct dw_desc *desc, *_desc;
 309         struct dw_desc *child;
 310         u32 status_xfer;
 311         unsigned long flags;
 312
 313         spin_lock_irqsave(&dwc->lock, flags);
 314         llp = channel_readl(dwc, LLP);
 315         status_xfer = dma_readl(dw, RAW.XFER);
 316
 317         if (status_xfer & dwc->mask) {
 318                 /* Everything we've submitted is done */
 319                 dma_writel(dw, CLEAR.XFER, dwc->mask);
 320
 321                 if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags)) {
 322                         struct list_head *head, *active = dwc->tx_node_active;
 323
 324                         /*
 325                          * We are inside first active descriptor.
 326                          * Otherwise something is really wrong.
 327                          */
 328                         desc = dwc_first_active(dwc);
 329
 330                         head = &desc->tx_list;
 331                         if (active != head) {
 332                                 /* Update residue to reflect last sent descriptor */
 333                                 if (active == head->next)
 334                                         desc->residue -= desc->len;
 335                                 else
 336                                         desc->residue -= to_dw_desc(active->prev)->len;
 337
 338                                 child = to_dw_desc(active);
 339
 340                                 /* Submit next block */
 341                                 dwc_do_single_block(dwc, child);
 342
 343                                 spin_unlock_irqrestore(&dwc->lock, flags);
 344                                 return;
 345                         }
 346
 347                         /* We are done here */
 348                         clear_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags);
 349                 }
 350
 351                 spin_unlock_irqrestore(&dwc->lock, flags);
 352
 353                 dwc_complete_all(dw, dwc);
 354                 return;
 355         }
 356
 357         if (list_empty(&dwc->active_list)) {
 358                 spin_unlock_irqrestore(&dwc->lock, flags);
 359                 return;
 360         }
 361
 362         if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags)) {
 363                 dev_vdbg(chan2dev(&dwc->chan), "%s: soft LLP mode\n", __func__);
 364                 spin_unlock_irqrestore(&dwc->lock, flags);
 365                 return;
 366         }
 367
 368         dev_vdbg(chan2dev(&dwc->chan), "%s: llp=%pad\n", __func__, &llp);
 369
 370         list_for_each_entry_safe(desc, _desc, &dwc->active_list, desc_node) {
 371                 /* Initial residue value */
 372                 desc->residue = desc->total_len;
 373
 374                 /* Check first descriptors addr */
 375                 if (desc->txd.phys == DWC_LLP_LOC(llp)) {
 376                         spin_unlock_irqrestore(&dwc->lock, flags);
 377                         return;
 378                 }
 379
 380                 /* Check first descriptors llp */
 381                 if (lli_read(desc, llp) == llp) {
 382                         /* This one is currently in progress */
 383                         desc->residue -= dwc_get_sent(dwc);
 384                         spin_unlock_irqrestore(&dwc->lock, flags);
 385                         return;
 386                 }
 387
 388                 desc->residue -= desc->len;
 389                 list_for_each_entry(child, &desc->tx_list, desc_node) {
 390                         if (lli_read(child, llp) == llp) {
 391                                 /* Currently in progress */
 392                                 desc->residue -= dwc_get_sent(dwc);
 393                                 spin_unlock_irqrestore(&dwc->lock, flags);
 394                                 return;
 395                         }
 396                         desc->residue -= child->len;
 397                 }
 398
 399                 /*
 400                  * No descriptors so far seem to be in progress, i.e.
 401                  * this one must be done.
 402                  */
 403                 spin_unlock_irqrestore(&dwc->lock, flags);
 404                 dwc_descriptor_complete(dwc, desc, true);
 405                 spin_lock_irqsave(&dwc->lock, flags);
 406         }
 407
 408         dev_err(chan2dev(&dwc->chan),
 409                 "BUG: All descriptors done, but channel not idle!\n");
 410
 411         /* Try to continue after resetting the channel... */
 412         dwc_chan_disable(dw, dwc);
 413
 414         dwc_dostart_first_queued(dwc);
 415         spin_unlock_irqrestore(&dwc->lock, flags);
 416 }
 417
 418 static inline void dwc_dump_lli(struct dw_dma_chan *dwc, struct dw_desc *desc)
 419 {
 420         dev_crit(chan2dev(&dwc->chan), "  desc: s0x%x d0x%x l0x%x c0x%x:%x\n",
 421                  lli_read(desc, sar),
 422                  lli_read(desc, dar),
 423                  lli_read(desc, llp),
 424                  lli_read(desc, ctlhi),
 425                  lli_read(desc, ctllo));
 426 }
 427
 428 static void dwc_handle_error(struct dw_dma *dw, struct dw_dma_chan *dwc)
 429 {
 430         struct dw_desc *bad_desc;
 431         struct dw_desc *child;
 432         unsigned long flags;
 433
 434         dwc_scan_descriptors(dw, dwc);
 435
 436         spin_lock_irqsave(&dwc->lock, flags);
 437
 438         /*
 439          * The descriptor currently at the head of the active list is
 440          * borked. Since we don't have any way to report errors, we'll
 441          * just have to scream loudly and try to carry on.
 442          */
 443         bad_desc = dwc_first_active(dwc);
 444         list_del_init(&bad_desc->desc_node);
 445         list_move(dwc->queue.next, dwc->active_list.prev);
 446
 447         /* Clear the error flag and try to restart the controller */
 448         dma_writel(dw, CLEAR.ERROR, dwc->mask);
 449         if (!list_empty(&dwc->active_list))
 450                 dwc_dostart(dwc, dwc_first_active(dwc));
 451
 452         /*
 453          * WARN may seem harsh, but since this only happens
 454          * when someone submits a bad physical address in a
 455          * descriptor, we should consider ourselves lucky that the
 456          * controller flagged an error instead of scribbling over
 457          * random memory locations.
 458          */
 459         dev_WARN(chan2dev(&dwc->chan), "Bad descriptor submitted for DMA!\n"
 460                                        "  cookie: %d\n", bad_desc->txd.cookie);
 461         dwc_dump_lli(dwc, bad_desc);
 462         list_for_each_entry(child, &bad_desc->tx_list, desc_node)
 463                 dwc_dump_lli(dwc, child);
 464
 465         spin_unlock_irqrestore(&dwc->lock, flags);
 466
 467         /* Pretend the descriptor completed successfully */
 468         dwc_descriptor_complete(dwc, bad_desc, true);
 469 }
 470
 471 static void dw_dma_tasklet(unsigned long data)
 472 {
 473         struct dw_dma *dw = (struct dw_dma *)data;
 474         struct dw_dma_chan *dwc;
 475         u32 status_xfer;
 476         u32 status_err;
 477         unsigned int i;
 478
 479         status_xfer = dma_readl(dw, RAW.XFER);
 480         status_err = dma_readl(dw, RAW.ERROR);
 481
 482         dev_vdbg(dw->dma.dev, "%s: status_err=%x\n", __func__, status_err);
 483
 484         for (i = 0; i < dw->dma.chancnt; i++) {
 485                 dwc = &dw->chan[i];
 486                 if (test_bit(DW_DMA_IS_CYCLIC, &dwc->flags))
 487                         dev_vdbg(dw->dma.dev, "Cyclic xfer is not implemented\n");
 488                 else if (status_err & (1 << i))
 489                         dwc_handle_error(dw, dwc);
 490                 else if (status_xfer & (1 << i))
 491                         dwc_scan_descriptors(dw, dwc);
 492         }
 493
 494         /* Re-enable interrupts */
 495         channel_set_bit(dw, MASK.XFER, dw->all_chan_mask);
 496         channel_set_bit(dw, MASK.ERROR, dw->all_chan_mask);
 497 }
 498
 499 static irqreturn_t dw_dma_interrupt(int irq, void *dev_id)
 500 {
 501         struct dw_dma *dw = dev_id;
 502         u32 status;
 503
 504         /* Check if we have any interrupt from the DMAC which is not in use */
 505         if (!dw->in_use)
 506                 return IRQ_NONE;
 507
 508         status = dma_readl(dw, STATUS_INT);
 509         dev_vdbg(dw->dma.dev, "%s: status=0x%x\n", __func__, status);
 510
 511         /* Check if we have any interrupt from the DMAC */
 512         if (!status)
 513                 return IRQ_NONE;
 514
 515         /*
 516          * Just disable the interrupts. We'll turn them back on in the
 517          * softirq handler.
 518          */
 519         channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
 520         channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask);
 521         channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
 522
 523         status = dma_readl(dw, STATUS_INT);
 524         if (status) {
 525                 dev_err(dw->dma.dev,
 526                         "BUG: Unexpected interrupts pending: 0x%x\n",
 527                         status);
 528
 529                 /* Try to recover */
 530                 channel_clear_bit(dw, MASK.XFER, (1 << 8) - 1);
 531                 channel_clear_bit(dw, MASK.BLOCK, (1 << 8) - 1);
 532                 channel_clear_bit(dw, MASK.SRC_TRAN, (1 << 8) - 1);
 533                 channel_clear_bit(dw, MASK.DST_TRAN, (1 << 8) - 1);
 534                 channel_clear_bit(dw, MASK.ERROR, (1 << 8) - 1);
 535         }
 536
 537         tasklet_schedule(&dw->tasklet);
 538
 539         return IRQ_HANDLED;
 540 }
 541
 542 /*----------------------------------------------------------------------*/
 543
 544 static struct dma_async_tx_descriptor *
 545 dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 546                 size_t len, unsigned long flags)
 547 {
 548         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
 549         struct dw_dma           *dw = to_dw_dma(chan->device);
 550         struct dw_desc          *desc;
 551         struct dw_desc          *first;
 552         struct dw_desc          *prev;
 553         size_t                  xfer_count;
 554         size_t                  offset;
 555         u8                      m_master = dwc->dws.m_master;
 556         unsigned int            src_width;
 557         unsigned int            dst_width;
 558         unsigned int            data_width = dw->pdata->data_width[m_master];
 559         u32                     ctllo, ctlhi;
 560         u8                      lms = DWC_LLP_LMS(m_master);
 561
 562         dev_vdbg(chan2dev(chan),
 563                         "%s: d%pad s%pad l0x%zx f0x%lx\n", __func__,
 564                         &dest, &src, len, flags);
 565
 566         if (unlikely(!len)) {
 567                 dev_dbg(chan2dev(chan), "%s: length is zero!\n", __func__);
 568                 return NULL;
 569         }
 570
 571         dwc->direction = DMA_MEM_TO_MEM;
 572
 573         src_width = dst_width = __ffs(data_width | src | dest | len);
 574
 575         ctllo = dw->prepare_ctllo(dwc)
 576                         | DWC_CTLL_DST_WIDTH(dst_width)
 577                         | DWC_CTLL_SRC_WIDTH(src_width)
 578                         | DWC_CTLL_DST_INC
 579                         | DWC_CTLL_SRC_INC
 580                         | DWC_CTLL_FC_M2M;
 581         prev = first = NULL;
 582
 583         for (offset = 0; offset < len; offset += xfer_count) {
 584                 desc = dwc_desc_get(dwc);
 585                 if (!desc)
 586                         goto err_desc_get;
 587
 588                 ctlhi = dw->bytes2block(dwc, len - offset, src_width, &xfer_count);
 589
 590                 lli_write(desc, sar, src + offset);
 591                 lli_write(desc, dar, dest + offset);
 592                 lli_write(desc, ctllo, ctllo);
 593                 lli_write(desc, ctlhi, ctlhi);
 594                 desc->len = xfer_count;
 595
 596                 if (!first) {
 597                         first = desc;
 598                 } else {
 599                         lli_write(prev, llp, desc->txd.phys | lms);
 600                         list_add_tail(&desc->desc_node, &first->tx_list);
 601                 }
 602                 prev = desc;
 603         }
 604
 605         if (flags & DMA_PREP_INTERRUPT)
 606                 /* Trigger interrupt after last block */
 607                 lli_set(prev, ctllo, DWC_CTLL_INT_EN);
 608
 609         prev->lli.llp = 0;
 610         lli_clear(prev, ctllo, DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);
 611         first->txd.flags = flags;
 612         first->total_len = len;
 613
 614         return &first->txd;
 615
 616 err_desc_get:
 617         dwc_desc_put(dwc, first);
 618         return NULL;
 619 }
 620
 621 static struct dma_async_tx_descriptor *
 622 dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 623                 unsigned int sg_len, enum dma_transfer_direction direction,
 624                 unsigned long flags, void *context)
 625 {
 626         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
 627         struct dw_dma           *dw = to_dw_dma(chan->device);
 628         struct dma_slave_config *sconfig = &dwc->dma_sconfig;
 629         struct dw_desc          *prev;
 630         struct dw_desc          *first;
 631         u32                     ctllo, ctlhi;
 632         u8                      m_master = dwc->dws.m_master;
 633         u8                      lms = DWC_LLP_LMS(m_master);
 634         dma_addr_t              reg;
 635         unsigned int            reg_width;
 636         unsigned int            mem_width;
 637         unsigned int            data_width = dw->pdata->data_width[m_master];
 638         unsigned int            i;
 639         struct scatterlist      *sg;
 640         size_t                  total_len = 0;
 641
 642         dev_vdbg(chan2dev(chan), "%s\n", __func__);
 643
 644         if (unlikely(!is_slave_direction(direction) || !sg_len))
 645                 return NULL;
 646
 647         dwc->direction = direction;
 648
 649         prev = first = NULL;
 650
 651         switch (direction) {
 652         case DMA_MEM_TO_DEV:
 653                 reg_width = __ffs(sconfig->dst_addr_width);
 654                 reg = sconfig->dst_addr;
 655                 ctllo = dw->prepare_ctllo(dwc)
 656                                 | DWC_CTLL_DST_WIDTH(reg_width)
 657                                 | DWC_CTLL_DST_FIX
 658                                 | DWC_CTLL_SRC_INC;
 659
 660                 ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_M2P) :
 661                         DWC_CTLL_FC(DW_DMA_FC_D_M2P);
 662
 663                 for_each_sg(sgl, sg, sg_len, i) {
 664                         struct dw_desc  *desc;
 665                         u32             len, mem;
 666                         size_t          dlen;
 667
 668                         mem = sg_dma_address(sg);
 669                         len = sg_dma_len(sg);
 670
 671                         mem_width = __ffs(data_width | mem | len);
 672
 673 slave_sg_todev_fill_desc:
 674                         desc = dwc_desc_get(dwc);
 675                         if (!desc)
 676                                 goto err_desc_get;
 677
 678                         ctlhi = dw->bytes2block(dwc, len, mem_width, &dlen);
 679
 680                         lli_write(desc, sar, mem);
 681                         lli_write(desc, dar, reg);
 682                         lli_write(desc, ctlhi, ctlhi);
 683                         lli_write(desc, ctllo, ctllo | DWC_CTLL_SRC_WIDTH(mem_width));
 684                         desc->len = dlen;
 685
 686                         if (!first) {
 687                                 first = desc;
 688                         } else {
 689                                 lli_write(prev, llp, desc->txd.phys | lms);
 690                                 list_add_tail(&desc->desc_node, &first->tx_list);
 691                         }
 692                         prev = desc;
 693
 694                         mem += dlen;
 695                         len -= dlen;
 696                         total_len += dlen;
 697
 698                         if (len)
 699                                 goto slave_sg_todev_fill_desc;
 700                 }
 701                 break;
 702         case DMA_DEV_TO_MEM:
 703                 reg_width = __ffs(sconfig->src_addr_width);
 704                 reg = sconfig->src_addr;
 705                 ctllo = dw->prepare_ctllo(dwc)
 706                                 | DWC_CTLL_SRC_WIDTH(reg_width)
 707                                 | DWC_CTLL_DST_INC
 708                                 | DWC_CTLL_SRC_FIX;
 709
 710                 ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_P2M) :
 711                         DWC_CTLL_FC(DW_DMA_FC_D_P2M);
 712
 713                 for_each_sg(sgl, sg, sg_len, i) {
 714                         struct dw_desc  *desc;
 715                         u32             len, mem;
 716                         size_t          dlen;
 717
 718                         mem = sg_dma_address(sg);
 719                         len = sg_dma_len(sg);
 720
 721 slave_sg_fromdev_fill_desc:
 722                         desc = dwc_desc_get(dwc);
 723                         if (!desc)
 724                                 goto err_desc_get;
 725
 726                         ctlhi = dw->bytes2block(dwc, len, reg_width, &dlen);
 727
 728                         lli_write(desc, sar, reg);
 729                         lli_write(desc, dar, mem);
 730                         lli_write(desc, ctlhi, ctlhi);
 731                         mem_width = __ffs(data_width | mem | dlen);
 732                         lli_write(desc, ctllo, ctllo | DWC_CTLL_DST_WIDTH(mem_width));
 733                         desc->len = dlen;
 734
 735                         if (!first) {
 736                                 first = desc;
 737                         } else {
 738                                 lli_write(prev, llp, desc->txd.phys | lms);
 739                                 list_add_tail(&desc->desc_node, &first->tx_list);
 740                         }
 741                         prev = desc;
 742
 743                         mem += dlen;
 744                         len -= dlen;
 745                         total_len += dlen;
 746
 747                         if (len)
 748                                 goto slave_sg_fromdev_fill_desc;
 749                 }
 750                 break;
 751         default:
 752                 return NULL;
 753         }
 754
 755         if (flags & DMA_PREP_INTERRUPT)
 756                 /* Trigger interrupt after last block */
 757                 lli_set(prev, ctllo, DWC_CTLL_INT_EN);
 758
 759         prev->lli.llp = 0;
 760         lli_clear(prev, ctllo, DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);
 761         first->total_len = total_len;
 762
 763         return &first->txd;
 764
 765 err_desc_get:
 766         dev_err(chan2dev(chan),
 767                 "not enough descriptors available. Direction %d\n", direction);
 768         dwc_desc_put(dwc, first);
 769         return NULL;
 770 }
 771
 772 bool dw_dma_filter(struct dma_chan *chan, void *param)
 773 {
 774         struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
 775         struct dw_dma_slave *dws = param;
 776
 777         if (dws->dma_dev != chan->device->dev)
 778                 return false;
 779
 780         /* We have to copy data since dws can be temporary storage */
 781         memcpy(&dwc->dws, dws, sizeof(struct dw_dma_slave));
 782
 783         return true;
 784 }
 785 EXPORT_SYMBOL_GPL(dw_dma_filter);
 786
 787 static int dwc_config(struct dma_chan *chan, struct dma_slave_config *sconfig)
 788 {
 789         struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
 790         struct dw_dma *dw = to_dw_dma(chan->device);
 791
 792         memcpy(&dwc->dma_sconfig, sconfig, sizeof(*sconfig));
 793
 794         dw->encode_maxburst(dwc, &dwc->dma_sconfig.src_maxburst);
 795         dw->encode_maxburst(dwc, &dwc->dma_sconfig.dst_maxburst);
 796
 797         return 0;
 798 }
 799
 800 static void dwc_chan_pause(struct dw_dma_chan *dwc, bool drain)
 801 {
 802         struct dw_dma *dw = to_dw_dma(dwc->chan.device);
 803         unsigned int            count = 20;     /* timeout iterations */
 804
 805         dw->suspend_chan(dwc, drain);
 806
 807         while (!(channel_readl(dwc, CFG_LO) & DWC_CFGL_FIFO_EMPTY) && count--)
 808                 udelay(2);
 809
 810         set_bit(DW_DMA_IS_PAUSED, &dwc->flags);
 811 }
 812
 813 static int dwc_pause(struct dma_chan *chan)
 814 {
 815         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
 816         unsigned long           flags;
 817
 818         spin_lock_irqsave(&dwc->lock, flags);
 819         dwc_chan_pause(dwc, false);
 820         spin_unlock_irqrestore(&dwc->lock, flags);
 821
 822         return 0;
 823 }
 824
 825 static inline void dwc_chan_resume(struct dw_dma_chan *dwc, bool drain)
 826 {
 827         struct dw_dma *dw = to_dw_dma(dwc->chan.device);
 828
 829         dw->resume_chan(dwc, drain);
 830
 831         clear_bit(DW_DMA_IS_PAUSED, &dwc->flags);
 832 }
 833
 834 static int dwc_resume(struct dma_chan *chan)
 835 {
 836         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
 837         unsigned long           flags;
 838
 839         spin_lock_irqsave(&dwc->lock, flags);
 840
 841         if (test_bit(DW_DMA_IS_PAUSED, &dwc->flags))
 842                 dwc_chan_resume(dwc, false);
 843
 844         spin_unlock_irqrestore(&dwc->lock, flags);
 845
 846         return 0;
 847 }
 848
 849 static int dwc_terminate_all(struct dma_chan *chan)
 850 {
 851         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
 852         struct dw_dma           *dw = to_dw_dma(chan->device);
 853         struct dw_desc          *desc, *_desc;
 854         unsigned long           flags;
 855         LIST_HEAD(list);
 856
 857         spin_lock_irqsave(&dwc->lock, flags);
 858
 859         clear_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags);
 860
 861         dwc_chan_pause(dwc, true);
 862
 863         dwc_chan_disable(dw, dwc);
 864
 865         dwc_chan_resume(dwc, true);
 866
 867         /* active_list entries will end up before queued entries */
 868         list_splice_init(&dwc->queue, &list);
 869         list_splice_init(&dwc->active_list, &list);
 870
 871         spin_unlock_irqrestore(&dwc->lock, flags);
 872
 873         /* Flush all pending and queued descriptors */
 874         list_for_each_entry_safe(desc, _desc, &list, desc_node)
 875                 dwc_descriptor_complete(dwc, desc, false);
 876
 877         return 0;
 878 }
 879
 880 static struct dw_desc *dwc_find_desc(struct dw_dma_chan *dwc, dma_cookie_t c)
 881 {
 882         struct dw_desc *desc;
 883
 884         list_for_each_entry(desc, &dwc->active_list, desc_node)
 885                 if (desc->txd.cookie == c)
 886                         return desc;
 887
 888         return NULL;
 889 }
 890
 891 static u32 dwc_get_residue(struct dw_dma_chan *dwc, dma_cookie_t cookie)
 892 {
 893         struct dw_desc *desc;
 894         unsigned long flags;
 895         u32 residue;
 896
 897         spin_lock_irqsave(&dwc->lock, flags);
 898
 899         desc = dwc_find_desc(dwc, cookie);
 900         if (desc) {
 901                 if (desc == dwc_first_active(dwc)) {
 902                         residue = desc->residue;
 903                         if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags) && residue)
 904                                 residue -= dwc_get_sent(dwc);
 905                 } else {
 906                         residue = desc->total_len;
 907                 }
 908         } else {
 909                 residue = 0;
 910         }
 911
 912         spin_unlock_irqrestore(&dwc->lock, flags);
 913         return residue;
 914 }
 915
 916 static enum dma_status
 917 dwc_tx_status(struct dma_chan *chan,
 918               dma_cookie_t cookie,
 919               struct dma_tx_state *txstate)
 920 {
 921         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
 922         enum dma_status         ret;
 923
 924         ret = dma_cookie_status(chan, cookie, txstate);
 925         if (ret == DMA_COMPLETE)
 926                 return ret;
 927
 928         dwc_scan_descriptors(to_dw_dma(chan->device), dwc);
 929
 930         ret = dma_cookie_status(chan, cookie, txstate);
 931         if (ret == DMA_COMPLETE)
 932                 return ret;
 933
 934         dma_set_residue(txstate, dwc_get_residue(dwc, cookie));
 935
 936         if (test_bit(DW_DMA_IS_PAUSED, &dwc->flags) && ret == DMA_IN_PROGRESS)
 937                 return DMA_PAUSED;
 938
 939         return ret;
 940 }
 941
 942 static void dwc_issue_pending(struct dma_chan *chan)
 943 {
 944         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
 945         unsigned long           flags;
 946
 947         spin_lock_irqsave(&dwc->lock, flags);
 948         if (list_empty(&dwc->active_list))
 949                 dwc_dostart_first_queued(dwc);
 950         spin_unlock_irqrestore(&dwc->lock, flags);
 951 }
 952
 953 /*----------------------------------------------------------------------*/
 954
 955 void do_dw_dma_off(struct dw_dma *dw)
 956 {
 957         unsigned int i;
 958
 959         dma_writel(dw, CFG, 0);
 960
 961         channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
 962         channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask);
 963         channel_clear_bit(dw, MASK.SRC_TRAN, dw->all_chan_mask);
 964         channel_clear_bit(dw, MASK.DST_TRAN, dw->all_chan_mask);
 965         channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
 966
 967         while (dma_readl(dw, CFG) & DW_CFG_DMA_EN)
 968                 cpu_relax();
 969
 970         for (i = 0; i < dw->dma.chancnt; i++)
 971                 clear_bit(DW_DMA_IS_INITIALIZED, &dw->chan[i].flags);
 972 }
 973
 974 void do_dw_dma_on(struct dw_dma *dw)
 975 {
 976         dma_writel(dw, CFG, DW_CFG_DMA_EN);
 977 }
 978
 979 static int dwc_alloc_chan_resources(struct dma_chan *chan)
 980 {
 981         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
 982         struct dw_dma           *dw = to_dw_dma(chan->device);
 983
 984         dev_vdbg(chan2dev(chan), "%s\n", __func__);
 985
 986         /* ASSERT:  channel is idle */
 987         if (dma_readl(dw, CH_EN) & dwc->mask) {
 988                 dev_dbg(chan2dev(chan), "DMA channel not idle?\n");
 989                 return -EIO;
 990         }
 991
 992         dma_cookie_init(chan);
 993
 994         /*
 995          * NOTE: some controllers may have additional features that we
 996          * need to initialize here, like "scatter-gather" (which
 997          * doesn't mean what you think it means), and status writeback.
 998          */
 999
1000         /*
1001          * We need controller-specific data to set up slave transfers.
1002          */
1003         if (chan->private && !dw_dma_filter(chan, chan->private)) {
1004                 dev_warn(chan2dev(chan), "Wrong controller-specific data\n");
1005                 return -EINVAL;
1006         }
1007
1008         /* Enable controller here if needed */
1009         if (!dw->in_use)
1010                 do_dw_dma_on(dw);
1011         dw->in_use |= dwc->mask;
1012
1013         return 0;
1014 }
1015
1016 static void dwc_free_chan_resources(struct dma_chan *chan)
1017 {
1018         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
1019         struct dw_dma           *dw = to_dw_dma(chan->device);
1020         unsigned long           flags;
1021
1022         dev_dbg(chan2dev(chan), "%s: descs allocated=%u\n", __func__,
1023                         dwc->descs_allocated);
1024
1025         /* ASSERT:  channel is idle */
1026         BUG_ON(!list_empty(&dwc->active_list));
1027         BUG_ON(!list_empty(&dwc->queue));
1028         BUG_ON(dma_readl(to_dw_dma(chan->device), CH_EN) & dwc->mask);
1029
1030         spin_lock_irqsave(&dwc->lock, flags);
1031
1032         /* Clear custom channel configuration */
1033         memset(&dwc->dws, 0, sizeof(struct dw_dma_slave));
1034
1035         clear_bit(DW_DMA_IS_INITIALIZED, &dwc->flags);
1036
1037         /* Disable interrupts */
1038         channel_clear_bit(dw, MASK.XFER, dwc->mask);
1039         channel_clear_bit(dw, MASK.BLOCK, dwc->mask);
1040         channel_clear_bit(dw, MASK.ERROR, dwc->mask);
1041
1042         spin_unlock_irqrestore(&dwc->lock, flags);
1043
1044         /* Disable controller in case it was a last user */
1045         dw->in_use &= ~dwc->mask;
1046         if (!dw->in_use)
1047                 do_dw_dma_off(dw);
1048
1049         dev_vdbg(chan2dev(chan), "%s: done\n", __func__);
1050 }
1051
1052 int do_dma_probe(struct dw_dma_chip *chip)
1053 {
1054         struct dw_dma *dw = chip->dw;
1055         struct dw_dma_platform_data *pdata;
1056         bool                    autocfg = false;
1057         unsigned int            dw_params;
1058         unsigned int            i;
1059         int                     err;
1060
1061         dw->pdata = devm_kzalloc(chip->dev, sizeof(*dw->pdata), GFP_KERNEL);
1062         if (!dw->pdata)
1063                 return -ENOMEM;
1064
1065         dw->regs = chip->regs;
1066
1067         pm_runtime_get_sync(chip->dev);
1068
1069         if (!chip->pdata) {
1070                 dw_params = dma_readl(dw, DW_PARAMS);
1071                 dev_dbg(chip->dev, "DW_PARAMS: 0x%08x\n", dw_params);
1072
1073                 autocfg = dw_params >> DW_PARAMS_EN & 1;
1074                 if (!autocfg) {
1075                         err = -EINVAL;
1076                         goto err_pdata;
1077                 }
1078
1079                 /* Reassign the platform data pointer */
1080                 pdata = dw->pdata;
1081
1082                 /* Get hardware configuration parameters */
1083                 pdata->nr_channels = (dw_params >> DW_PARAMS_NR_CHAN & 7) + 1;
1084                 pdata->nr_masters = (dw_params >> DW_PARAMS_NR_MASTER & 3) + 1;
1085                 for (i = 0; i < pdata->nr_masters; i++) {
1086                         pdata->data_width[i] =
1087                                 4 << (dw_params >> DW_PARAMS_DATA_WIDTH(i) & 3);
1088                 }
1089                 pdata->block_size = dma_readl(dw, MAX_BLK_SIZE);
1090
1091                 /* Fill platform data with the default values */
1092                 pdata->chan_allocation_order = CHAN_ALLOCATION_ASCENDING;
1093                 pdata->chan_priority = CHAN_PRIORITY_ASCENDING;
1094         } else if (chip->pdata->nr_channels > DW_DMA_MAX_NR_CHANNELS) {
1095                 err = -EINVAL;
1096                 goto err_pdata;
1097         } else {
1098                 memcpy(dw->pdata, chip->pdata, sizeof(*dw->pdata));
1099
1100                 /* Reassign the platform data pointer */
1101                 pdata = dw->pdata;
1102         }
1103
1104         dw->chan = devm_kcalloc(chip->dev, pdata->nr_channels, sizeof(*dw->chan),
1105                                 GFP_KERNEL);
1106         if (!dw->chan) {
1107                 err = -ENOMEM;
1108                 goto err_pdata;
1109         }
1110
1111         /* Calculate all channel mask before DMA setup */
1112         dw->all_chan_mask = (1 << pdata->nr_channels) - 1;
1113
1114         /* Force dma off, just in case */
1115         dw->disable(dw);
1116
1117         /* Device and instance ID for IRQ and DMA pool */
1118         dw->set_device_name(dw, chip->id);
1119
1120         /* Create a pool of consistent memory blocks for hardware descriptors */
1121         dw->desc_pool = dmam_pool_create(dw->name, chip->dev,
1122                                          sizeof(struct dw_desc), 4, 0);
1123         if (!dw->desc_pool) {
1124                 dev_err(chip->dev, "No memory for descriptors dma pool\n");
1125                 err = -ENOMEM;
1126                 goto err_pdata;
1127         }
1128
1129         tasklet_init(&dw->tasklet, dw_dma_tasklet, (unsigned long)dw);
1130
1131         err = request_irq(chip->irq, dw_dma_interrupt, IRQF_SHARED,
1132                           dw->name, dw);
1133         if (err)
1134                 goto err_pdata;
1135
1136         INIT_LIST_HEAD(&dw->dma.channels);
1137         for (i = 0; i < pdata->nr_channels; i++) {
1138                 struct dw_dma_chan      *dwc = &dw->chan[i];
1139
1140                 dwc->chan.device = &dw->dma;
1141                 dma_cookie_init(&dwc->chan);
1142                 if (pdata->chan_allocation_order == CHAN_ALLOCATION_ASCENDING)
1143                         list_add_tail(&dwc->chan.device_node,
1144                                         &dw->dma.channels);
1145                 else
1146                         list_add(&dwc->chan.device_node, &dw->dma.channels);
1147
1148                 /* 7 is highest priority & 0 is lowest. */
1149                 if (pdata->chan_priority == CHAN_PRIORITY_ASCENDING)
1150                         dwc->priority = pdata->nr_channels - i - 1;
1151                 else
1152                         dwc->priority = i;
1153
1154                 dwc->ch_regs = &__dw_regs(dw)->CHAN[i];
1155                 spin_lock_init(&dwc->lock);
1156                 dwc->mask = 1 << i;
1157
1158                 INIT_LIST_HEAD(&dwc->active_list);
1159                 INIT_LIST_HEAD(&dwc->queue);
1160
1161                 channel_clear_bit(dw, CH_EN, dwc->mask);
1162
1163                 dwc->direction = DMA_TRANS_NONE;
1164
1165                 /* Hardware configuration */
1166                 if (autocfg) {
1167                         unsigned int r = DW_DMA_MAX_NR_CHANNELS - i - 1;
1168                         void __iomem *addr = &__dw_regs(dw)->DWC_PARAMS[r];
1169                         unsigned int dwc_params = readl(addr);
1170
1171                         dev_dbg(chip->dev, "DWC_PARAMS[%d]: 0x%08x\n", i,
1172                                            dwc_params);
1173
1174                         /*
1175                          * Decode maximum block size for given channel. The
1176                          * stored 4 bit value represents blocks from 0x00 for 3
1177                          * up to 0x0a for 4095.
1178                          */
1179                         dwc->block_size =
1180                                 (4 << ((pdata->block_size >> 4 * i) & 0xf)) - 1;
1181                         dwc->nollp =
1182                                 (dwc_params >> DWC_PARAMS_MBLK_EN & 0x1) == 0;
1183                 } else {
1184                         dwc->block_size = pdata->block_size;
1185                         dwc->nollp = !pdata->multi_block[i];
1186                 }
1187         }
1188
1189         /* Clear all interrupts on all channels. */
1190         dma_writel(dw, CLEAR.XFER, dw->all_chan_mask);
1191         dma_writel(dw, CLEAR.BLOCK, dw->all_chan_mask);
1192         dma_writel(dw, CLEAR.SRC_TRAN, dw->all_chan_mask);
1193         dma_writel(dw, CLEAR.DST_TRAN, dw->all_chan_mask);
1194         dma_writel(dw, CLEAR.ERROR, dw->all_chan_mask);
1195
1196         /* Set capabilities */
1197         dma_cap_set(DMA_SLAVE, dw->dma.cap_mask);
1198         dma_cap_set(DMA_PRIVATE, dw->dma.cap_mask);
1199         dma_cap_set(DMA_MEMCPY, dw->dma.cap_mask);
1200
1201         dw->dma.dev = chip->dev;
1202         dw->dma.device_alloc_chan_resources = dwc_alloc_chan_resources;
1203         dw->dma.device_free_chan_resources = dwc_free_chan_resources;
1204
1205         dw->dma.device_prep_dma_memcpy = dwc_prep_dma_memcpy;
1206         dw->dma.device_prep_slave_sg = dwc_prep_slave_sg;
1207
1208         dw->dma.device_config = dwc_config;
1209         dw->dma.device_pause = dwc_pause;
1210         dw->dma.device_resume = dwc_resume;
1211         dw->dma.device_terminate_all = dwc_terminate_all;
1212
1213         dw->dma.device_tx_status = dwc_tx_status;
1214         dw->dma.device_issue_pending = dwc_issue_pending;
1215
1216         /* DMA capabilities */
1217         dw->dma.src_addr_widths = DW_DMA_BUSWIDTHS;
1218         dw->dma.dst_addr_widths = DW_DMA_BUSWIDTHS;
1219         dw->dma.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) |
1220                              BIT(DMA_MEM_TO_MEM);
1221         dw->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1222
1223         err = dma_async_device_register(&dw->dma);
1224         if (err)
1225                 goto err_dma_register;
1226
1227         dev_info(chip->dev, "DesignWare DMA Controller, %d channels\n",
1228                  pdata->nr_channels);
1229
1230         pm_runtime_put_sync_suspend(chip->dev);
1231
1232         return 0;
1233
1234 err_dma_register:
1235         free_irq(chip->irq, dw);
1236 err_pdata:
1237         pm_runtime_put_sync_suspend(chip->dev);
1238         return err;
1239 }
1240
1241 int do_dma_remove(struct dw_dma_chip *chip)
1242 {
1243         struct dw_dma           *dw = chip->dw;
1244         struct dw_dma_chan      *dwc, *_dwc;
1245
1246         pm_runtime_get_sync(chip->dev);
1247
1248         do_dw_dma_off(dw);
1249         dma_async_device_unregister(&dw->dma);
1250
1251         free_irq(chip->irq, dw);
1252         tasklet_kill(&dw->tasklet);
1253
1254         list_for_each_entry_safe(dwc, _dwc, &dw->dma.channels,
1255                         chan.device_node) {
1256                 list_del(&dwc->chan.device_node);
1257                 channel_clear_bit(dw, CH_EN, dwc->mask);
1258         }
1259
1260         pm_runtime_put_sync_suspend(chip->dev);
1261         return 0;
1262 }
1263
1264 int do_dw_dma_disable(struct dw_dma_chip *chip)
1265 {
1266         struct dw_dma *dw = chip->dw;
1267
1268         dw->disable(dw);
1269         return 0;
1270 }
1271 EXPORT_SYMBOL_GPL(do_dw_dma_disable);
1272
1273 int do_dw_dma_enable(struct dw_dma_chip *chip)
1274 {
1275         struct dw_dma *dw = chip->dw;
1276
1277         dw->enable(dw);
1278         return 0;
1279 }
1280 EXPORT_SYMBOL_GPL(do_dw_dma_enable);
1281
1282 MODULE_LICENSE("GPL v2");
1283 MODULE_DESCRIPTION("Synopsys DesignWare DMA Controller core driver");
1284 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1285 MODULE_AUTHOR("Viresh Kumar <[email protected]>");