drivers/dma/sh/rz-dmac.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Renesas RZ/G2L DMA Controller Driver
   4  *
   5  * Based on imx-dma.c
   6  *
   7  * Copyright (C) 2021 Renesas Electronics Corp.
   8  * Copyright 2010 Sascha Hauer, Pengutronix <[email protected]>
   9  * Copyright 2012 Javier Martin, Vista Silicon <[email protected]>
  10  */
  11
  12 #include <linux/dma-mapping.h>
  13 #include <linux/dmaengine.h>
  14 #include <linux/interrupt.h>
  15 #include <linux/list.h>
  16 #include <linux/module.h>
  17 #include <linux/of.h>
  18 #include <linux/of_dma.h>
  19 #include <linux/of_platform.h>
  20 #include <linux/platform_device.h>
  21 #include <linux/pm_runtime.h>
  22 #include <linux/slab.h>
  23 #include <linux/spinlock.h>
  24
  25 #include "../dmaengine.h"
  26 #include "../virt-dma.h"
  27
  28 enum  rz_dmac_prep_type {
  29         RZ_DMAC_DESC_MEMCPY,
  30         RZ_DMAC_DESC_SLAVE_SG,
  31 };
  32
  33 struct rz_lmdesc {
  34         u32 header;
  35         u32 sa;
  36         u32 da;
  37         u32 tb;
  38         u32 chcfg;
  39         u32 chitvl;
  40         u32 chext;
  41         u32 nxla;
  42 };
  43
  44 struct rz_dmac_desc {
  45         struct virt_dma_desc vd;
  46         dma_addr_t src;
  47         dma_addr_t dest;
  48         size_t len;
  49         struct list_head node;
  50         enum dma_transfer_direction direction;
  51         enum rz_dmac_prep_type type;
  52         /* For slave sg */
  53         struct scatterlist *sg;
  54         unsigned int sgcount;
  55 };
  56
  57 #define to_rz_dmac_desc(d)      container_of(d, struct rz_dmac_desc, vd)
  58
  59 struct rz_dmac_chan {
  60         struct virt_dma_chan vc;
  61         void __iomem *ch_base;
  62         void __iomem *ch_cmn_base;
  63         unsigned int index;
  64         int irq;
  65         struct rz_dmac_desc *desc;
  66         int descs_allocated;
  67
  68         enum dma_slave_buswidth src_word_size;
  69         enum dma_slave_buswidth dst_word_size;
  70         dma_addr_t src_per_address;
  71         dma_addr_t dst_per_address;
  72
  73         u32 chcfg;
  74         u32 chctrl;
  75         int mid_rid;
  76
  77         struct list_head ld_free;
  78         struct list_head ld_queue;
  79         struct list_head ld_active;
  80
  81         struct {
  82                 struct rz_lmdesc *base;
  83                 struct rz_lmdesc *head;
  84                 struct rz_lmdesc *tail;
  85                 dma_addr_t base_dma;
  86         } lmdesc;
  87 };
  88
  89 #define to_rz_dmac_chan(c)      container_of(c, struct rz_dmac_chan, vc.chan)
  90
  91 struct rz_dmac {
  92         struct dma_device engine;
  93         struct device *dev;
  94         void __iomem *base;
  95         void __iomem *ext_base;
  96
  97         unsigned int n_channels;
  98         struct rz_dmac_chan *channels;
  99
 100         DECLARE_BITMAP(modules, 1024);
 101 };
 102
 103 #define to_rz_dmac(d)   container_of(d, struct rz_dmac, engine)
 104
 105 /*
 106  * -----------------------------------------------------------------------------
 107  * Registers
 108  */
 109
 110 #define CHSTAT                          0x0024
 111 #define CHCTRL                          0x0028
 112 #define CHCFG                           0x002c
 113 #define NXLA                            0x0038
 114
 115 #define DCTRL                           0x0000
 116
 117 #define EACH_CHANNEL_OFFSET             0x0040
 118 #define CHANNEL_0_7_OFFSET              0x0000
 119 #define CHANNEL_0_7_COMMON_BASE         0x0300
 120 #define CHANNEL_8_15_OFFSET             0x0400
 121 #define CHANNEL_8_15_COMMON_BASE        0x0700
 122
 123 #define CHSTAT_ER                       BIT(4)
 124 #define CHSTAT_EN                       BIT(0)
 125
 126 #define CHCTRL_CLRINTMSK                BIT(17)
 127 #define CHCTRL_CLRSUS                   BIT(9)
 128 #define CHCTRL_CLRTC                    BIT(6)
 129 #define CHCTRL_CLREND                   BIT(5)
 130 #define CHCTRL_CLRRQ                    BIT(4)
 131 #define CHCTRL_SWRST                    BIT(3)
 132 #define CHCTRL_STG                      BIT(2)
 133 #define CHCTRL_CLREN                    BIT(1)
 134 #define CHCTRL_SETEN                    BIT(0)
 135 #define CHCTRL_DEFAULT                  (CHCTRL_CLRINTMSK | CHCTRL_CLRSUS | \
 136                                          CHCTRL_CLRTC | CHCTRL_CLREND | \
 137                                          CHCTRL_CLRRQ | CHCTRL_SWRST | \
 138                                          CHCTRL_CLREN)
 139
 140 #define CHCFG_DMS                       BIT(31)
 141 #define CHCFG_DEM                       BIT(24)
 142 #define CHCFG_DAD                       BIT(21)
 143 #define CHCFG_SAD                       BIT(20)
 144 #define CHCFG_REQD                      BIT(3)
 145 #define CHCFG_SEL(bits)                 ((bits) & 0x07)
 146 #define CHCFG_MEM_COPY                  (0x80400008)
 147 #define CHCFG_FILL_DDS(a)               (((a) << 16) & GENMASK(19, 16))
 148 #define CHCFG_FILL_SDS(a)               (((a) << 12) & GENMASK(15, 12))
 149 #define CHCFG_FILL_TM(a)                (((a) & BIT(5)) << 22)
 150 #define CHCFG_FILL_AM(a)                (((a) & GENMASK(4, 2)) << 6)
 151 #define CHCFG_FILL_LVL(a)               (((a) & BIT(1)) << 5)
 152 #define CHCFG_FILL_HIEN(a)              (((a) & BIT(0)) << 5)
 153
 154 #define MID_RID_MASK                    GENMASK(9, 0)
 155 #define CHCFG_MASK                      GENMASK(15, 10)
 156 #define CHCFG_DS_INVALID                0xFF
 157 #define DCTRL_LVINT                     BIT(1)
 158 #define DCTRL_PR                        BIT(0)
 159 #define DCTRL_DEFAULT                   (DCTRL_LVINT | DCTRL_PR)
 160
 161 /* LINK MODE DESCRIPTOR */
 162 #define HEADER_LV                       BIT(0)
 163
 164 #define RZ_DMAC_MAX_CHAN_DESCRIPTORS    16
 165 #define RZ_DMAC_MAX_CHANNELS            16
 166 #define DMAC_NR_LMDESC                  64
 167
 168 /*
 169  * -----------------------------------------------------------------------------
 170  * Device access
 171  */
 172
 173 static void rz_dmac_writel(struct rz_dmac *dmac, unsigned int val,
 174                            unsigned int offset)
 175 {
 176         writel(val, dmac->base + offset);
 177 }
 178
 179 static void rz_dmac_ext_writel(struct rz_dmac *dmac, unsigned int val,
 180                                unsigned int offset)
 181 {
 182         writel(val, dmac->ext_base + offset);
 183 }
 184
 185 static u32 rz_dmac_ext_readl(struct rz_dmac *dmac, unsigned int offset)
 186 {
 187         return readl(dmac->ext_base + offset);
 188 }
 189
 190 static void rz_dmac_ch_writel(struct rz_dmac_chan *channel, unsigned int val,
 191                               unsigned int offset, int which)
 192 {
 193         if (which)
 194                 writel(val, channel->ch_base + offset);
 195         else
 196                 writel(val, channel->ch_cmn_base + offset);
 197 }
 198
 199 static u32 rz_dmac_ch_readl(struct rz_dmac_chan *channel,
 200                             unsigned int offset, int which)
 201 {
 202         if (which)
 203                 return readl(channel->ch_base + offset);
 204         else
 205                 return readl(channel->ch_cmn_base + offset);
 206 }
 207
 208 /*
 209  * -----------------------------------------------------------------------------
 210  * Initialization
 211  */
 212
 213 static void rz_lmdesc_setup(struct rz_dmac_chan *channel,
 214                             struct rz_lmdesc *lmdesc)
 215 {
 216         u32 nxla;
 217
 218         channel->lmdesc.base = lmdesc;
 219         channel->lmdesc.head = lmdesc;
 220         channel->lmdesc.tail = lmdesc;
 221         nxla = channel->lmdesc.base_dma;
 222         while (lmdesc < (channel->lmdesc.base + (DMAC_NR_LMDESC - 1))) {
 223                 lmdesc->header = 0;
 224                 nxla += sizeof(*lmdesc);
 225                 lmdesc->nxla = nxla;
 226                 lmdesc++;
 227         }
 228
 229         lmdesc->header = 0;
 230         lmdesc->nxla = channel->lmdesc.base_dma;
 231 }
 232
 233 /*
 234  * -----------------------------------------------------------------------------
 235  * Descriptors preparation
 236  */
 237
 238 static void rz_dmac_lmdesc_recycle(struct rz_dmac_chan *channel)
 239 {
 240         struct rz_lmdesc *lmdesc = channel->lmdesc.head;
 241
 242         while (!(lmdesc->header & HEADER_LV)) {
 243                 lmdesc->header = 0;
 244                 lmdesc++;
 245                 if (lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC))
 246                         lmdesc = channel->lmdesc.base;
 247         }
 248         channel->lmdesc.head = lmdesc;
 249 }
 250
 251 static void rz_dmac_enable_hw(struct rz_dmac_chan *channel)
 252 {
 253         struct dma_chan *chan = &channel->vc.chan;
 254         struct rz_dmac *dmac = to_rz_dmac(chan->device);
 255         unsigned long flags;
 256         u32 nxla;
 257         u32 chctrl;
 258         u32 chstat;
 259
 260         dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index);
 261
 262         local_irq_save(flags);
 263
 264         rz_dmac_lmdesc_recycle(channel);
 265
 266         nxla = channel->lmdesc.base_dma +
 267                 (sizeof(struct rz_lmdesc) * (channel->lmdesc.head -
 268                                              channel->lmdesc.base));
 269
 270         chstat = rz_dmac_ch_readl(channel, CHSTAT, 1);
 271         if (!(chstat & CHSTAT_EN)) {
 272                 chctrl = (channel->chctrl | CHCTRL_SETEN);
 273                 rz_dmac_ch_writel(channel, nxla, NXLA, 1);
 274                 rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1);
 275                 rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1);
 276                 rz_dmac_ch_writel(channel, chctrl, CHCTRL, 1);
 277         }
 278
 279         local_irq_restore(flags);
 280 }
 281
 282 static void rz_dmac_disable_hw(struct rz_dmac_chan *channel)
 283 {
 284         struct dma_chan *chan = &channel->vc.chan;
 285         struct rz_dmac *dmac = to_rz_dmac(chan->device);
 286         unsigned long flags;
 287
 288         dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index);
 289
 290         local_irq_save(flags);
 291         rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
 292         local_irq_restore(flags);
 293 }
 294
 295 static void rz_dmac_set_dmars_register(struct rz_dmac *dmac, int nr, u32 dmars)
 296 {
 297         u32 dmars_offset = (nr / 2) * 4;
 298         u32 shift = (nr % 2) * 16;
 299         u32 dmars32;
 300
 301         dmars32 = rz_dmac_ext_readl(dmac, dmars_offset);
 302         dmars32 &= ~(0xffff << shift);
 303         dmars32 |= dmars << shift;
 304
 305         rz_dmac_ext_writel(dmac, dmars32, dmars_offset);
 306 }
 307
 308 static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel)
 309 {
 310         struct dma_chan *chan = &channel->vc.chan;
 311         struct rz_dmac *dmac = to_rz_dmac(chan->device);
 312         struct rz_lmdesc *lmdesc = channel->lmdesc.tail;
 313         struct rz_dmac_desc *d = channel->desc;
 314         u32 chcfg = CHCFG_MEM_COPY;
 315
 316         /* prepare descriptor */
 317         lmdesc->sa = d->src;
 318         lmdesc->da = d->dest;
 319         lmdesc->tb = d->len;
 320         lmdesc->chcfg = chcfg;
 321         lmdesc->chitvl = 0;
 322         lmdesc->chext = 0;
 323         lmdesc->header = HEADER_LV;
 324
 325         rz_dmac_set_dmars_register(dmac, channel->index, 0);
 326
 327         channel->chcfg = chcfg;
 328         channel->chctrl = CHCTRL_STG | CHCTRL_SETEN;
 329 }
 330
 331 static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel)
 332 {
 333         struct dma_chan *chan = &channel->vc.chan;
 334         struct rz_dmac *dmac = to_rz_dmac(chan->device);
 335         struct rz_dmac_desc *d = channel->desc;
 336         struct scatterlist *sg, *sgl = d->sg;
 337         struct rz_lmdesc *lmdesc;
 338         unsigned int i, sg_len = d->sgcount;
 339
 340         channel->chcfg |= CHCFG_SEL(channel->index) | CHCFG_DEM | CHCFG_DMS;
 341
 342         if (d->direction == DMA_DEV_TO_MEM) {
 343                 channel->chcfg |= CHCFG_SAD;
 344                 channel->chcfg &= ~CHCFG_REQD;
 345         } else {
 346                 channel->chcfg |= CHCFG_DAD | CHCFG_REQD;
 347         }
 348
 349         lmdesc = channel->lmdesc.tail;
 350
 351         for (i = 0, sg = sgl; i < sg_len; i++, sg = sg_next(sg)) {
 352                 if (d->direction == DMA_DEV_TO_MEM) {
 353                         lmdesc->sa = channel->src_per_address;
 354                         lmdesc->da = sg_dma_address(sg);
 355                 } else {
 356                         lmdesc->sa = sg_dma_address(sg);
 357                         lmdesc->da = channel->dst_per_address;
 358                 }
 359
 360                 lmdesc->tb = sg_dma_len(sg);
 361                 lmdesc->chitvl = 0;
 362                 lmdesc->chext = 0;
 363                 if (i == (sg_len - 1)) {
 364                         lmdesc->chcfg = (channel->chcfg & ~CHCFG_DEM);
 365                         lmdesc->header = HEADER_LV;
 366                 } else {
 367                         lmdesc->chcfg = channel->chcfg;
 368                         lmdesc->header = HEADER_LV;
 369                 }
 370                 if (++lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC))
 371                         lmdesc = channel->lmdesc.base;
 372         }
 373
 374         channel->lmdesc.tail = lmdesc;
 375
 376         rz_dmac_set_dmars_register(dmac, channel->index, channel->mid_rid);
 377         channel->chctrl = CHCTRL_SETEN;
 378 }
 379
 380 static int rz_dmac_xfer_desc(struct rz_dmac_chan *chan)
 381 {
 382         struct rz_dmac_desc *d = chan->desc;
 383         struct virt_dma_desc *vd;
 384
 385         vd = vchan_next_desc(&chan->vc);
 386         if (!vd)
 387                 return 0;
 388
 389         list_del(&vd->node);
 390
 391         switch (d->type) {
 392         case RZ_DMAC_DESC_MEMCPY:
 393                 rz_dmac_prepare_desc_for_memcpy(chan);
 394                 break;
 395
 396         case RZ_DMAC_DESC_SLAVE_SG:
 397                 rz_dmac_prepare_descs_for_slave_sg(chan);
 398                 break;
 399
 400         default:
 401                 return -EINVAL;
 402         }
 403
 404         rz_dmac_enable_hw(chan);
 405
 406         return 0;
 407 }
 408
 409 /*
 410  * -----------------------------------------------------------------------------
 411  * DMA engine operations
 412  */
 413
 414 static int rz_dmac_alloc_chan_resources(struct dma_chan *chan)
 415 {
 416         struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
 417
 418         while (channel->descs_allocated < RZ_DMAC_MAX_CHAN_DESCRIPTORS) {
 419                 struct rz_dmac_desc *desc;
 420
 421                 desc = kzalloc(sizeof(*desc), GFP_KERNEL);
 422                 if (!desc)
 423                         break;
 424
 425                 list_add_tail(&desc->node, &channel->ld_free);
 426                 channel->descs_allocated++;
 427         }
 428
 429         if (!channel->descs_allocated)
 430                 return -ENOMEM;
 431
 432         return channel->descs_allocated;
 433 }
 434
 435 static void rz_dmac_free_chan_resources(struct dma_chan *chan)
 436 {
 437         struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
 438         struct rz_dmac *dmac = to_rz_dmac(chan->device);
 439         struct rz_lmdesc *lmdesc = channel->lmdesc.base;
 440         struct rz_dmac_desc *desc, *_desc;
 441         unsigned long flags;
 442         unsigned int i;
 443
 444         spin_lock_irqsave(&channel->vc.lock, flags);
 445
 446         for (i = 0; i < DMAC_NR_LMDESC; i++)
 447                 lmdesc[i].header = 0;
 448
 449         rz_dmac_disable_hw(channel);
 450         list_splice_tail_init(&channel->ld_active, &channel->ld_free);
 451         list_splice_tail_init(&channel->ld_queue, &channel->ld_free);
 452
 453         if (channel->mid_rid >= 0) {
 454                 clear_bit(channel->mid_rid, dmac->modules);
 455                 channel->mid_rid = -EINVAL;
 456         }
 457
 458         spin_unlock_irqrestore(&channel->vc.lock, flags);
 459
 460         list_for_each_entry_safe(desc, _desc, &channel->ld_free, node) {
 461                 kfree(desc);
 462                 channel->descs_allocated--;
 463         }
 464
 465         INIT_LIST_HEAD(&channel->ld_free);
 466         vchan_free_chan_resources(&channel->vc);
 467 }
 468
 469 static struct dma_async_tx_descriptor *
 470 rz_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 471                         size_t len, unsigned long flags)
 472 {
 473         struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
 474         struct rz_dmac *dmac = to_rz_dmac(chan->device);
 475         struct rz_dmac_desc *desc;
 476
 477         dev_dbg(dmac->dev, "%s channel: %d src=0x%pad dst=0x%pad len=%zu\n",
 478                 __func__, channel->index, &src, &dest, len);
 479
 480         if (list_empty(&channel->ld_free))
 481                 return NULL;
 482
 483         desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node);
 484
 485         desc->type = RZ_DMAC_DESC_MEMCPY;
 486         desc->src = src;
 487         desc->dest = dest;
 488         desc->len = len;
 489         desc->direction = DMA_MEM_TO_MEM;
 490
 491         list_move_tail(channel->ld_free.next, &channel->ld_queue);
 492         return vchan_tx_prep(&channel->vc, &desc->vd, flags);
 493 }
 494
 495 static struct dma_async_tx_descriptor *
 496 rz_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 497                       unsigned int sg_len,
 498                       enum dma_transfer_direction direction,
 499                       unsigned long flags, void *context)
 500 {
 501         struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
 502         struct rz_dmac_desc *desc;
 503         struct scatterlist *sg;
 504         int dma_length = 0;
 505         int i = 0;
 506
 507         if (list_empty(&channel->ld_free))
 508                 return NULL;
 509
 510         desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node);
 511
 512         for_each_sg(sgl, sg, sg_len, i) {
 513                 dma_length += sg_dma_len(sg);
 514         }
 515
 516         desc->type = RZ_DMAC_DESC_SLAVE_SG;
 517         desc->sg = sgl;
 518         desc->sgcount = sg_len;
 519         desc->len = dma_length;
 520         desc->direction = direction;
 521
 522         if (direction == DMA_DEV_TO_MEM)
 523                 desc->src = channel->src_per_address;
 524         else
 525                 desc->dest = channel->dst_per_address;
 526
 527         list_move_tail(channel->ld_free.next, &channel->ld_queue);
 528         return vchan_tx_prep(&channel->vc, &desc->vd, flags);
 529 }
 530
 531 static int rz_dmac_terminate_all(struct dma_chan *chan)
 532 {
 533         struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
 534         unsigned long flags;
 535         LIST_HEAD(head);
 536
 537         rz_dmac_disable_hw(channel);
 538         spin_lock_irqsave(&channel->vc.lock, flags);
 539         list_splice_tail_init(&channel->ld_active, &channel->ld_free);
 540         list_splice_tail_init(&channel->ld_queue, &channel->ld_free);
 541         spin_unlock_irqrestore(&channel->vc.lock, flags);
 542         vchan_get_all_descriptors(&channel->vc, &head);
 543         vchan_dma_desc_free_list(&channel->vc, &head);
 544
 545         return 0;
 546 }
 547
 548 static void rz_dmac_issue_pending(struct dma_chan *chan)
 549 {
 550         struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
 551         struct rz_dmac *dmac = to_rz_dmac(chan->device);
 552         struct rz_dmac_desc *desc;
 553         unsigned long flags;
 554
 555         spin_lock_irqsave(&channel->vc.lock, flags);
 556
 557         if (!list_empty(&channel->ld_queue)) {
 558                 desc = list_first_entry(&channel->ld_queue,
 559                                         struct rz_dmac_desc, node);
 560                 channel->desc = desc;
 561                 if (vchan_issue_pending(&channel->vc)) {
 562                         if (rz_dmac_xfer_desc(channel) < 0)
 563                                 dev_warn(dmac->dev, "ch: %d couldn't issue DMA xfer\n",
 564                                          channel->index);
 565                         else
 566                                 list_move_tail(channel->ld_queue.next,
 567                                                &channel->ld_active);
 568                 }
 569         }
 570
 571         spin_unlock_irqrestore(&channel->vc.lock, flags);
 572 }
 573
 574 static u8 rz_dmac_ds_to_val_mapping(enum dma_slave_buswidth ds)
 575 {
 576         u8 i;
 577         static const enum dma_slave_buswidth ds_lut[] = {
 578                 DMA_SLAVE_BUSWIDTH_1_BYTE,
 579                 DMA_SLAVE_BUSWIDTH_2_BYTES,
 580                 DMA_SLAVE_BUSWIDTH_4_BYTES,
 581                 DMA_SLAVE_BUSWIDTH_8_BYTES,
 582                 DMA_SLAVE_BUSWIDTH_16_BYTES,
 583                 DMA_SLAVE_BUSWIDTH_32_BYTES,
 584                 DMA_SLAVE_BUSWIDTH_64_BYTES,
 585                 DMA_SLAVE_BUSWIDTH_128_BYTES,
 586         };
 587
 588         for (i = 0; i < ARRAY_SIZE(ds_lut); i++) {
 589                 if (ds_lut[i] == ds)
 590                         return i;
 591         }
 592
 593         return CHCFG_DS_INVALID;
 594 }
 595
 596 static int rz_dmac_config(struct dma_chan *chan,
 597                           struct dma_slave_config *config)
 598 {
 599         struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
 600         u32 val;
 601
 602         channel->src_per_address = config->src_addr;
 603         channel->src_word_size = config->src_addr_width;
 604         channel->dst_per_address = config->dst_addr;
 605         channel->dst_word_size = config->dst_addr_width;
 606
 607         val = rz_dmac_ds_to_val_mapping(config->dst_addr_width);
 608         if (val == CHCFG_DS_INVALID)
 609                 return -EINVAL;
 610
 611         channel->chcfg |= CHCFG_FILL_DDS(val);
 612
 613         val = rz_dmac_ds_to_val_mapping(config->src_addr_width);
 614         if (val == CHCFG_DS_INVALID)
 615                 return -EINVAL;
 616
 617         channel->chcfg |= CHCFG_FILL_SDS(val);
 618
 619         return 0;
 620 }
 621
 622 static void rz_dmac_virt_desc_free(struct virt_dma_desc *vd)
 623 {
 624         /*
 625          * Place holder
 626          * Descriptor allocation is done during alloc_chan_resources and
 627          * get freed during free_chan_resources.
 628          * list is used to manage the descriptors and avoid any memory
 629          * allocation/free during DMA read/write.
 630          */
 631 }
 632
 633 /*
 634  * -----------------------------------------------------------------------------
 635  * IRQ handling
 636  */
 637
 638 static void rz_dmac_irq_handle_channel(struct rz_dmac_chan *channel)
 639 {
 640         struct dma_chan *chan = &channel->vc.chan;
 641         struct rz_dmac *dmac = to_rz_dmac(chan->device);
 642         u32 chstat, chctrl;
 643
 644         chstat = rz_dmac_ch_readl(channel, CHSTAT, 1);
 645         if (chstat & CHSTAT_ER) {
 646                 dev_err(dmac->dev, "DMAC err CHSTAT_%d = %08X\n",
 647                         channel->index, chstat);
 648                 rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
 649                 goto done;
 650         }
 651
 652         chctrl = rz_dmac_ch_readl(channel, CHCTRL, 1);
 653         rz_dmac_ch_writel(channel, chctrl | CHCTRL_CLREND, CHCTRL, 1);
 654 done:
 655         return;
 656 }
 657
 658 static irqreturn_t rz_dmac_irq_handler(int irq, void *dev_id)
 659 {
 660         struct rz_dmac_chan *channel = dev_id;
 661
 662         if (channel) {
 663                 rz_dmac_irq_handle_channel(channel);
 664                 return IRQ_WAKE_THREAD;
 665         }
 666         /* handle DMAERR irq */
 667         return IRQ_HANDLED;
 668 }
 669
 670 static irqreturn_t rz_dmac_irq_handler_thread(int irq, void *dev_id)
 671 {
 672         struct rz_dmac_chan *channel = dev_id;
 673         struct rz_dmac_desc *desc = NULL;
 674         unsigned long flags;
 675
 676         spin_lock_irqsave(&channel->vc.lock, flags);
 677
 678         if (list_empty(&channel->ld_active)) {
 679                 /* Someone might have called terminate all */
 680                 goto out;
 681         }
 682
 683         desc = list_first_entry(&channel->ld_active, struct rz_dmac_desc, node);
 684         vchan_cookie_complete(&desc->vd);
 685         list_move_tail(channel->ld_active.next, &channel->ld_free);
 686         if (!list_empty(&channel->ld_queue)) {
 687                 desc = list_first_entry(&channel->ld_queue, struct rz_dmac_desc,
 688                                         node);
 689                 channel->desc = desc;
 690                 if (rz_dmac_xfer_desc(channel) == 0)
 691                         list_move_tail(channel->ld_queue.next, &channel->ld_active);
 692         }
 693 out:
 694         spin_unlock_irqrestore(&channel->vc.lock, flags);
 695
 696         return IRQ_HANDLED;
 697 }
 698
 699 /*
 700  * -----------------------------------------------------------------------------
 701  * OF xlate and channel filter
 702  */
 703
 704 static bool rz_dmac_chan_filter(struct dma_chan *chan, void *arg)
 705 {
 706         struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
 707         struct rz_dmac *dmac = to_rz_dmac(chan->device);
 708         struct of_phandle_args *dma_spec = arg;
 709         u32 ch_cfg;
 710
 711         channel->mid_rid = dma_spec->args[0] & MID_RID_MASK;
 712         ch_cfg = (dma_spec->args[0] & CHCFG_MASK) >> 10;
 713         channel->chcfg = CHCFG_FILL_TM(ch_cfg) | CHCFG_FILL_AM(ch_cfg) |
 714                          CHCFG_FILL_LVL(ch_cfg) | CHCFG_FILL_HIEN(ch_cfg);
 715
 716         return !test_and_set_bit(channel->mid_rid, dmac->modules);
 717 }
 718
 719 static struct dma_chan *rz_dmac_of_xlate(struct of_phandle_args *dma_spec,
 720                                          struct of_dma *ofdma)
 721 {
 722         dma_cap_mask_t mask;
 723
 724         if (dma_spec->args_count != 1)
 725                 return NULL;
 726
 727         /* Only slave DMA channels can be allocated via DT */
 728         dma_cap_zero(mask);
 729         dma_cap_set(DMA_SLAVE, mask);
 730
 731         return dma_request_channel(mask, rz_dmac_chan_filter, dma_spec);
 732 }
 733
 734 /*
 735  * -----------------------------------------------------------------------------
 736  * Probe and remove
 737  */
 738
 739 static int rz_dmac_chan_probe(struct rz_dmac *dmac,
 740                               struct rz_dmac_chan *channel,
 741                               unsigned int index)
 742 {
 743         struct platform_device *pdev = to_platform_device(dmac->dev);
 744         struct rz_lmdesc *lmdesc;
 745         char pdev_irqname[5];
 746         char *irqname;
 747         int ret;
 748
 749         channel->index = index;
 750         channel->mid_rid = -EINVAL;
 751
 752         /* Request the channel interrupt. */
 753         sprintf(pdev_irqname, "ch%u", index);
 754         channel->irq = platform_get_irq_byname(pdev, pdev_irqname);
 755         if (channel->irq < 0)
 756                 return channel->irq;
 757
 758         irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u",
 759                                  dev_name(dmac->dev), index);
 760         if (!irqname)
 761                 return -ENOMEM;
 762
 763         ret = devm_request_threaded_irq(dmac->dev, channel->irq,
 764                                         rz_dmac_irq_handler,
 765                                         rz_dmac_irq_handler_thread, 0,
 766                                         irqname, channel);
 767         if (ret) {
 768                 dev_err(dmac->dev, "failed to request IRQ %u (%d)\n",
 769                         channel->irq, ret);
 770                 return ret;
 771         }
 772
 773         /* Set io base address for each channel */
 774         if (index < 8) {
 775                 channel->ch_base = dmac->base + CHANNEL_0_7_OFFSET +
 776                         EACH_CHANNEL_OFFSET * index;
 777                 channel->ch_cmn_base = dmac->base + CHANNEL_0_7_COMMON_BASE;
 778         } else {
 779                 channel->ch_base = dmac->base + CHANNEL_8_15_OFFSET +
 780                         EACH_CHANNEL_OFFSET * (index - 8);
 781                 channel->ch_cmn_base = dmac->base + CHANNEL_8_15_COMMON_BASE;
 782         }
 783
 784         /* Allocate descriptors */
 785         lmdesc = dma_alloc_coherent(&pdev->dev,
 786                                     sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
 787                                     &channel->lmdesc.base_dma, GFP_KERNEL);
 788         if (!lmdesc) {
 789                 dev_err(&pdev->dev, "Can't allocate memory (lmdesc)\n");
 790                 return -ENOMEM;
 791         }
 792         rz_lmdesc_setup(channel, lmdesc);
 793
 794         /* Initialize register for each channel */
 795         rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
 796
 797         channel->vc.desc_free = rz_dmac_virt_desc_free;
 798         vchan_init(&channel->vc, &dmac->engine);
 799         INIT_LIST_HEAD(&channel->ld_queue);
 800         INIT_LIST_HEAD(&channel->ld_free);
 801         INIT_LIST_HEAD(&channel->ld_active);
 802
 803         return 0;
 804 }
 805
 806 static int rz_dmac_parse_of(struct device *dev, struct rz_dmac *dmac)
 807 {
 808         struct device_node *np = dev->of_node;
 809         int ret;
 810
 811         ret = of_property_read_u32(np, "dma-channels", &dmac->n_channels);
 812         if (ret < 0) {
 813                 dev_err(dev, "unable to read dma-channels property\n");
 814                 return ret;
 815         }
 816
 817         if (!dmac->n_channels || dmac->n_channels > RZ_DMAC_MAX_CHANNELS) {
 818                 dev_err(dev, "invalid number of channels %u\n", dmac->n_channels);
 819                 return -EINVAL;
 820         }
 821
 822         return 0;
 823 }
 824
 825 static int rz_dmac_probe(struct platform_device *pdev)
 826 {
 827         const char *irqname = "error";
 828         struct dma_device *engine;
 829         struct rz_dmac *dmac;
 830         int channel_num;
 831         unsigned int i;
 832         int ret;
 833         int irq;
 834
 835         dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL);
 836         if (!dmac)
 837                 return -ENOMEM;
 838
 839         dmac->dev = &pdev->dev;
 840         platform_set_drvdata(pdev, dmac);
 841
 842         ret = rz_dmac_parse_of(&pdev->dev, dmac);
 843         if (ret < 0)
 844                 return ret;
 845
 846         dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels,
 847                                       sizeof(*dmac->channels), GFP_KERNEL);
 848         if (!dmac->channels)
 849                 return -ENOMEM;
 850
 851         /* Request resources */
 852         dmac->base = devm_platform_ioremap_resource(pdev, 0);
 853         if (IS_ERR(dmac->base))
 854                 return PTR_ERR(dmac->base);
 855
 856         dmac->ext_base = devm_platform_ioremap_resource(pdev, 1);
 857         if (IS_ERR(dmac->ext_base))
 858                 return PTR_ERR(dmac->ext_base);
 859
 860         /* Register interrupt handler for error */
 861         irq = platform_get_irq_byname(pdev, irqname);
 862         if (irq < 0)
 863                 return irq;
 864
 865         ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0,
 866                                irqname, NULL);
 867         if (ret) {
 868                 dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n",
 869                         irq, ret);
 870                 return ret;
 871         }
 872
 873         /* Initialize the channels. */
 874         INIT_LIST_HEAD(&dmac->engine.channels);
 875
 876         pm_runtime_enable(&pdev->dev);
 877         ret = pm_runtime_resume_and_get(&pdev->dev);
 878         if (ret < 0) {
 879                 dev_err(&pdev->dev, "pm_runtime_resume_and_get failed\n");
 880                 goto err_pm_disable;
 881         }
 882
 883         for (i = 0; i < dmac->n_channels; i++) {
 884                 ret = rz_dmac_chan_probe(dmac, &dmac->channels[i], i);
 885                 if (ret < 0)
 886                         goto err;
 887         }
 888
 889         /* Register the DMAC as a DMA provider for DT. */
 890         ret = of_dma_controller_register(pdev->dev.of_node, rz_dmac_of_xlate,
 891                                          NULL);
 892         if (ret < 0)
 893                 goto err;
 894
 895         /* Register the DMA engine device. */
 896         engine = &dmac->engine;
 897         dma_cap_set(DMA_SLAVE, engine->cap_mask);
 898         dma_cap_set(DMA_MEMCPY, engine->cap_mask);
 899         rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_0_7_COMMON_BASE + DCTRL);
 900         rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_8_15_COMMON_BASE + DCTRL);
 901
 902         engine->dev = &pdev->dev;
 903
 904         engine->device_alloc_chan_resources = rz_dmac_alloc_chan_resources;
 905         engine->device_free_chan_resources = rz_dmac_free_chan_resources;
 906         engine->device_tx_status = dma_cookie_status;
 907         engine->device_prep_slave_sg = rz_dmac_prep_slave_sg;
 908         engine->device_prep_dma_memcpy = rz_dmac_prep_dma_memcpy;
 909         engine->device_config = rz_dmac_config;
 910         engine->device_terminate_all = rz_dmac_terminate_all;
 911         engine->device_issue_pending = rz_dmac_issue_pending;
 912
 913         engine->copy_align = DMAENGINE_ALIGN_1_BYTE;
 914         dma_set_max_seg_size(engine->dev, U32_MAX);
 915
 916         ret = dma_async_device_register(engine);
 917         if (ret < 0) {
 918                 dev_err(&pdev->dev, "unable to register\n");
 919                 goto dma_register_err;
 920         }
 921         return 0;
 922
 923 dma_register_err:
 924         of_dma_controller_free(pdev->dev.of_node);
 925 err:
 926         channel_num = i ? i - 1 : 0;
 927         for (i = 0; i < channel_num; i++) {
 928                 struct rz_dmac_chan *channel = &dmac->channels[i];
 929
 930                 dma_free_coherent(&pdev->dev,
 931                                   sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
 932                                   channel->lmdesc.base,
 933                                   channel->lmdesc.base_dma);
 934         }
 935
 936         pm_runtime_put(&pdev->dev);
 937 err_pm_disable:
 938         pm_runtime_disable(&pdev->dev);
 939
 940         return ret;
 941 }
 942
 943 static int rz_dmac_remove(struct platform_device *pdev)
 944 {
 945         struct rz_dmac *dmac = platform_get_drvdata(pdev);
 946         unsigned int i;
 947
 948         for (i = 0; i < dmac->n_channels; i++) {
 949                 struct rz_dmac_chan *channel = &dmac->channels[i];
 950
 951                 dma_free_coherent(&pdev->dev,
 952                                   sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
 953                                   channel->lmdesc.base,
 954                                   channel->lmdesc.base_dma);
 955         }
 956         of_dma_controller_free(pdev->dev.of_node);
 957         dma_async_device_unregister(&dmac->engine);
 958         pm_runtime_put(&pdev->dev);
 959         pm_runtime_disable(&pdev->dev);
 960
 961         return 0;
 962 }
 963
 964 static const struct of_device_id of_rz_dmac_match[] = {
 965         { .compatible = "renesas,rz-dmac", },
 966         { /* Sentinel */ }
 967 };
 968 MODULE_DEVICE_TABLE(of, of_rz_dmac_match);
 969
 970 static struct platform_driver rz_dmac_driver = {
 971         .driver         = {
 972                 .name   = "rz-dmac",
 973                 .of_match_table = of_rz_dmac_match,
 974         },
 975         .probe          = rz_dmac_probe,
 976         .remove         = rz_dmac_remove,
 977 };
 978
 979 module_platform_driver(rz_dmac_driver);
 980
 981 MODULE_DESCRIPTION("Renesas RZ/G2L DMA Controller Driver");
 982 MODULE_AUTHOR("Biju Das <[email protected]>");
 983 MODULE_LICENSE("GPL v2");