drivers/dma/sh/rz-dmac.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Renesas RZ/G2L DMA Controller Driver
   4  *
   5  * Based on imx-dma.c
   6  *
   7  * Copyright (C) 2021 Renesas Electronics Corp.
   8  * Copyright 2010 Sascha Hauer, Pengutronix <[email protected]>
   9  * Copyright 2012 Javier Martin, Vista Silicon <[email protected]>
  10  */
  11
  12 #include <linux/dma-mapping.h>
  13 #include <linux/dmaengine.h>
  14 #include <linux/interrupt.h>
  15 #include <linux/iopoll.h>
  16 #include <linux/list.h>
  17 #include <linux/module.h>
  18 #include <linux/of.h>
  19 #include <linux/of_dma.h>
  20 #include <linux/of_platform.h>
  21 #include <linux/platform_device.h>
  22 #include <linux/pm_runtime.h>
  23 #include <linux/reset.h>
  24 #include <linux/slab.h>
  25 #include <linux/spinlock.h>
  26
  27 #include "../dmaengine.h"
  28 #include "../virt-dma.h"
  29
  30 enum  rz_dmac_prep_type {
  31         RZ_DMAC_DESC_MEMCPY,
  32         RZ_DMAC_DESC_SLAVE_SG,
  33 };
  34
  35 struct rz_lmdesc {
  36         u32 header;
  37         u32 sa;
  38         u32 da;
  39         u32 tb;
  40         u32 chcfg;
  41         u32 chitvl;
  42         u32 chext;
  43         u32 nxla;
  44 };
  45
  46 struct rz_dmac_desc {
  47         struct virt_dma_desc vd;
  48         dma_addr_t src;
  49         dma_addr_t dest;
  50         size_t len;
  51         struct list_head node;
  52         enum dma_transfer_direction direction;
  53         enum rz_dmac_prep_type type;
  54         /* For slave sg */
  55         struct scatterlist *sg;
  56         unsigned int sgcount;
  57 };
  58
  59 #define to_rz_dmac_desc(d)      container_of(d, struct rz_dmac_desc, vd)
  60
  61 struct rz_dmac_chan {
  62         struct virt_dma_chan vc;
  63         void __iomem *ch_base;
  64         void __iomem *ch_cmn_base;
  65         unsigned int index;
  66         int irq;
  67         struct rz_dmac_desc *desc;
  68         int descs_allocated;
  69
  70         dma_addr_t src_per_address;
  71         dma_addr_t dst_per_address;
  72
  73         u32 chcfg;
  74         u32 chctrl;
  75         int mid_rid;
  76
  77         struct list_head ld_free;
  78         struct list_head ld_queue;
  79         struct list_head ld_active;
  80
  81         struct {
  82                 struct rz_lmdesc *base;
  83                 struct rz_lmdesc *head;
  84                 struct rz_lmdesc *tail;
  85                 dma_addr_t base_dma;
  86         } lmdesc;
  87 };
  88
  89 #define to_rz_dmac_chan(c)      container_of(c, struct rz_dmac_chan, vc.chan)
  90
  91 struct rz_dmac {
  92         struct dma_device engine;
  93         struct device *dev;
  94         struct reset_control *rstc;
  95         void __iomem *base;
  96         void __iomem *ext_base;
  97
  98         unsigned int n_channels;
  99         struct rz_dmac_chan *channels;
 100
 101         DECLARE_BITMAP(modules, 1024);
 102 };
 103
 104 #define to_rz_dmac(d)   container_of(d, struct rz_dmac, engine)
 105
 106 /*
 107  * -----------------------------------------------------------------------------
 108  * Registers
 109  */
 110
 111 #define CHSTAT                          0x0024
 112 #define CHCTRL                          0x0028
 113 #define CHCFG                           0x002c
 114 #define NXLA                            0x0038
 115
 116 #define DCTRL                           0x0000
 117
 118 #define EACH_CHANNEL_OFFSET             0x0040
 119 #define CHANNEL_0_7_OFFSET              0x0000
 120 #define CHANNEL_0_7_COMMON_BASE         0x0300
 121 #define CHANNEL_8_15_OFFSET             0x0400
 122 #define CHANNEL_8_15_COMMON_BASE        0x0700
 123
 124 #define CHSTAT_ER                       BIT(4)
 125 #define CHSTAT_EN                       BIT(0)
 126
 127 #define CHCTRL_CLRINTMSK                BIT(17)
 128 #define CHCTRL_CLRSUS                   BIT(9)
 129 #define CHCTRL_CLRTC                    BIT(6)
 130 #define CHCTRL_CLREND                   BIT(5)
 131 #define CHCTRL_CLRRQ                    BIT(4)
 132 #define CHCTRL_SWRST                    BIT(3)
 133 #define CHCTRL_STG                      BIT(2)
 134 #define CHCTRL_CLREN                    BIT(1)
 135 #define CHCTRL_SETEN                    BIT(0)
 136 #define CHCTRL_DEFAULT                  (CHCTRL_CLRINTMSK | CHCTRL_CLRSUS | \
 137                                          CHCTRL_CLRTC | CHCTRL_CLREND | \
 138                                          CHCTRL_CLRRQ | CHCTRL_SWRST | \
 139                                          CHCTRL_CLREN)
 140
 141 #define CHCFG_DMS                       BIT(31)
 142 #define CHCFG_DEM                       BIT(24)
 143 #define CHCFG_DAD                       BIT(21)
 144 #define CHCFG_SAD                       BIT(20)
 145 #define CHCFG_REQD                      BIT(3)
 146 #define CHCFG_SEL(bits)                 ((bits) & 0x07)
 147 #define CHCFG_MEM_COPY                  (0x80400008)
 148 #define CHCFG_FILL_DDS(a)               (((a) << 16) & GENMASK(19, 16))
 149 #define CHCFG_FILL_SDS(a)               (((a) << 12) & GENMASK(15, 12))
 150 #define CHCFG_FILL_TM(a)                (((a) & BIT(5)) << 22)
 151 #define CHCFG_FILL_AM(a)                (((a) & GENMASK(4, 2)) << 6)
 152 #define CHCFG_FILL_LVL(a)               (((a) & BIT(1)) << 5)
 153 #define CHCFG_FILL_HIEN(a)              (((a) & BIT(0)) << 5)
 154
 155 #define MID_RID_MASK                    GENMASK(9, 0)
 156 #define CHCFG_MASK                      GENMASK(15, 10)
 157 #define CHCFG_DS_INVALID                0xFF
 158 #define DCTRL_LVINT                     BIT(1)
 159 #define DCTRL_PR                        BIT(0)
 160 #define DCTRL_DEFAULT                   (DCTRL_LVINT | DCTRL_PR)
 161
 162 /* LINK MODE DESCRIPTOR */
 163 #define HEADER_LV                       BIT(0)
 164
 165 #define RZ_DMAC_MAX_CHAN_DESCRIPTORS    16
 166 #define RZ_DMAC_MAX_CHANNELS            16
 167 #define DMAC_NR_LMDESC                  64
 168
 169 /*
 170  * -----------------------------------------------------------------------------
 171  * Device access
 172  */
 173
 174 static void rz_dmac_writel(struct rz_dmac *dmac, unsigned int val,
 175                            unsigned int offset)
 176 {
 177         writel(val, dmac->base + offset);
 178 }
 179
 180 static void rz_dmac_ext_writel(struct rz_dmac *dmac, unsigned int val,
 181                                unsigned int offset)
 182 {
 183         writel(val, dmac->ext_base + offset);
 184 }
 185
 186 static u32 rz_dmac_ext_readl(struct rz_dmac *dmac, unsigned int offset)
 187 {
 188         return readl(dmac->ext_base + offset);
 189 }
 190
 191 static void rz_dmac_ch_writel(struct rz_dmac_chan *channel, unsigned int val,
 192                               unsigned int offset, int which)
 193 {
 194         if (which)
 195                 writel(val, channel->ch_base + offset);
 196         else
 197                 writel(val, channel->ch_cmn_base + offset);
 198 }
 199
 200 static u32 rz_dmac_ch_readl(struct rz_dmac_chan *channel,
 201                             unsigned int offset, int which)
 202 {
 203         if (which)
 204                 return readl(channel->ch_base + offset);
 205         else
 206                 return readl(channel->ch_cmn_base + offset);
 207 }
 208
 209 /*
 210  * -----------------------------------------------------------------------------
 211  * Initialization
 212  */
 213
 214 static void rz_lmdesc_setup(struct rz_dmac_chan *channel,
 215                             struct rz_lmdesc *lmdesc)
 216 {
 217         u32 nxla;
 218
 219         channel->lmdesc.base = lmdesc;
 220         channel->lmdesc.head = lmdesc;
 221         channel->lmdesc.tail = lmdesc;
 222         nxla = channel->lmdesc.base_dma;
 223         while (lmdesc < (channel->lmdesc.base + (DMAC_NR_LMDESC - 1))) {
 224                 lmdesc->header = 0;
 225                 nxla += sizeof(*lmdesc);
 226                 lmdesc->nxla = nxla;
 227                 lmdesc++;
 228         }
 229
 230         lmdesc->header = 0;
 231         lmdesc->nxla = channel->lmdesc.base_dma;
 232 }
 233
 234 /*
 235  * -----------------------------------------------------------------------------
 236  * Descriptors preparation
 237  */
 238
 239 static void rz_dmac_lmdesc_recycle(struct rz_dmac_chan *channel)
 240 {
 241         struct rz_lmdesc *lmdesc = channel->lmdesc.head;
 242
 243         while (!(lmdesc->header & HEADER_LV)) {
 244                 lmdesc->header = 0;
 245                 lmdesc++;
 246                 if (lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC))
 247                         lmdesc = channel->lmdesc.base;
 248         }
 249         channel->lmdesc.head = lmdesc;
 250 }
 251
 252 static void rz_dmac_enable_hw(struct rz_dmac_chan *channel)
 253 {
 254         struct dma_chan *chan = &channel->vc.chan;
 255         struct rz_dmac *dmac = to_rz_dmac(chan->device);
 256         unsigned long flags;
 257         u32 nxla;
 258         u32 chctrl;
 259         u32 chstat;
 260
 261         dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index);
 262
 263         local_irq_save(flags);
 264
 265         rz_dmac_lmdesc_recycle(channel);
 266
 267         nxla = channel->lmdesc.base_dma +
 268                 (sizeof(struct rz_lmdesc) * (channel->lmdesc.head -
 269                                              channel->lmdesc.base));
 270
 271         chstat = rz_dmac_ch_readl(channel, CHSTAT, 1);
 272         if (!(chstat & CHSTAT_EN)) {
 273                 chctrl = (channel->chctrl | CHCTRL_SETEN);
 274                 rz_dmac_ch_writel(channel, nxla, NXLA, 1);
 275                 rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1);
 276                 rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1);
 277                 rz_dmac_ch_writel(channel, chctrl, CHCTRL, 1);
 278         }
 279
 280         local_irq_restore(flags);
 281 }
 282
 283 static void rz_dmac_disable_hw(struct rz_dmac_chan *channel)
 284 {
 285         struct dma_chan *chan = &channel->vc.chan;
 286         struct rz_dmac *dmac = to_rz_dmac(chan->device);
 287         unsigned long flags;
 288
 289         dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index);
 290
 291         local_irq_save(flags);
 292         rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
 293         local_irq_restore(flags);
 294 }
 295
 296 static void rz_dmac_set_dmars_register(struct rz_dmac *dmac, int nr, u32 dmars)
 297 {
 298         u32 dmars_offset = (nr / 2) * 4;
 299         u32 shift = (nr % 2) * 16;
 300         u32 dmars32;
 301
 302         dmars32 = rz_dmac_ext_readl(dmac, dmars_offset);
 303         dmars32 &= ~(0xffff << shift);
 304         dmars32 |= dmars << shift;
 305
 306         rz_dmac_ext_writel(dmac, dmars32, dmars_offset);
 307 }
 308
 309 static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel)
 310 {
 311         struct dma_chan *chan = &channel->vc.chan;
 312         struct rz_dmac *dmac = to_rz_dmac(chan->device);
 313         struct rz_lmdesc *lmdesc = channel->lmdesc.tail;
 314         struct rz_dmac_desc *d = channel->desc;
 315         u32 chcfg = CHCFG_MEM_COPY;
 316
 317         /* prepare descriptor */
 318         lmdesc->sa = d->src;
 319         lmdesc->da = d->dest;
 320         lmdesc->tb = d->len;
 321         lmdesc->chcfg = chcfg;
 322         lmdesc->chitvl = 0;
 323         lmdesc->chext = 0;
 324         lmdesc->header = HEADER_LV;
 325
 326         rz_dmac_set_dmars_register(dmac, channel->index, 0);
 327
 328         channel->chcfg = chcfg;
 329         channel->chctrl = CHCTRL_STG | CHCTRL_SETEN;
 330 }
 331
 332 static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel)
 333 {
 334         struct dma_chan *chan = &channel->vc.chan;
 335         struct rz_dmac *dmac = to_rz_dmac(chan->device);
 336         struct rz_dmac_desc *d = channel->desc;
 337         struct scatterlist *sg, *sgl = d->sg;
 338         struct rz_lmdesc *lmdesc;
 339         unsigned int i, sg_len = d->sgcount;
 340
 341         channel->chcfg |= CHCFG_SEL(channel->index) | CHCFG_DEM | CHCFG_DMS;
 342
 343         if (d->direction == DMA_DEV_TO_MEM) {
 344                 channel->chcfg |= CHCFG_SAD;
 345                 channel->chcfg &= ~CHCFG_REQD;
 346         } else {
 347                 channel->chcfg |= CHCFG_DAD | CHCFG_REQD;
 348         }
 349
 350         lmdesc = channel->lmdesc.tail;
 351
 352         for (i = 0, sg = sgl; i < sg_len; i++, sg = sg_next(sg)) {
 353                 if (d->direction == DMA_DEV_TO_MEM) {
 354                         lmdesc->sa = channel->src_per_address;
 355                         lmdesc->da = sg_dma_address(sg);
 356                 } else {
 357                         lmdesc->sa = sg_dma_address(sg);
 358                         lmdesc->da = channel->dst_per_address;
 359                 }
 360
 361                 lmdesc->tb = sg_dma_len(sg);
 362                 lmdesc->chitvl = 0;
 363                 lmdesc->chext = 0;
 364                 if (i == (sg_len - 1)) {
 365                         lmdesc->chcfg = (channel->chcfg & ~CHCFG_DEM);
 366                         lmdesc->header = HEADER_LV;
 367                 } else {
 368                         lmdesc->chcfg = channel->chcfg;
 369                         lmdesc->header = HEADER_LV;
 370                 }
 371                 if (++lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC))
 372                         lmdesc = channel->lmdesc.base;
 373         }
 374
 375         channel->lmdesc.tail = lmdesc;
 376
 377         rz_dmac_set_dmars_register(dmac, channel->index, channel->mid_rid);
 378         channel->chctrl = CHCTRL_SETEN;
 379 }
 380
 381 static int rz_dmac_xfer_desc(struct rz_dmac_chan *chan)
 382 {
 383         struct rz_dmac_desc *d = chan->desc;
 384         struct virt_dma_desc *vd;
 385
 386         vd = vchan_next_desc(&chan->vc);
 387         if (!vd)
 388                 return 0;
 389
 390         list_del(&vd->node);
 391
 392         switch (d->type) {
 393         case RZ_DMAC_DESC_MEMCPY:
 394                 rz_dmac_prepare_desc_for_memcpy(chan);
 395                 break;
 396
 397         case RZ_DMAC_DESC_SLAVE_SG:
 398                 rz_dmac_prepare_descs_for_slave_sg(chan);
 399                 break;
 400
 401         default:
 402                 return -EINVAL;
 403         }
 404
 405         rz_dmac_enable_hw(chan);
 406
 407         return 0;
 408 }
 409
 410 /*
 411  * -----------------------------------------------------------------------------
 412  * DMA engine operations
 413  */
 414
 415 static int rz_dmac_alloc_chan_resources(struct dma_chan *chan)
 416 {
 417         struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
 418
 419         while (channel->descs_allocated < RZ_DMAC_MAX_CHAN_DESCRIPTORS) {
 420                 struct rz_dmac_desc *desc;
 421
 422                 desc = kzalloc(sizeof(*desc), GFP_KERNEL);
 423                 if (!desc)
 424                         break;
 425
 426                 list_add_tail(&desc->node, &channel->ld_free);
 427                 channel->descs_allocated++;
 428         }
 429
 430         if (!channel->descs_allocated)
 431                 return -ENOMEM;
 432
 433         return channel->descs_allocated;
 434 }
 435
 436 static void rz_dmac_free_chan_resources(struct dma_chan *chan)
 437 {
 438         struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
 439         struct rz_dmac *dmac = to_rz_dmac(chan->device);
 440         struct rz_lmdesc *lmdesc = channel->lmdesc.base;
 441         struct rz_dmac_desc *desc, *_desc;
 442         unsigned long flags;
 443         unsigned int i;
 444
 445         spin_lock_irqsave(&channel->vc.lock, flags);
 446
 447         for (i = 0; i < DMAC_NR_LMDESC; i++)
 448                 lmdesc[i].header = 0;
 449
 450         rz_dmac_disable_hw(channel);
 451         list_splice_tail_init(&channel->ld_active, &channel->ld_free);
 452         list_splice_tail_init(&channel->ld_queue, &channel->ld_free);
 453
 454         if (channel->mid_rid >= 0) {
 455                 clear_bit(channel->mid_rid, dmac->modules);
 456                 channel->mid_rid = -EINVAL;
 457         }
 458
 459         spin_unlock_irqrestore(&channel->vc.lock, flags);
 460
 461         list_for_each_entry_safe(desc, _desc, &channel->ld_free, node) {
 462                 kfree(desc);
 463                 channel->descs_allocated--;
 464         }
 465
 466         INIT_LIST_HEAD(&channel->ld_free);
 467         vchan_free_chan_resources(&channel->vc);
 468 }
 469
 470 static struct dma_async_tx_descriptor *
 471 rz_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 472                         size_t len, unsigned long flags)
 473 {
 474         struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
 475         struct rz_dmac *dmac = to_rz_dmac(chan->device);
 476         struct rz_dmac_desc *desc;
 477
 478         dev_dbg(dmac->dev, "%s channel: %d src=0x%pad dst=0x%pad len=%zu\n",
 479                 __func__, channel->index, &src, &dest, len);
 480
 481         if (list_empty(&channel->ld_free))
 482                 return NULL;
 483
 484         desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node);
 485
 486         desc->type = RZ_DMAC_DESC_MEMCPY;
 487         desc->src = src;
 488         desc->dest = dest;
 489         desc->len = len;
 490         desc->direction = DMA_MEM_TO_MEM;
 491
 492         list_move_tail(channel->ld_free.next, &channel->ld_queue);
 493         return vchan_tx_prep(&channel->vc, &desc->vd, flags);
 494 }
 495
 496 static struct dma_async_tx_descriptor *
 497 rz_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 498                       unsigned int sg_len,
 499                       enum dma_transfer_direction direction,
 500                       unsigned long flags, void *context)
 501 {
 502         struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
 503         struct rz_dmac_desc *desc;
 504         struct scatterlist *sg;
 505         int dma_length = 0;
 506         int i = 0;
 507
 508         if (list_empty(&channel->ld_free))
 509                 return NULL;
 510
 511         desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node);
 512
 513         for_each_sg(sgl, sg, sg_len, i) {
 514                 dma_length += sg_dma_len(sg);
 515         }
 516
 517         desc->type = RZ_DMAC_DESC_SLAVE_SG;
 518         desc->sg = sgl;
 519         desc->sgcount = sg_len;
 520         desc->len = dma_length;
 521         desc->direction = direction;
 522
 523         if (direction == DMA_DEV_TO_MEM)
 524                 desc->src = channel->src_per_address;
 525         else
 526                 desc->dest = channel->dst_per_address;
 527
 528         list_move_tail(channel->ld_free.next, &channel->ld_queue);
 529         return vchan_tx_prep(&channel->vc, &desc->vd, flags);
 530 }
 531
 532 static int rz_dmac_terminate_all(struct dma_chan *chan)
 533 {
 534         struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
 535         unsigned long flags;
 536         LIST_HEAD(head);
 537
 538         rz_dmac_disable_hw(channel);
 539         spin_lock_irqsave(&channel->vc.lock, flags);
 540         list_splice_tail_init(&channel->ld_active, &channel->ld_free);
 541         list_splice_tail_init(&channel->ld_queue, &channel->ld_free);
 542         spin_unlock_irqrestore(&channel->vc.lock, flags);
 543         vchan_get_all_descriptors(&channel->vc, &head);
 544         vchan_dma_desc_free_list(&channel->vc, &head);
 545
 546         return 0;
 547 }
 548
 549 static void rz_dmac_issue_pending(struct dma_chan *chan)
 550 {
 551         struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
 552         struct rz_dmac *dmac = to_rz_dmac(chan->device);
 553         struct rz_dmac_desc *desc;
 554         unsigned long flags;
 555
 556         spin_lock_irqsave(&channel->vc.lock, flags);
 557
 558         if (!list_empty(&channel->ld_queue)) {
 559                 desc = list_first_entry(&channel->ld_queue,
 560                                         struct rz_dmac_desc, node);
 561                 channel->desc = desc;
 562                 if (vchan_issue_pending(&channel->vc)) {
 563                         if (rz_dmac_xfer_desc(channel) < 0)
 564                                 dev_warn(dmac->dev, "ch: %d couldn't issue DMA xfer\n",
 565                                          channel->index);
 566                         else
 567                                 list_move_tail(channel->ld_queue.next,
 568                                                &channel->ld_active);
 569                 }
 570         }
 571
 572         spin_unlock_irqrestore(&channel->vc.lock, flags);
 573 }
 574
 575 static u8 rz_dmac_ds_to_val_mapping(enum dma_slave_buswidth ds)
 576 {
 577         u8 i;
 578         static const enum dma_slave_buswidth ds_lut[] = {
 579                 DMA_SLAVE_BUSWIDTH_1_BYTE,
 580                 DMA_SLAVE_BUSWIDTH_2_BYTES,
 581                 DMA_SLAVE_BUSWIDTH_4_BYTES,
 582                 DMA_SLAVE_BUSWIDTH_8_BYTES,
 583                 DMA_SLAVE_BUSWIDTH_16_BYTES,
 584                 DMA_SLAVE_BUSWIDTH_32_BYTES,
 585                 DMA_SLAVE_BUSWIDTH_64_BYTES,
 586                 DMA_SLAVE_BUSWIDTH_128_BYTES,
 587         };
 588
 589         for (i = 0; i < ARRAY_SIZE(ds_lut); i++) {
 590                 if (ds_lut[i] == ds)
 591                         return i;
 592         }
 593
 594         return CHCFG_DS_INVALID;
 595 }
 596
 597 static int rz_dmac_config(struct dma_chan *chan,
 598                           struct dma_slave_config *config)
 599 {
 600         struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
 601         u32 val;
 602
 603         channel->src_per_address = config->src_addr;
 604         channel->dst_per_address = config->dst_addr;
 605
 606         val = rz_dmac_ds_to_val_mapping(config->dst_addr_width);
 607         if (val == CHCFG_DS_INVALID)
 608                 return -EINVAL;
 609
 610         channel->chcfg |= CHCFG_FILL_DDS(val);
 611
 612         val = rz_dmac_ds_to_val_mapping(config->src_addr_width);
 613         if (val == CHCFG_DS_INVALID)
 614                 return -EINVAL;
 615
 616         channel->chcfg |= CHCFG_FILL_SDS(val);
 617
 618         return 0;
 619 }
 620
 621 static void rz_dmac_virt_desc_free(struct virt_dma_desc *vd)
 622 {
 623         /*
 624          * Place holder
 625          * Descriptor allocation is done during alloc_chan_resources and
 626          * get freed during free_chan_resources.
 627          * list is used to manage the descriptors and avoid any memory
 628          * allocation/free during DMA read/write.
 629          */
 630 }
 631
 632 static void rz_dmac_device_synchronize(struct dma_chan *chan)
 633 {
 634         struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
 635         struct rz_dmac *dmac = to_rz_dmac(chan->device);
 636         u32 chstat;
 637         int ret;
 638
 639         ret = read_poll_timeout(rz_dmac_ch_readl, chstat, !(chstat & CHSTAT_EN),
 640                                 100, 100000, false, channel, CHSTAT, 1);
 641         if (ret < 0)
 642                 dev_warn(dmac->dev, "DMA Timeout");
 643
 644         rz_dmac_set_dmars_register(dmac, channel->index, 0);
 645 }
 646
 647 /*
 648  * -----------------------------------------------------------------------------
 649  * IRQ handling
 650  */
 651
 652 static void rz_dmac_irq_handle_channel(struct rz_dmac_chan *channel)
 653 {
 654         struct dma_chan *chan = &channel->vc.chan;
 655         struct rz_dmac *dmac = to_rz_dmac(chan->device);
 656         u32 chstat, chctrl;
 657
 658         chstat = rz_dmac_ch_readl(channel, CHSTAT, 1);
 659         if (chstat & CHSTAT_ER) {
 660                 dev_err(dmac->dev, "DMAC err CHSTAT_%d = %08X\n",
 661                         channel->index, chstat);
 662                 rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
 663                 goto done;
 664         }
 665
 666         chctrl = rz_dmac_ch_readl(channel, CHCTRL, 1);
 667         rz_dmac_ch_writel(channel, chctrl | CHCTRL_CLREND, CHCTRL, 1);
 668 done:
 669         return;
 670 }
 671
 672 static irqreturn_t rz_dmac_irq_handler(int irq, void *dev_id)
 673 {
 674         struct rz_dmac_chan *channel = dev_id;
 675
 676         if (channel) {
 677                 rz_dmac_irq_handle_channel(channel);
 678                 return IRQ_WAKE_THREAD;
 679         }
 680         /* handle DMAERR irq */
 681         return IRQ_HANDLED;
 682 }
 683
 684 static irqreturn_t rz_dmac_irq_handler_thread(int irq, void *dev_id)
 685 {
 686         struct rz_dmac_chan *channel = dev_id;
 687         struct rz_dmac_desc *desc = NULL;
 688         unsigned long flags;
 689
 690         spin_lock_irqsave(&channel->vc.lock, flags);
 691
 692         if (list_empty(&channel->ld_active)) {
 693                 /* Someone might have called terminate all */
 694                 goto out;
 695         }
 696
 697         desc = list_first_entry(&channel->ld_active, struct rz_dmac_desc, node);
 698         vchan_cookie_complete(&desc->vd);
 699         list_move_tail(channel->ld_active.next, &channel->ld_free);
 700         if (!list_empty(&channel->ld_queue)) {
 701                 desc = list_first_entry(&channel->ld_queue, struct rz_dmac_desc,
 702                                         node);
 703                 channel->desc = desc;
 704                 if (rz_dmac_xfer_desc(channel) == 0)
 705                         list_move_tail(channel->ld_queue.next, &channel->ld_active);
 706         }
 707 out:
 708         spin_unlock_irqrestore(&channel->vc.lock, flags);
 709
 710         return IRQ_HANDLED;
 711 }
 712
 713 /*
 714  * -----------------------------------------------------------------------------
 715  * OF xlate and channel filter
 716  */
 717
 718 static bool rz_dmac_chan_filter(struct dma_chan *chan, void *arg)
 719 {
 720         struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
 721         struct rz_dmac *dmac = to_rz_dmac(chan->device);
 722         struct of_phandle_args *dma_spec = arg;
 723         u32 ch_cfg;
 724
 725         channel->mid_rid = dma_spec->args[0] & MID_RID_MASK;
 726         ch_cfg = (dma_spec->args[0] & CHCFG_MASK) >> 10;
 727         channel->chcfg = CHCFG_FILL_TM(ch_cfg) | CHCFG_FILL_AM(ch_cfg) |
 728                          CHCFG_FILL_LVL(ch_cfg) | CHCFG_FILL_HIEN(ch_cfg);
 729
 730         return !test_and_set_bit(channel->mid_rid, dmac->modules);
 731 }
 732
 733 static struct dma_chan *rz_dmac_of_xlate(struct of_phandle_args *dma_spec,
 734                                          struct of_dma *ofdma)
 735 {
 736         dma_cap_mask_t mask;
 737
 738         if (dma_spec->args_count != 1)
 739                 return NULL;
 740
 741         /* Only slave DMA channels can be allocated via DT */
 742         dma_cap_zero(mask);
 743         dma_cap_set(DMA_SLAVE, mask);
 744
 745         return dma_request_channel(mask, rz_dmac_chan_filter, dma_spec);
 746 }
 747
 748 /*
 749  * -----------------------------------------------------------------------------
 750  * Probe and remove
 751  */
 752
 753 static int rz_dmac_chan_probe(struct rz_dmac *dmac,
 754                               struct rz_dmac_chan *channel,
 755                               unsigned int index)
 756 {
 757         struct platform_device *pdev = to_platform_device(dmac->dev);
 758         struct rz_lmdesc *lmdesc;
 759         char pdev_irqname[5];
 760         char *irqname;
 761         int ret;
 762
 763         channel->index = index;
 764         channel->mid_rid = -EINVAL;
 765
 766         /* Request the channel interrupt. */
 767         sprintf(pdev_irqname, "ch%u", index);
 768         channel->irq = platform_get_irq_byname(pdev, pdev_irqname);
 769         if (channel->irq < 0)
 770                 return channel->irq;
 771
 772         irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u",
 773                                  dev_name(dmac->dev), index);
 774         if (!irqname)
 775                 return -ENOMEM;
 776
 777         ret = devm_request_threaded_irq(dmac->dev, channel->irq,
 778                                         rz_dmac_irq_handler,
 779                                         rz_dmac_irq_handler_thread, 0,
 780                                         irqname, channel);
 781         if (ret) {
 782                 dev_err(dmac->dev, "failed to request IRQ %u (%d)\n",
 783                         channel->irq, ret);
 784                 return ret;
 785         }
 786
 787         /* Set io base address for each channel */
 788         if (index < 8) {
 789                 channel->ch_base = dmac->base + CHANNEL_0_7_OFFSET +
 790                         EACH_CHANNEL_OFFSET * index;
 791                 channel->ch_cmn_base = dmac->base + CHANNEL_0_7_COMMON_BASE;
 792         } else {
 793                 channel->ch_base = dmac->base + CHANNEL_8_15_OFFSET +
 794                         EACH_CHANNEL_OFFSET * (index - 8);
 795                 channel->ch_cmn_base = dmac->base + CHANNEL_8_15_COMMON_BASE;
 796         }
 797
 798         /* Allocate descriptors */
 799         lmdesc = dma_alloc_coherent(&pdev->dev,
 800                                     sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
 801                                     &channel->lmdesc.base_dma, GFP_KERNEL);
 802         if (!lmdesc) {
 803                 dev_err(&pdev->dev, "Can't allocate memory (lmdesc)\n");
 804                 return -ENOMEM;
 805         }
 806         rz_lmdesc_setup(channel, lmdesc);
 807
 808         /* Initialize register for each channel */
 809         rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
 810
 811         channel->vc.desc_free = rz_dmac_virt_desc_free;
 812         vchan_init(&channel->vc, &dmac->engine);
 813         INIT_LIST_HEAD(&channel->ld_queue);
 814         INIT_LIST_HEAD(&channel->ld_free);
 815         INIT_LIST_HEAD(&channel->ld_active);
 816
 817         return 0;
 818 }
 819
 820 static int rz_dmac_parse_of(struct device *dev, struct rz_dmac *dmac)
 821 {
 822         struct device_node *np = dev->of_node;
 823         int ret;
 824
 825         ret = of_property_read_u32(np, "dma-channels", &dmac->n_channels);
 826         if (ret < 0) {
 827                 dev_err(dev, "unable to read dma-channels property\n");
 828                 return ret;
 829         }
 830
 831         if (!dmac->n_channels || dmac->n_channels > RZ_DMAC_MAX_CHANNELS) {
 832                 dev_err(dev, "invalid number of channels %u\n", dmac->n_channels);
 833                 return -EINVAL;
 834         }
 835
 836         return 0;
 837 }
 838
 839 static int rz_dmac_probe(struct platform_device *pdev)
 840 {
 841         const char *irqname = "error";
 842         struct dma_device *engine;
 843         struct rz_dmac *dmac;
 844         int channel_num;
 845         unsigned int i;
 846         int ret;
 847         int irq;
 848
 849         dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL);
 850         if (!dmac)
 851                 return -ENOMEM;
 852
 853         dmac->dev = &pdev->dev;
 854         platform_set_drvdata(pdev, dmac);
 855
 856         ret = rz_dmac_parse_of(&pdev->dev, dmac);
 857         if (ret < 0)
 858                 return ret;
 859
 860         dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels,
 861                                       sizeof(*dmac->channels), GFP_KERNEL);
 862         if (!dmac->channels)
 863                 return -ENOMEM;
 864
 865         /* Request resources */
 866         dmac->base = devm_platform_ioremap_resource(pdev, 0);
 867         if (IS_ERR(dmac->base))
 868                 return PTR_ERR(dmac->base);
 869
 870         dmac->ext_base = devm_platform_ioremap_resource(pdev, 1);
 871         if (IS_ERR(dmac->ext_base))
 872                 return PTR_ERR(dmac->ext_base);
 873
 874         /* Register interrupt handler for error */
 875         irq = platform_get_irq_byname(pdev, irqname);
 876         if (irq < 0)
 877                 return irq;
 878
 879         ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0,
 880                                irqname, NULL);
 881         if (ret) {
 882                 dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n",
 883                         irq, ret);
 884                 return ret;
 885         }
 886
 887         /* Initialize the channels. */
 888         INIT_LIST_HEAD(&dmac->engine.channels);
 889
 890         dmac->rstc = devm_reset_control_array_get_exclusive(&pdev->dev);
 891         if (IS_ERR(dmac->rstc))
 892                 return dev_err_probe(&pdev->dev, PTR_ERR(dmac->rstc),
 893                                      "failed to get resets\n");
 894
 895         pm_runtime_enable(&pdev->dev);
 896         ret = pm_runtime_resume_and_get(&pdev->dev);
 897         if (ret < 0) {
 898                 dev_err(&pdev->dev, "pm_runtime_resume_and_get failed\n");
 899                 goto err_pm_disable;
 900         }
 901
 902         ret = reset_control_deassert(dmac->rstc);
 903         if (ret)
 904                 goto err_pm_runtime_put;
 905
 906         for (i = 0; i < dmac->n_channels; i++) {
 907                 ret = rz_dmac_chan_probe(dmac, &dmac->channels[i], i);
 908                 if (ret < 0)
 909                         goto err;
 910         }
 911
 912         /* Register the DMAC as a DMA provider for DT. */
 913         ret = of_dma_controller_register(pdev->dev.of_node, rz_dmac_of_xlate,
 914                                          NULL);
 915         if (ret < 0)
 916                 goto err;
 917
 918         /* Register the DMA engine device. */
 919         engine = &dmac->engine;
 920         dma_cap_set(DMA_SLAVE, engine->cap_mask);
 921         dma_cap_set(DMA_MEMCPY, engine->cap_mask);
 922         rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_0_7_COMMON_BASE + DCTRL);
 923         rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_8_15_COMMON_BASE + DCTRL);
 924
 925         engine->dev = &pdev->dev;
 926
 927         engine->device_alloc_chan_resources = rz_dmac_alloc_chan_resources;
 928         engine->device_free_chan_resources = rz_dmac_free_chan_resources;
 929         engine->device_tx_status = dma_cookie_status;
 930         engine->device_prep_slave_sg = rz_dmac_prep_slave_sg;
 931         engine->device_prep_dma_memcpy = rz_dmac_prep_dma_memcpy;
 932         engine->device_config = rz_dmac_config;
 933         engine->device_terminate_all = rz_dmac_terminate_all;
 934         engine->device_issue_pending = rz_dmac_issue_pending;
 935         engine->device_synchronize = rz_dmac_device_synchronize;
 936
 937         engine->copy_align = DMAENGINE_ALIGN_1_BYTE;
 938         dma_set_max_seg_size(engine->dev, U32_MAX);
 939
 940         ret = dma_async_device_register(engine);
 941         if (ret < 0) {
 942                 dev_err(&pdev->dev, "unable to register\n");
 943                 goto dma_register_err;
 944         }
 945         return 0;
 946
 947 dma_register_err:
 948         of_dma_controller_free(pdev->dev.of_node);
 949 err:
 950         reset_control_assert(dmac->rstc);
 951         channel_num = i ? i - 1 : 0;
 952         for (i = 0; i < channel_num; i++) {
 953                 struct rz_dmac_chan *channel = &dmac->channels[i];
 954
 955                 dma_free_coherent(&pdev->dev,
 956                                   sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
 957                                   channel->lmdesc.base,
 958                                   channel->lmdesc.base_dma);
 959         }
 960
 961 err_pm_runtime_put:
 962         pm_runtime_put(&pdev->dev);
 963 err_pm_disable:
 964         pm_runtime_disable(&pdev->dev);
 965
 966         return ret;
 967 }
 968
 969 static int rz_dmac_remove(struct platform_device *pdev)
 970 {
 971         struct rz_dmac *dmac = platform_get_drvdata(pdev);
 972         unsigned int i;
 973
 974         for (i = 0; i < dmac->n_channels; i++) {
 975                 struct rz_dmac_chan *channel = &dmac->channels[i];
 976
 977                 dma_free_coherent(&pdev->dev,
 978                                   sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
 979                                   channel->lmdesc.base,
 980                                   channel->lmdesc.base_dma);
 981         }
 982         of_dma_controller_free(pdev->dev.of_node);
 983         dma_async_device_unregister(&dmac->engine);
 984         reset_control_assert(dmac->rstc);
 985         pm_runtime_put(&pdev->dev);
 986         pm_runtime_disable(&pdev->dev);
 987
 988         return 0;
 989 }
 990
 991 static const struct of_device_id of_rz_dmac_match[] = {
 992         { .compatible = "renesas,rz-dmac", },
 993         { /* Sentinel */ }
 994 };
 995 MODULE_DEVICE_TABLE(of, of_rz_dmac_match);
 996
 997 static struct platform_driver rz_dmac_driver = {
 998         .driver         = {
 999                 .name   = "rz-dmac",
1000                 .of_match_table = of_rz_dmac_match,
1001         },
1002         .probe          = rz_dmac_probe,
1003         .remove         = rz_dmac_remove,
1004 };
1005
1006 module_platform_driver(rz_dmac_driver);
1007
1008 MODULE_DESCRIPTION("Renesas RZ/G2L DMA Controller Driver");
1009 MODULE_AUTHOR("Biju Das <[email protected]>");
1010 MODULE_LICENSE("GPL v2");