drivers/dma/sf-pdma/sf-pdma.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * SiFive FU540 Platform DMA driver
   4  * Copyright (C) 2019 SiFive
   5  *
   6  * Based partially on:
   7  * - drivers/dma/fsl-edma.c
   8  * - drivers/dma/dw-edma/
   9  * - drivers/dma/pxa-dma.c
  10  *
  11  * See the following sources for further documentation:
  12  * - Chapter 12 "Platform DMA Engine (PDMA)" of
  13  *   SiFive FU540-C000 v1.0
  14  *   https://static.dev.sifive.com/FU540-C000-v1.0.pdf
  15  */
  16 #include <linux/module.h>
  17 #include <linux/device.h>
  18 #include <linux/kernel.h>
  19 #include <linux/platform_device.h>
  20 #include <linux/mod_devicetable.h>
  21 #include <linux/dma-mapping.h>
  22 #include <linux/of.h>
  23 #include <linux/slab.h>
  24
  25 #include "sf-pdma.h"
  26
  27 #ifndef readq
  28 static inline unsigned long long readq(void __iomem *addr)
  29 {
  30         return readl(addr) | (((unsigned long long)readl(addr + 4)) << 32LL);
  31 }
  32 #endif
  33
  34 #ifndef writeq
  35 static inline void writeq(unsigned long long v, void __iomem *addr)
  36 {
  37         writel(lower_32_bits(v), addr);
  38         writel(upper_32_bits(v), addr + 4);
  39 }
  40 #endif
  41
  42 static inline struct sf_pdma_chan *to_sf_pdma_chan(struct dma_chan *dchan)
  43 {
  44         return container_of(dchan, struct sf_pdma_chan, vchan.chan);
  45 }
  46
  47 static inline struct sf_pdma_desc *to_sf_pdma_desc(struct virt_dma_desc *vd)
  48 {
  49         return container_of(vd, struct sf_pdma_desc, vdesc);
  50 }
  51
  52 static struct sf_pdma_desc *sf_pdma_alloc_desc(struct sf_pdma_chan *chan)
  53 {
  54         struct sf_pdma_desc *desc;
  55         unsigned long flags;
  56
  57         spin_lock_irqsave(&chan->lock, flags);
  58
  59         if (chan->desc && !chan->desc->in_use) {
  60                 spin_unlock_irqrestore(&chan->lock, flags);
  61                 return chan->desc;
  62         }
  63
  64         spin_unlock_irqrestore(&chan->lock, flags);
  65
  66         desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
  67         if (!desc)
  68                 return NULL;
  69
  70         desc->chan = chan;
  71
  72         return desc;
  73 }
  74
  75 static void sf_pdma_fill_desc(struct sf_pdma_desc *desc,
  76                               u64 dst, u64 src, u64 size)
  77 {
  78         desc->xfer_type = PDMA_FULL_SPEED;
  79         desc->xfer_size = size;
  80         desc->dst_addr = dst;
  81         desc->src_addr = src;
  82 }
  83
  84 static void sf_pdma_disclaim_chan(struct sf_pdma_chan *chan)
  85 {
  86         struct pdma_regs *regs = &chan->regs;
  87
  88         writel(PDMA_CLEAR_CTRL, regs->ctrl);
  89 }
  90
  91 static struct dma_async_tx_descriptor *
  92 sf_pdma_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dest, dma_addr_t src,
  93                         size_t len, unsigned long flags)
  94 {
  95         struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
  96         struct sf_pdma_desc *desc;
  97
  98         if (chan && (!len || !dest || !src)) {
  99                 dev_err(chan->pdma->dma_dev.dev,
 100                         "Please check dma len, dest, src!\n");
 101                 return NULL;
 102         }
 103
 104         desc = sf_pdma_alloc_desc(chan);
 105         if (!desc)
 106                 return NULL;
 107
 108         desc->in_use = true;
 109         desc->dirn = DMA_MEM_TO_MEM;
 110         desc->async_tx = vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
 111
 112         spin_lock_irqsave(&chan->vchan.lock, flags);
 113         chan->desc = desc;
 114         sf_pdma_fill_desc(desc, dest, src, len);
 115         spin_unlock_irqrestore(&chan->vchan.lock, flags);
 116
 117         return desc->async_tx;
 118 }
 119
 120 static int sf_pdma_slave_config(struct dma_chan *dchan,
 121                                 struct dma_slave_config *cfg)
 122 {
 123         struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
 124
 125         memcpy(&chan->cfg, cfg, sizeof(*cfg));
 126
 127         return 0;
 128 }
 129
 130 static int sf_pdma_alloc_chan_resources(struct dma_chan *dchan)
 131 {
 132         struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
 133         struct pdma_regs *regs = &chan->regs;
 134
 135         dma_cookie_init(dchan);
 136         writel(PDMA_CLAIM_MASK, regs->ctrl);
 137
 138         return 0;
 139 }
 140
 141 static void sf_pdma_disable_request(struct sf_pdma_chan *chan)
 142 {
 143         struct pdma_regs *regs = &chan->regs;
 144
 145         writel(readl(regs->ctrl) & ~PDMA_RUN_MASK, regs->ctrl);
 146 }
 147
 148 static void sf_pdma_free_chan_resources(struct dma_chan *dchan)
 149 {
 150         struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
 151         unsigned long flags;
 152         LIST_HEAD(head);
 153
 154         spin_lock_irqsave(&chan->vchan.lock, flags);
 155         sf_pdma_disable_request(chan);
 156         kfree(chan->desc);
 157         chan->desc = NULL;
 158         vchan_get_all_descriptors(&chan->vchan, &head);
 159         sf_pdma_disclaim_chan(chan);
 160         spin_unlock_irqrestore(&chan->vchan.lock, flags);
 161         vchan_dma_desc_free_list(&chan->vchan, &head);
 162 }
 163
 164 static size_t sf_pdma_desc_residue(struct sf_pdma_chan *chan,
 165                                    dma_cookie_t cookie)
 166 {
 167         struct virt_dma_desc *vd = NULL;
 168         struct pdma_regs *regs = &chan->regs;
 169         unsigned long flags;
 170         u64 residue = 0;
 171         struct sf_pdma_desc *desc;
 172         struct dma_async_tx_descriptor *tx;
 173
 174         spin_lock_irqsave(&chan->vchan.lock, flags);
 175
 176         tx = &chan->desc->vdesc.tx;
 177         if (cookie == tx->chan->completed_cookie)
 178                 goto out;
 179
 180         if (cookie == tx->cookie) {
 181                 residue = readq(regs->residue);
 182         } else {
 183                 vd = vchan_find_desc(&chan->vchan, cookie);
 184                 if (!vd)
 185                         goto out;
 186
 187                 desc = to_sf_pdma_desc(vd);
 188                 residue = desc->xfer_size;
 189         }
 190
 191 out:
 192         spin_unlock_irqrestore(&chan->vchan.lock, flags);
 193         return residue;
 194 }
 195
 196 static enum dma_status
 197 sf_pdma_tx_status(struct dma_chan *dchan,
 198                   dma_cookie_t cookie,
 199                   struct dma_tx_state *txstate)
 200 {
 201         struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
 202         enum dma_status status;
 203
 204         status = dma_cookie_status(dchan, cookie, txstate);
 205
 206         if (txstate && status != DMA_ERROR)
 207                 dma_set_residue(txstate, sf_pdma_desc_residue(chan, cookie));
 208
 209         return status;
 210 }
 211
 212 static int sf_pdma_terminate_all(struct dma_chan *dchan)
 213 {
 214         struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
 215         unsigned long flags;
 216         LIST_HEAD(head);
 217
 218         spin_lock_irqsave(&chan->vchan.lock, flags);
 219         sf_pdma_disable_request(chan);
 220         kfree(chan->desc);
 221         chan->desc = NULL;
 222         chan->xfer_err = false;
 223         vchan_get_all_descriptors(&chan->vchan, &head);
 224         spin_unlock_irqrestore(&chan->vchan.lock, flags);
 225         vchan_dma_desc_free_list(&chan->vchan, &head);
 226
 227         return 0;
 228 }
 229
 230 static void sf_pdma_enable_request(struct sf_pdma_chan *chan)
 231 {
 232         struct pdma_regs *regs = &chan->regs;
 233         u32 v;
 234
 235         v = PDMA_CLAIM_MASK |
 236                 PDMA_ENABLE_DONE_INT_MASK |
 237                 PDMA_ENABLE_ERR_INT_MASK |
 238                 PDMA_RUN_MASK;
 239
 240         writel(v, regs->ctrl);
 241 }
 242
 243 static void sf_pdma_xfer_desc(struct sf_pdma_chan *chan)
 244 {
 245         struct sf_pdma_desc *desc = chan->desc;
 246         struct pdma_regs *regs = &chan->regs;
 247
 248         if (!desc) {
 249                 dev_err(chan->pdma->dma_dev.dev, "NULL desc.\n");
 250                 return;
 251         }
 252
 253         writel(desc->xfer_type, regs->xfer_type);
 254         writeq(desc->xfer_size, regs->xfer_size);
 255         writeq(desc->dst_addr, regs->dst_addr);
 256         writeq(desc->src_addr, regs->src_addr);
 257
 258         chan->desc = desc;
 259         chan->status = DMA_IN_PROGRESS;
 260         sf_pdma_enable_request(chan);
 261 }
 262
 263 static void sf_pdma_issue_pending(struct dma_chan *dchan)
 264 {
 265         struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
 266         unsigned long flags;
 267
 268         spin_lock_irqsave(&chan->vchan.lock, flags);
 269
 270         if (vchan_issue_pending(&chan->vchan) && chan->desc)
 271                 sf_pdma_xfer_desc(chan);
 272
 273         spin_unlock_irqrestore(&chan->vchan.lock, flags);
 274 }
 275
 276 static void sf_pdma_free_desc(struct virt_dma_desc *vdesc)
 277 {
 278         struct sf_pdma_desc *desc;
 279
 280         desc = to_sf_pdma_desc(vdesc);
 281         desc->in_use = false;
 282 }
 283
 284 static void sf_pdma_donebh_tasklet(struct tasklet_struct *t)
 285 {
 286         struct sf_pdma_chan *chan = from_tasklet(chan, t, done_tasklet);
 287         unsigned long flags;
 288
 289         spin_lock_irqsave(&chan->lock, flags);
 290         if (chan->xfer_err) {
 291                 chan->retries = MAX_RETRY;
 292                 chan->status = DMA_COMPLETE;
 293                 chan->xfer_err = false;
 294         }
 295         spin_unlock_irqrestore(&chan->lock, flags);
 296
 297         spin_lock_irqsave(&chan->vchan.lock, flags);
 298         list_del(&chan->desc->vdesc.node);
 299         vchan_cookie_complete(&chan->desc->vdesc);
 300         spin_unlock_irqrestore(&chan->vchan.lock, flags);
 301 }
 302
 303 static void sf_pdma_errbh_tasklet(struct tasklet_struct *t)
 304 {
 305         struct sf_pdma_chan *chan = from_tasklet(chan, t, err_tasklet);
 306         struct sf_pdma_desc *desc = chan->desc;
 307         unsigned long flags;
 308
 309         spin_lock_irqsave(&chan->lock, flags);
 310         if (chan->retries <= 0) {
 311                 /* fail to recover */
 312                 spin_unlock_irqrestore(&chan->lock, flags);
 313                 dmaengine_desc_get_callback_invoke(desc->async_tx, NULL);
 314         } else {
 315                 /* retry */
 316                 chan->retries--;
 317                 chan->xfer_err = true;
 318                 chan->status = DMA_ERROR;
 319
 320                 sf_pdma_enable_request(chan);
 321                 spin_unlock_irqrestore(&chan->lock, flags);
 322         }
 323 }
 324
 325 static irqreturn_t sf_pdma_done_isr(int irq, void *dev_id)
 326 {
 327         struct sf_pdma_chan *chan = dev_id;
 328         struct pdma_regs *regs = &chan->regs;
 329         unsigned long flags;
 330         u64 residue;
 331
 332         spin_lock_irqsave(&chan->vchan.lock, flags);
 333         writel((readl(regs->ctrl)) & ~PDMA_DONE_STATUS_MASK, regs->ctrl);
 334         residue = readq(regs->residue);
 335
 336         if (!residue) {
 337                 tasklet_hi_schedule(&chan->done_tasklet);
 338         } else {
 339                 /* submit next trascatioin if possible */
 340                 struct sf_pdma_desc *desc = chan->desc;
 341
 342                 desc->src_addr += desc->xfer_size - residue;
 343                 desc->dst_addr += desc->xfer_size - residue;
 344                 desc->xfer_size = residue;
 345
 346                 sf_pdma_xfer_desc(chan);
 347         }
 348
 349         spin_unlock_irqrestore(&chan->vchan.lock, flags);
 350
 351         return IRQ_HANDLED;
 352 }
 353
 354 static irqreturn_t sf_pdma_err_isr(int irq, void *dev_id)
 355 {
 356         struct sf_pdma_chan *chan = dev_id;
 357         struct pdma_regs *regs = &chan->regs;
 358         unsigned long flags;
 359
 360         spin_lock_irqsave(&chan->lock, flags);
 361         writel((readl(regs->ctrl)) & ~PDMA_ERR_STATUS_MASK, regs->ctrl);
 362         spin_unlock_irqrestore(&chan->lock, flags);
 363
 364         tasklet_schedule(&chan->err_tasklet);
 365
 366         return IRQ_HANDLED;
 367 }
 368
 369 /**
 370  * sf_pdma_irq_init() - Init PDMA IRQ Handlers
 371  * @pdev: pointer of platform_device
 372  * @pdma: pointer of PDMA engine. Caller should check NULL
 373  *
 374  * Initialize DONE and ERROR interrupt handler for 4 channels. Caller should
 375  * make sure the pointer passed in are non-NULL. This function should be called
 376  * only one time during the device probe.
 377  *
 378  * Context: Any context.
 379  *
 380  * Return:
 381  * * 0          - OK to init all IRQ handlers
 382  * * -EINVAL    - Fail to request IRQ
 383  */
 384 static int sf_pdma_irq_init(struct platform_device *pdev, struct sf_pdma *pdma)
 385 {
 386         int irq, r, i;
 387         struct sf_pdma_chan *chan;
 388
 389         for (i = 0; i < pdma->n_chans; i++) {
 390                 chan = &pdma->chans[i];
 391
 392                 irq = platform_get_irq(pdev, i * 2);
 393                 if (irq < 0) {
 394                         dev_err(&pdev->dev, "ch(%d) Can't get done irq.\n", i);
 395                         return -EINVAL;
 396                 }
 397
 398                 r = devm_request_irq(&pdev->dev, irq, sf_pdma_done_isr, 0,
 399                                      dev_name(&pdev->dev), (void *)chan);
 400                 if (r) {
 401                         dev_err(&pdev->dev, "Fail to attach done ISR: %d\n", r);
 402                         return -EINVAL;
 403                 }
 404
 405                 chan->txirq = irq;
 406
 407                 irq = platform_get_irq(pdev, (i * 2) + 1);
 408                 if (irq < 0) {
 409                         dev_err(&pdev->dev, "ch(%d) Can't get err irq.\n", i);
 410                         return -EINVAL;
 411                 }
 412
 413                 r = devm_request_irq(&pdev->dev, irq, sf_pdma_err_isr, 0,
 414                                      dev_name(&pdev->dev), (void *)chan);
 415                 if (r) {
 416                         dev_err(&pdev->dev, "Fail to attach err ISR: %d\n", r);
 417                         return -EINVAL;
 418                 }
 419
 420                 chan->errirq = irq;
 421         }
 422
 423         return 0;
 424 }
 425
 426 /**
 427  * sf_pdma_setup_chans() - Init settings of each channel
 428  * @pdma: pointer of PDMA engine. Caller should check NULL
 429  *
 430  * Initialize all data structure and register base. Caller should make sure
 431  * the pointer passed in are non-NULL. This function should be called only
 432  * one time during the device probe.
 433  *
 434  * Context: Any context.
 435  *
 436  * Return: none
 437  */
 438 static void sf_pdma_setup_chans(struct sf_pdma *pdma)
 439 {
 440         int i;
 441         struct sf_pdma_chan *chan;
 442
 443         INIT_LIST_HEAD(&pdma->dma_dev.channels);
 444
 445         for (i = 0; i < pdma->n_chans; i++) {
 446                 chan = &pdma->chans[i];
 447
 448                 chan->regs.ctrl =
 449                         SF_PDMA_REG_BASE(i) + PDMA_CTRL;
 450                 chan->regs.xfer_type =
 451                         SF_PDMA_REG_BASE(i) + PDMA_XFER_TYPE;
 452                 chan->regs.xfer_size =
 453                         SF_PDMA_REG_BASE(i) + PDMA_XFER_SIZE;
 454                 chan->regs.dst_addr =
 455                         SF_PDMA_REG_BASE(i) + PDMA_DST_ADDR;
 456                 chan->regs.src_addr =
 457                         SF_PDMA_REG_BASE(i) + PDMA_SRC_ADDR;
 458                 chan->regs.act_type =
 459                         SF_PDMA_REG_BASE(i) + PDMA_ACT_TYPE;
 460                 chan->regs.residue =
 461                         SF_PDMA_REG_BASE(i) + PDMA_REMAINING_BYTE;
 462                 chan->regs.cur_dst_addr =
 463                         SF_PDMA_REG_BASE(i) + PDMA_CUR_DST_ADDR;
 464                 chan->regs.cur_src_addr =
 465                         SF_PDMA_REG_BASE(i) + PDMA_CUR_SRC_ADDR;
 466
 467                 chan->pdma = pdma;
 468                 chan->pm_state = RUNNING;
 469                 chan->slave_id = i;
 470                 chan->xfer_err = false;
 471                 spin_lock_init(&chan->lock);
 472
 473                 chan->vchan.desc_free = sf_pdma_free_desc;
 474                 vchan_init(&chan->vchan, &pdma->dma_dev);
 475
 476                 writel(PDMA_CLEAR_CTRL, chan->regs.ctrl);
 477
 478                 tasklet_setup(&chan->done_tasklet, sf_pdma_donebh_tasklet);
 479                 tasklet_setup(&chan->err_tasklet, sf_pdma_errbh_tasklet);
 480         }
 481 }
 482
 483 static int sf_pdma_probe(struct platform_device *pdev)
 484 {
 485         struct sf_pdma *pdma;
 486         struct sf_pdma_chan *chan;
 487         struct resource *res;
 488         int len, chans;
 489         int ret;
 490         const enum dma_slave_buswidth widths =
 491                 DMA_SLAVE_BUSWIDTH_1_BYTE | DMA_SLAVE_BUSWIDTH_2_BYTES |
 492                 DMA_SLAVE_BUSWIDTH_4_BYTES | DMA_SLAVE_BUSWIDTH_8_BYTES |
 493                 DMA_SLAVE_BUSWIDTH_16_BYTES | DMA_SLAVE_BUSWIDTH_32_BYTES |
 494                 DMA_SLAVE_BUSWIDTH_64_BYTES;
 495
 496         chans = PDMA_NR_CH;
 497         len = sizeof(*pdma) + sizeof(*chan) * chans;
 498         pdma = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);
 499         if (!pdma)
 500                 return -ENOMEM;
 501
 502         pdma->n_chans = chans;
 503
 504         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 505         pdma->membase = devm_ioremap_resource(&pdev->dev, res);
 506         if (IS_ERR(pdma->membase))
 507                 return PTR_ERR(pdma->membase);
 508
 509         ret = sf_pdma_irq_init(pdev, pdma);
 510         if (ret)
 511                 return ret;
 512
 513         sf_pdma_setup_chans(pdma);
 514
 515         pdma->dma_dev.dev = &pdev->dev;
 516
 517         /* Setup capability */
 518         dma_cap_set(DMA_MEMCPY, pdma->dma_dev.cap_mask);
 519         pdma->dma_dev.copy_align = 2;
 520         pdma->dma_dev.src_addr_widths = widths;
 521         pdma->dma_dev.dst_addr_widths = widths;
 522         pdma->dma_dev.directions = BIT(DMA_MEM_TO_MEM);
 523         pdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
 524         pdma->dma_dev.descriptor_reuse = true;
 525
 526         /* Setup DMA APIs */
 527         pdma->dma_dev.device_alloc_chan_resources =
 528                 sf_pdma_alloc_chan_resources;
 529         pdma->dma_dev.device_free_chan_resources =
 530                 sf_pdma_free_chan_resources;
 531         pdma->dma_dev.device_tx_status = sf_pdma_tx_status;
 532         pdma->dma_dev.device_prep_dma_memcpy = sf_pdma_prep_dma_memcpy;
 533         pdma->dma_dev.device_config = sf_pdma_slave_config;
 534         pdma->dma_dev.device_terminate_all = sf_pdma_terminate_all;
 535         pdma->dma_dev.device_issue_pending = sf_pdma_issue_pending;
 536
 537         platform_set_drvdata(pdev, pdma);
 538
 539         ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
 540         if (ret)
 541                 dev_warn(&pdev->dev,
 542                          "Failed to set DMA mask. Fall back to default.\n");
 543
 544         ret = dma_async_device_register(&pdma->dma_dev);
 545         if (ret) {
 546                 dev_err(&pdev->dev,
 547                         "Can't register SiFive Platform DMA. (%d)\n", ret);
 548                 return ret;
 549         }
 550
 551         return 0;
 552 }
 553
 554 static int sf_pdma_remove(struct platform_device *pdev)
 555 {
 556         struct sf_pdma *pdma = platform_get_drvdata(pdev);
 557         struct sf_pdma_chan *ch;
 558         int i;
 559
 560         for (i = 0; i < PDMA_NR_CH; i++) {
 561                 ch = &pdma->chans[i];
 562
 563                 devm_free_irq(&pdev->dev, ch->txirq, ch);
 564                 devm_free_irq(&pdev->dev, ch->errirq, ch);
 565                 list_del(&ch->vchan.chan.device_node);
 566                 tasklet_kill(&ch->vchan.task);
 567                 tasklet_kill(&ch->done_tasklet);
 568                 tasklet_kill(&ch->err_tasklet);
 569         }
 570
 571         dma_async_device_unregister(&pdma->dma_dev);
 572
 573         return 0;
 574 }
 575
 576 static const struct of_device_id sf_pdma_dt_ids[] = {
 577         { .compatible = "sifive,fu540-c000-pdma" },
 578         {},
 579 };
 580 MODULE_DEVICE_TABLE(of, sf_pdma_dt_ids);
 581
 582 static struct platform_driver sf_pdma_driver = {
 583         .probe          = sf_pdma_probe,
 584         .remove         = sf_pdma_remove,
 585         .driver         = {
 586                 .name   = "sf-pdma",
 587                 .of_match_table = of_match_ptr(sf_pdma_dt_ids),
 588         },
 589 };
 590
 591 static int __init sf_pdma_init(void)
 592 {
 593         return platform_driver_register(&sf_pdma_driver);
 594 }
 595
 596 static void __exit sf_pdma_exit(void)
 597 {
 598         platform_driver_unregister(&sf_pdma_driver);
 599 }
 600
 601 /* do early init */
 602 subsys_initcall(sf_pdma_init);
 603 module_exit(sf_pdma_exit);
 604
 605 MODULE_LICENSE("GPL v2");
 606 MODULE_DESCRIPTION("SiFive Platform DMA driver");
 607 MODULE_AUTHOR("Green Wan <[email protected]>");