drivers/dma/stm32/stm32-dma3.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * STM32 DMA3 controller driver
   4  *
   5  * Copyright (C) STMicroelectronics 2024
   6  * Author(s): Amelie Delaunay <[email protected]>
   7  */
   8
   9 #include <linux/bitfield.h>
  10 #include <linux/clk.h>
  11 #include <linux/dma-mapping.h>
  12 #include <linux/dmaengine.h>
  13 #include <linux/dmapool.h>
  14 #include <linux/init.h>
  15 #include <linux/iopoll.h>
  16 #include <linux/list.h>
  17 #include <linux/module.h>
  18 #include <linux/of_dma.h>
  19 #include <linux/platform_device.h>
  20 #include <linux/pm_runtime.h>
  21 #include <linux/reset.h>
  22 #include <linux/slab.h>
  23
  24 #include "../virt-dma.h"
  25
  26 #define STM32_DMA3_SECCFGR              0x00
  27 #define STM32_DMA3_PRIVCFGR             0x04
  28 #define STM32_DMA3_RCFGLOCKR            0x08
  29 #define STM32_DMA3_MISR                 0x0c
  30 #define STM32_DMA3_SMISR                0x10
  31
  32 #define STM32_DMA3_CLBAR(x)             (0x50 + 0x80 * (x))
  33 #define STM32_DMA3_CCIDCFGR(x)          (0x54 + 0x80 * (x))
  34 #define STM32_DMA3_CSEMCR(x)            (0x58 + 0x80 * (x))
  35 #define STM32_DMA3_CFCR(x)              (0x5c + 0x80 * (x))
  36 #define STM32_DMA3_CSR(x)               (0x60 + 0x80 * (x))
  37 #define STM32_DMA3_CCR(x)               (0x64 + 0x80 * (x))
  38 #define STM32_DMA3_CTR1(x)              (0x90 + 0x80 * (x))
  39 #define STM32_DMA3_CTR2(x)              (0x94 + 0x80 * (x))
  40 #define STM32_DMA3_CBR1(x)              (0x98 + 0x80 * (x))
  41 #define STM32_DMA3_CSAR(x)              (0x9c + 0x80 * (x))
  42 #define STM32_DMA3_CDAR(x)              (0xa0 + 0x80 * (x))
  43 #define STM32_DMA3_CLLR(x)              (0xcc + 0x80 * (x))
  44
  45 #define STM32_DMA3_HWCFGR13             0xfc0 /* G_PER_CTRL(X) x=8..15 */
  46 #define STM32_DMA3_HWCFGR12             0xfc4 /* G_PER_CTRL(X) x=0..7 */
  47 #define STM32_DMA3_HWCFGR4              0xfe4 /* G_FIFO_SIZE(X) x=8..15 */
  48 #define STM32_DMA3_HWCFGR3              0xfe8 /* G_FIFO_SIZE(X) x=0..7 */
  49 #define STM32_DMA3_HWCFGR2              0xfec /* G_MAX_REQ_ID */
  50 #define STM32_DMA3_HWCFGR1              0xff0 /* G_MASTER_PORTS, G_NUM_CHANNELS, G_Mx_DATA_WIDTH */
  51 #define STM32_DMA3_VERR                 0xff4
  52
  53 /* SECCFGR DMA secure configuration register */
  54 #define SECCFGR_SEC(x)                  BIT(x)
  55
  56 /* MISR DMA non-secure/secure masked interrupt status register */
  57 #define MISR_MIS(x)                     BIT(x)
  58
  59 /* CxLBAR DMA channel x linked_list base address register */
  60 #define CLBAR_LBA                       GENMASK(31, 16)
  61
  62 /* CxCIDCFGR DMA channel x CID register */
  63 #define CCIDCFGR_CFEN                   BIT(0)
  64 #define CCIDCFGR_SEM_EN                 BIT(1)
  65 #define CCIDCFGR_SCID                   GENMASK(5, 4)
  66 #define CCIDCFGR_SEM_WLIST_CID0         BIT(16)
  67 #define CCIDCFGR_SEM_WLIST_CID1         BIT(17)
  68 #define CCIDCFGR_SEM_WLIST_CID2         BIT(18)
  69
  70 enum ccidcfgr_cid {
  71         CCIDCFGR_CID0,
  72         CCIDCFGR_CID1,
  73         CCIDCFGR_CID2,
  74 };
  75
  76 /* CxSEMCR DMA channel x semaphore control register */
  77 #define CSEMCR_SEM_MUTEX                BIT(0)
  78 #define CSEMCR_SEM_CCID                 GENMASK(5, 4)
  79
  80 /* CxFCR DMA channel x flag clear register */
  81 #define CFCR_TCF                        BIT(8)
  82 #define CFCR_HTF                        BIT(9)
  83 #define CFCR_DTEF                       BIT(10)
  84 #define CFCR_ULEF                       BIT(11)
  85 #define CFCR_USEF                       BIT(12)
  86 #define CFCR_SUSPF                      BIT(13)
  87
  88 /* CxSR DMA channel x status register */
  89 #define CSR_IDLEF                       BIT(0)
  90 #define CSR_TCF                         BIT(8)
  91 #define CSR_HTF                         BIT(9)
  92 #define CSR_DTEF                        BIT(10)
  93 #define CSR_ULEF                        BIT(11)
  94 #define CSR_USEF                        BIT(12)
  95 #define CSR_SUSPF                       BIT(13)
  96 #define CSR_ALL_F                       GENMASK(13, 8)
  97 #define CSR_FIFOL                       GENMASK(24, 16)
  98
  99 /* CxCR DMA channel x control register */
 100 #define CCR_EN                          BIT(0)
 101 #define CCR_RESET                       BIT(1)
 102 #define CCR_SUSP                        BIT(2)
 103 #define CCR_TCIE                        BIT(8)
 104 #define CCR_HTIE                        BIT(9)
 105 #define CCR_DTEIE                       BIT(10)
 106 #define CCR_ULEIE                       BIT(11)
 107 #define CCR_USEIE                       BIT(12)
 108 #define CCR_SUSPIE                      BIT(13)
 109 #define CCR_ALLIE                       GENMASK(13, 8)
 110 #define CCR_LSM                         BIT(16)
 111 #define CCR_LAP                         BIT(17)
 112 #define CCR_PRIO                        GENMASK(23, 22)
 113
 114 enum ccr_prio {
 115         CCR_PRIO_LOW,
 116         CCR_PRIO_MID,
 117         CCR_PRIO_HIGH,
 118         CCR_PRIO_VERY_HIGH,
 119 };
 120
 121 /* CxTR1 DMA channel x transfer register 1 */
 122 #define CTR1_SINC                       BIT(3)
 123 #define CTR1_SBL_1                      GENMASK(9, 4)
 124 #define CTR1_DINC                       BIT(19)
 125 #define CTR1_DBL_1                      GENMASK(25, 20)
 126 #define CTR1_SDW_LOG2                   GENMASK(1, 0)
 127 #define CTR1_PAM                        GENMASK(12, 11)
 128 #define CTR1_SAP                        BIT(14)
 129 #define CTR1_DDW_LOG2                   GENMASK(17, 16)
 130 #define CTR1_DAP                        BIT(30)
 131
 132 enum ctr1_dw {
 133         CTR1_DW_BYTE,
 134         CTR1_DW_HWORD,
 135         CTR1_DW_WORD,
 136         CTR1_DW_DWORD, /* Depends on HWCFGR1.G_M0_DATA_WIDTH_ENC and .G_M1_DATA_WIDTH_ENC */
 137 };
 138
 139 enum ctr1_pam {
 140         CTR1_PAM_0S_LT,         /* if DDW > SDW, padded with 0s else left-truncated */
 141         CTR1_PAM_SE_RT,         /* if DDW > SDW, sign extended else right-truncated */
 142         CTR1_PAM_PACK_UNPACK,   /* FIFO queued */
 143 };
 144
 145 /* CxTR2 DMA channel x transfer register 2 */
 146 #define CTR2_REQSEL                     GENMASK(7, 0)
 147 #define CTR2_SWREQ                      BIT(9)
 148 #define CTR2_DREQ                       BIT(10)
 149 #define CTR2_BREQ                       BIT(11)
 150 #define CTR2_PFREQ                      BIT(12)
 151 #define CTR2_TCEM                       GENMASK(31, 30)
 152
 153 enum ctr2_tcem {
 154         CTR2_TCEM_BLOCK,
 155         CTR2_TCEM_REPEAT_BLOCK,
 156         CTR2_TCEM_LLI,
 157         CTR2_TCEM_CHANNEL,
 158 };
 159
 160 /* CxBR1 DMA channel x block register 1 */
 161 #define CBR1_BNDT                       GENMASK(15, 0)
 162
 163 /* CxLLR DMA channel x linked-list address register */
 164 #define CLLR_LA                         GENMASK(15, 2)
 165 #define CLLR_ULL                        BIT(16)
 166 #define CLLR_UDA                        BIT(27)
 167 #define CLLR_USA                        BIT(28)
 168 #define CLLR_UB1                        BIT(29)
 169 #define CLLR_UT2                        BIT(30)
 170 #define CLLR_UT1                        BIT(31)
 171
 172 /* HWCFGR13 DMA hardware configuration register 13 x=8..15 */
 173 /* HWCFGR12 DMA hardware configuration register 12 x=0..7 */
 174 #define G_PER_CTRL(x)                   (ULL(0x1) << (4 * (x)))
 175
 176 /* HWCFGR4 DMA hardware configuration register 4 x=8..15 */
 177 /* HWCFGR3 DMA hardware configuration register 3 x=0..7 */
 178 #define G_FIFO_SIZE(x)                  (ULL(0x7) << (4 * (x)))
 179
 180 #define get_chan_hwcfg(x, mask, reg)    (((reg) & (mask)) >> (4 * (x)))
 181
 182 /* HWCFGR2 DMA hardware configuration register 2 */
 183 #define G_MAX_REQ_ID                    GENMASK(7, 0)
 184
 185 /* HWCFGR1 DMA hardware configuration register 1 */
 186 #define G_MASTER_PORTS                  GENMASK(2, 0)
 187 #define G_NUM_CHANNELS                  GENMASK(12, 8)
 188 #define G_M0_DATA_WIDTH_ENC             GENMASK(25, 24)
 189 #define G_M1_DATA_WIDTH_ENC             GENMASK(29, 28)
 190
 191 enum stm32_dma3_master_ports {
 192         AXI64,          /* 1x AXI: 64-bit port 0 */
 193         AHB32,          /* 1x AHB: 32-bit port 0 */
 194         AHB32_AHB32,    /* 2x AHB: 32-bit port 0 and 32-bit port 1 */
 195         AXI64_AHB32,    /* 1x AXI 64-bit port 0 and 1x AHB 32-bit port 1 */
 196         AXI64_AXI64,    /* 2x AXI: 64-bit port 0 and 64-bit port 1 */
 197         AXI128_AHB32,   /* 1x AXI 128-bit port 0 and 1x AHB 32-bit port 1 */
 198 };
 199
 200 enum stm32_dma3_port_data_width {
 201         DW_32,          /* 32-bit, for AHB */
 202         DW_64,          /* 64-bit, for AXI */
 203         DW_128,         /* 128-bit, for AXI */
 204         DW_INVALID,
 205 };
 206
 207 /* VERR DMA version register */
 208 #define VERR_MINREV                     GENMASK(3, 0)
 209 #define VERR_MAJREV                     GENMASK(7, 4)
 210
 211 /* Device tree */
 212 /* struct stm32_dma3_dt_conf */
 213 /* .ch_conf */
 214 #define STM32_DMA3_DT_PRIO              GENMASK(1, 0) /* CCR_PRIO */
 215 #define STM32_DMA3_DT_FIFO              GENMASK(7, 4)
 216 /* .tr_conf */
 217 #define STM32_DMA3_DT_SINC              BIT(0) /* CTR1_SINC */
 218 #define STM32_DMA3_DT_SAP               BIT(1) /* CTR1_SAP */
 219 #define STM32_DMA3_DT_DINC              BIT(4) /* CTR1_DINC */
 220 #define STM32_DMA3_DT_DAP               BIT(5) /* CTR1_DAP */
 221 #define STM32_DMA3_DT_BREQ              BIT(8) /* CTR2_BREQ */
 222 #define STM32_DMA3_DT_PFREQ             BIT(9) /* CTR2_PFREQ */
 223 #define STM32_DMA3_DT_TCEM              GENMASK(13, 12) /* CTR2_TCEM */
 224
 225 /* struct stm32_dma3_chan .config_set bitfield */
 226 #define STM32_DMA3_CFG_SET_DT           BIT(0)
 227 #define STM32_DMA3_CFG_SET_DMA          BIT(1)
 228 #define STM32_DMA3_CFG_SET_BOTH         (STM32_DMA3_CFG_SET_DT | STM32_DMA3_CFG_SET_DMA)
 229
 230 #define STM32_DMA3_MAX_BLOCK_SIZE       ALIGN_DOWN(CBR1_BNDT, 64)
 231 #define port_is_ahb(maxdw)              ({ typeof(maxdw) (_maxdw) = (maxdw); \
 232                                            ((_maxdw) != DW_INVALID) && ((_maxdw) == DW_32); })
 233 #define port_is_axi(maxdw)              ({ typeof(maxdw) (_maxdw) = (maxdw); \
 234                                            ((_maxdw) != DW_INVALID) && ((_maxdw) != DW_32); })
 235 #define get_chan_max_dw(maxdw, maxburst)((port_is_ahb(maxdw) ||                      \
 236                                           (maxburst) < DMA_SLAVE_BUSWIDTH_8_BYTES) ? \
 237                                          DMA_SLAVE_BUSWIDTH_4_BYTES : DMA_SLAVE_BUSWIDTH_8_BYTES)
 238
 239 /* Static linked-list data structure (depends on update bits UT1/UT2/UB1/USA/UDA/ULL) */
 240 struct stm32_dma3_hwdesc {
 241         u32 ctr1;
 242         u32 ctr2;
 243         u32 cbr1;
 244         u32 csar;
 245         u32 cdar;
 246         u32 cllr;
 247 } __packed __aligned(32);
 248
 249 /*
 250  * CLLR_LA / sizeof(struct stm32_dma3_hwdesc) represents the number of hdwdesc that can be addressed
 251  * by the pointer to the next linked-list data structure. The __aligned forces the 32-byte
 252  * alignment. So use hardcoded 32. Multiplied by the max block size of each item, it represents
 253  * the sg size limitation.
 254  */
 255 #define STM32_DMA3_MAX_SEG_SIZE         ((CLLR_LA / 32) * STM32_DMA3_MAX_BLOCK_SIZE)
 256
 257 /*
 258  * Linked-list items
 259  */
 260 struct stm32_dma3_lli {
 261         struct stm32_dma3_hwdesc *hwdesc;
 262         dma_addr_t hwdesc_addr;
 263 };
 264
 265 struct stm32_dma3_swdesc {
 266         struct virt_dma_desc vdesc;
 267         u32 ccr;
 268         bool cyclic;
 269         u32 lli_size;
 270         struct stm32_dma3_lli lli[] __counted_by(lli_size);
 271 };
 272
 273 struct stm32_dma3_dt_conf {
 274         u32 ch_id;
 275         u32 req_line;
 276         u32 ch_conf;
 277         u32 tr_conf;
 278 };
 279
 280 struct stm32_dma3_chan {
 281         struct virt_dma_chan vchan;
 282         u32 id;
 283         int irq;
 284         u32 fifo_size;
 285         u32 max_burst;
 286         bool semaphore_mode;
 287         struct stm32_dma3_dt_conf dt_config;
 288         struct dma_slave_config dma_config;
 289         u8 config_set;
 290         struct dma_pool *lli_pool;
 291         struct stm32_dma3_swdesc *swdesc;
 292         enum ctr2_tcem tcem;
 293         u32 dma_status;
 294 };
 295
 296 struct stm32_dma3_ddata {
 297         struct dma_device dma_dev;
 298         void __iomem *base;
 299         struct clk *clk;
 300         struct stm32_dma3_chan *chans;
 301         u32 dma_channels;
 302         u32 dma_requests;
 303         enum stm32_dma3_port_data_width ports_max_dw[2];
 304 };
 305
 306 static inline struct stm32_dma3_ddata *to_stm32_dma3_ddata(struct stm32_dma3_chan *chan)
 307 {
 308         return container_of(chan->vchan.chan.device, struct stm32_dma3_ddata, dma_dev);
 309 }
 310
 311 static inline struct stm32_dma3_chan *to_stm32_dma3_chan(struct dma_chan *c)
 312 {
 313         return container_of(c, struct stm32_dma3_chan, vchan.chan);
 314 }
 315
 316 static inline struct stm32_dma3_swdesc *to_stm32_dma3_swdesc(struct virt_dma_desc *vdesc)
 317 {
 318         return container_of(vdesc, struct stm32_dma3_swdesc, vdesc);
 319 }
 320
 321 static struct device *chan2dev(struct stm32_dma3_chan *chan)
 322 {
 323         return &chan->vchan.chan.dev->device;
 324 }
 325
 326 static void stm32_dma3_chan_dump_reg(struct stm32_dma3_chan *chan)
 327 {
 328         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
 329         struct device *dev = chan2dev(chan);
 330         u32 id = chan->id, offset;
 331
 332         offset = STM32_DMA3_SECCFGR;
 333         dev_dbg(dev, "SECCFGR(0x%03x): %08x\n", offset, readl_relaxed(ddata->base + offset));
 334         offset = STM32_DMA3_PRIVCFGR;
 335         dev_dbg(dev, "PRIVCFGR(0x%03x): %08x\n", offset, readl_relaxed(ddata->base + offset));
 336         offset = STM32_DMA3_CCIDCFGR(id);
 337         dev_dbg(dev, "C%dCIDCFGR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
 338         offset = STM32_DMA3_CSEMCR(id);
 339         dev_dbg(dev, "C%dSEMCR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
 340         offset = STM32_DMA3_CSR(id);
 341         dev_dbg(dev, "C%dSR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
 342         offset = STM32_DMA3_CCR(id);
 343         dev_dbg(dev, "C%dCR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
 344         offset = STM32_DMA3_CTR1(id);
 345         dev_dbg(dev, "C%dTR1(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
 346         offset = STM32_DMA3_CTR2(id);
 347         dev_dbg(dev, "C%dTR2(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
 348         offset = STM32_DMA3_CBR1(id);
 349         dev_dbg(dev, "C%dBR1(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
 350         offset = STM32_DMA3_CSAR(id);
 351         dev_dbg(dev, "C%dSAR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
 352         offset = STM32_DMA3_CDAR(id);
 353         dev_dbg(dev, "C%dDAR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
 354         offset = STM32_DMA3_CLLR(id);
 355         dev_dbg(dev, "C%dLLR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
 356         offset = STM32_DMA3_CLBAR(id);
 357         dev_dbg(dev, "C%dLBAR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
 358 }
 359
 360 static void stm32_dma3_chan_dump_hwdesc(struct stm32_dma3_chan *chan,
 361                                         struct stm32_dma3_swdesc *swdesc)
 362 {
 363         struct stm32_dma3_hwdesc *hwdesc;
 364         int i;
 365
 366         for (i = 0; i < swdesc->lli_size; i++) {
 367                 hwdesc = swdesc->lli[i].hwdesc;
 368                 if (i)
 369                         dev_dbg(chan2dev(chan), "V\n");
 370                 dev_dbg(chan2dev(chan), "[%d]@%pad\n", i, &swdesc->lli[i].hwdesc_addr);
 371                 dev_dbg(chan2dev(chan), "| C%dTR1: %08x\n", chan->id, hwdesc->ctr1);
 372                 dev_dbg(chan2dev(chan), "| C%dTR2: %08x\n", chan->id, hwdesc->ctr2);
 373                 dev_dbg(chan2dev(chan), "| C%dBR1: %08x\n", chan->id, hwdesc->cbr1);
 374                 dev_dbg(chan2dev(chan), "| C%dSAR: %08x\n", chan->id, hwdesc->csar);
 375                 dev_dbg(chan2dev(chan), "| C%dDAR: %08x\n", chan->id, hwdesc->cdar);
 376                 dev_dbg(chan2dev(chan), "| C%dLLR: %08x\n", chan->id, hwdesc->cllr);
 377         }
 378
 379         if (swdesc->cyclic) {
 380                 dev_dbg(chan2dev(chan), "|\n");
 381                 dev_dbg(chan2dev(chan), "-->[0]@%pad\n", &swdesc->lli[0].hwdesc_addr);
 382         } else {
 383                 dev_dbg(chan2dev(chan), "X\n");
 384         }
 385 }
 386
 387 static struct stm32_dma3_swdesc *stm32_dma3_chan_desc_alloc(struct stm32_dma3_chan *chan, u32 count)
 388 {
 389         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
 390         struct stm32_dma3_swdesc *swdesc;
 391         int i;
 392
 393         /*
 394          * If the memory to be allocated for the number of hwdesc (6 u32 members but 32-bytes
 395          * aligned) is greater than the maximum address of CLLR_LA, then the last items can't be
 396          * addressed, so abort the allocation.
 397          */
 398         if ((count * 32) > CLLR_LA) {
 399                 dev_err(chan2dev(chan), "Transfer is too big (> %luB)\n", STM32_DMA3_MAX_SEG_SIZE);
 400                 return NULL;
 401         }
 402
 403         swdesc = kzalloc(struct_size(swdesc, lli, count), GFP_NOWAIT);
 404         if (!swdesc)
 405                 return NULL;
 406
 407         for (i = 0; i < count; i++) {
 408                 swdesc->lli[i].hwdesc = dma_pool_zalloc(chan->lli_pool, GFP_NOWAIT,
 409                                                         &swdesc->lli[i].hwdesc_addr);
 410                 if (!swdesc->lli[i].hwdesc)
 411                         goto err_pool_free;
 412         }
 413         swdesc->lli_size = count;
 414         swdesc->ccr = 0;
 415
 416         /* Set LL base address */
 417         writel_relaxed(swdesc->lli[0].hwdesc_addr & CLBAR_LBA,
 418                        ddata->base + STM32_DMA3_CLBAR(chan->id));
 419
 420         /* Set LL allocated port */
 421         swdesc->ccr &= ~CCR_LAP;
 422
 423         return swdesc;
 424
 425 err_pool_free:
 426         dev_err(chan2dev(chan), "Failed to alloc descriptors\n");
 427         while (--i >= 0)
 428                 dma_pool_free(chan->lli_pool, swdesc->lli[i].hwdesc, swdesc->lli[i].hwdesc_addr);
 429         kfree(swdesc);
 430
 431         return NULL;
 432 }
 433
 434 static void stm32_dma3_chan_desc_free(struct stm32_dma3_chan *chan,
 435                                       struct stm32_dma3_swdesc *swdesc)
 436 {
 437         int i;
 438
 439         for (i = 0; i < swdesc->lli_size; i++)
 440                 dma_pool_free(chan->lli_pool, swdesc->lli[i].hwdesc, swdesc->lli[i].hwdesc_addr);
 441
 442         kfree(swdesc);
 443 }
 444
 445 static void stm32_dma3_chan_vdesc_free(struct virt_dma_desc *vdesc)
 446 {
 447         struct stm32_dma3_swdesc *swdesc = to_stm32_dma3_swdesc(vdesc);
 448         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(vdesc->tx.chan);
 449
 450         stm32_dma3_chan_desc_free(chan, swdesc);
 451 }
 452
 453 static void stm32_dma3_check_user_setting(struct stm32_dma3_chan *chan)
 454 {
 455         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
 456         struct device *dev = chan2dev(chan);
 457         u32 ctr1 = readl_relaxed(ddata->base + STM32_DMA3_CTR1(chan->id));
 458         u32 cbr1 = readl_relaxed(ddata->base + STM32_DMA3_CBR1(chan->id));
 459         u32 csar = readl_relaxed(ddata->base + STM32_DMA3_CSAR(chan->id));
 460         u32 cdar = readl_relaxed(ddata->base + STM32_DMA3_CDAR(chan->id));
 461         u32 cllr = readl_relaxed(ddata->base + STM32_DMA3_CLLR(chan->id));
 462         u32 bndt = FIELD_GET(CBR1_BNDT, cbr1);
 463         u32 sdw = 1 << FIELD_GET(CTR1_SDW_LOG2, ctr1);
 464         u32 ddw = 1 << FIELD_GET(CTR1_DDW_LOG2, ctr1);
 465         u32 sap = FIELD_GET(CTR1_SAP, ctr1);
 466         u32 dap = FIELD_GET(CTR1_DAP, ctr1);
 467
 468         if (!bndt && !FIELD_GET(CLLR_UB1, cllr))
 469                 dev_err(dev, "null source block size and no update of this value\n");
 470         if (bndt % sdw)
 471                 dev_err(dev, "source block size not multiple of src data width\n");
 472         if (FIELD_GET(CTR1_PAM, ctr1) == CTR1_PAM_PACK_UNPACK && bndt % ddw)
 473                 dev_err(dev, "(un)packing mode w/ src block size not multiple of dst data width\n");
 474         if (csar % sdw)
 475                 dev_err(dev, "unaligned source address not multiple of src data width\n");
 476         if (cdar % ddw)
 477                 dev_err(dev, "unaligned destination address not multiple of dst data width\n");
 478         if (sdw == DMA_SLAVE_BUSWIDTH_8_BYTES && port_is_ahb(ddata->ports_max_dw[sap]))
 479                 dev_err(dev, "double-word source data width not supported on port %u\n", sap);
 480         if (ddw == DMA_SLAVE_BUSWIDTH_8_BYTES && port_is_ahb(ddata->ports_max_dw[dap]))
 481                 dev_err(dev, "double-word destination data width not supported on port %u\n", dap);
 482 }
 483
 484 static void stm32_dma3_chan_prep_hwdesc(struct stm32_dma3_chan *chan,
 485                                         struct stm32_dma3_swdesc *swdesc,
 486                                         u32 curr, dma_addr_t src, dma_addr_t dst, u32 len,
 487                                         u32 ctr1, u32 ctr2, bool is_last, bool is_cyclic)
 488 {
 489         struct stm32_dma3_hwdesc *hwdesc;
 490         dma_addr_t next_lli;
 491         u32 next = curr + 1;
 492
 493         hwdesc = swdesc->lli[curr].hwdesc;
 494         hwdesc->ctr1 = ctr1;
 495         hwdesc->ctr2 = ctr2;
 496         hwdesc->cbr1 = FIELD_PREP(CBR1_BNDT, len);
 497         hwdesc->csar = src;
 498         hwdesc->cdar = dst;
 499
 500         if (is_last) {
 501                 if (is_cyclic)
 502                         next_lli = swdesc->lli[0].hwdesc_addr;
 503                 else
 504                         next_lli = 0;
 505         } else {
 506                 next_lli = swdesc->lli[next].hwdesc_addr;
 507         }
 508
 509         hwdesc->cllr = 0;
 510         if (next_lli) {
 511                 hwdesc->cllr |= CLLR_UT1 | CLLR_UT2 | CLLR_UB1;
 512                 hwdesc->cllr |= CLLR_USA | CLLR_UDA | CLLR_ULL;
 513                 hwdesc->cllr |= (next_lli & CLLR_LA);
 514         }
 515
 516         /*
 517          * Make sure to flush the CPU's write buffers so that the descriptors are ready to be read
 518          * by DMA3. By explicitly using a write memory barrier here, instead of doing it with writel
 519          * to enable the channel, we avoid an unnecessary barrier in the case where the descriptors
 520          * are reused (DMA_CTRL_REUSE).
 521          */
 522         if (is_last)
 523                 dma_wmb();
 524 }
 525
 526 static enum dma_slave_buswidth stm32_dma3_get_max_dw(u32 chan_max_burst,
 527                                                      enum stm32_dma3_port_data_width port_max_dw,
 528                                                      u32 len, dma_addr_t addr)
 529 {
 530         enum dma_slave_buswidth max_dw = get_chan_max_dw(port_max_dw, chan_max_burst);
 531
 532         /* len and addr must be a multiple of dw */
 533         return 1 << __ffs(len | addr | max_dw);
 534 }
 535
 536 static u32 stm32_dma3_get_max_burst(u32 len, enum dma_slave_buswidth dw, u32 chan_max_burst)
 537 {
 538         u32 max_burst = chan_max_burst ? chan_max_burst / dw : 1;
 539
 540         /* len is a multiple of dw, so if len is < chan_max_burst, shorten burst */
 541         if (len < chan_max_burst)
 542                 max_burst = len / dw;
 543
 544         /*
 545          * HW doesn't modify the burst if burst size <= half of the fifo size.
 546          * If len is not a multiple of burst size, last burst is shortened by HW.
 547          */
 548         return max_burst;
 549 }
 550
 551 static int stm32_dma3_chan_prep_hw(struct stm32_dma3_chan *chan, enum dma_transfer_direction dir,
 552                                    u32 *ccr, u32 *ctr1, u32 *ctr2,
 553                                    dma_addr_t src_addr, dma_addr_t dst_addr, u32 len)
 554 {
 555         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
 556         struct dma_device dma_device = ddata->dma_dev;
 557         u32 sdw, ddw, sbl_max, dbl_max, tcem, init_dw, init_bl_max;
 558         u32 _ctr1 = 0, _ctr2 = 0;
 559         u32 ch_conf = chan->dt_config.ch_conf;
 560         u32 tr_conf = chan->dt_config.tr_conf;
 561         u32 sap = FIELD_GET(STM32_DMA3_DT_SAP, tr_conf), sap_max_dw;
 562         u32 dap = FIELD_GET(STM32_DMA3_DT_DAP, tr_conf), dap_max_dw;
 563
 564         dev_dbg(chan2dev(chan), "%s from %pad to %pad\n",
 565                 dmaengine_get_direction_text(dir), &src_addr, &dst_addr);
 566
 567         sdw = chan->dma_config.src_addr_width ? : get_chan_max_dw(sap, chan->max_burst);
 568         ddw = chan->dma_config.dst_addr_width ? : get_chan_max_dw(dap, chan->max_burst);
 569         sbl_max = chan->dma_config.src_maxburst ? : 1;
 570         dbl_max = chan->dma_config.dst_maxburst ? : 1;
 571
 572         /* Following conditions would raise User Setting Error interrupt */
 573         if (!(dma_device.src_addr_widths & BIT(sdw)) || !(dma_device.dst_addr_widths & BIT(ddw))) {
 574                 dev_err(chan2dev(chan), "Bus width (src=%u, dst=%u) not supported\n", sdw, ddw);
 575                 return -EINVAL;
 576         }
 577
 578         if (ddata->ports_max_dw[1] == DW_INVALID && (sap || dap)) {
 579                 dev_err(chan2dev(chan), "Only one master port, port 1 is not supported\n");
 580                 return -EINVAL;
 581         }
 582
 583         sap_max_dw = ddata->ports_max_dw[sap];
 584         dap_max_dw = ddata->ports_max_dw[dap];
 585         if ((port_is_ahb(sap_max_dw) && sdw == DMA_SLAVE_BUSWIDTH_8_BYTES) ||
 586             (port_is_ahb(dap_max_dw) && ddw == DMA_SLAVE_BUSWIDTH_8_BYTES)) {
 587                 dev_err(chan2dev(chan),
 588                         "8 bytes buswidth (src=%u, dst=%u) not supported on port (sap=%u, dap=%u\n",
 589                         sdw, ddw, sap, dap);
 590                 return -EINVAL;
 591         }
 592
 593         if (FIELD_GET(STM32_DMA3_DT_SINC, tr_conf))
 594                 _ctr1 |= CTR1_SINC;
 595         if (sap)
 596                 _ctr1 |= CTR1_SAP;
 597         if (FIELD_GET(STM32_DMA3_DT_DINC, tr_conf))
 598                 _ctr1 |= CTR1_DINC;
 599         if (dap)
 600                 _ctr1 |= CTR1_DAP;
 601
 602         _ctr2 |= FIELD_PREP(CTR2_REQSEL, chan->dt_config.req_line) & ~CTR2_SWREQ;
 603         if (FIELD_GET(STM32_DMA3_DT_BREQ, tr_conf))
 604                 _ctr2 |= CTR2_BREQ;
 605         if (dir == DMA_DEV_TO_MEM && FIELD_GET(STM32_DMA3_DT_PFREQ, tr_conf))
 606                 _ctr2 |= CTR2_PFREQ;
 607         tcem = FIELD_GET(STM32_DMA3_DT_TCEM, tr_conf);
 608         _ctr2 |= FIELD_PREP(CTR2_TCEM, tcem);
 609
 610         /* Store TCEM to know on which event TC flag occurred */
 611         chan->tcem = tcem;
 612         /* Store direction for residue computation */
 613         chan->dma_config.direction = dir;
 614
 615         switch (dir) {
 616         case DMA_MEM_TO_DEV:
 617                 /* Set destination (device) data width and burst */
 618                 ddw = min_t(u32, ddw, stm32_dma3_get_max_dw(chan->max_burst, dap_max_dw,
 619                                                             len, dst_addr));
 620                 dbl_max = min_t(u32, dbl_max, stm32_dma3_get_max_burst(len, ddw, chan->max_burst));
 621
 622                 /* Set source (memory) data width and burst */
 623                 sdw = stm32_dma3_get_max_dw(chan->max_burst, sap_max_dw, len, src_addr);
 624                 sbl_max = stm32_dma3_get_max_burst(len, sdw, chan->max_burst);
 625
 626                 _ctr1 |= FIELD_PREP(CTR1_SDW_LOG2, ilog2(sdw));
 627                 _ctr1 |= FIELD_PREP(CTR1_SBL_1, sbl_max - 1);
 628                 _ctr1 |= FIELD_PREP(CTR1_DDW_LOG2, ilog2(ddw));
 629                 _ctr1 |= FIELD_PREP(CTR1_DBL_1, dbl_max - 1);
 630
 631                 if (ddw != sdw) {
 632                         _ctr1 |= FIELD_PREP(CTR1_PAM, CTR1_PAM_PACK_UNPACK);
 633                         /* Should never reach this case as ddw is clamped down */
 634                         if (len & (ddw - 1)) {
 635                                 dev_err(chan2dev(chan),
 636                                         "Packing mode is enabled and len is not multiple of ddw");
 637                                 return -EINVAL;
 638                         }
 639                 }
 640
 641                 /* dst = dev */
 642                 _ctr2 |= CTR2_DREQ;
 643
 644                 break;
 645
 646         case DMA_DEV_TO_MEM:
 647                 /* Set source (device) data width and burst */
 648                 sdw = min_t(u32, sdw, stm32_dma3_get_max_dw(chan->max_burst, sap_max_dw,
 649                                                             len, src_addr));
 650                 sbl_max = min_t(u32, sbl_max, stm32_dma3_get_max_burst(len, sdw, chan->max_burst));
 651
 652                 /* Set destination (memory) data width and burst */
 653                 ddw = stm32_dma3_get_max_dw(chan->max_burst, dap_max_dw, len, dst_addr);
 654                 dbl_max = stm32_dma3_get_max_burst(len, ddw, chan->max_burst);
 655
 656                 _ctr1 |= FIELD_PREP(CTR1_SDW_LOG2, ilog2(sdw));
 657                 _ctr1 |= FIELD_PREP(CTR1_SBL_1, sbl_max - 1);
 658                 _ctr1 |= FIELD_PREP(CTR1_DDW_LOG2, ilog2(ddw));
 659                 _ctr1 |= FIELD_PREP(CTR1_DBL_1, dbl_max - 1);
 660
 661                 if (ddw != sdw) {
 662                         _ctr1 |= FIELD_PREP(CTR1_PAM, CTR1_PAM_PACK_UNPACK);
 663                         /* Should never reach this case as ddw is clamped down */
 664                         if (len & (ddw - 1)) {
 665                                 dev_err(chan2dev(chan),
 666                                         "Packing mode is enabled and len is not multiple of ddw\n");
 667                                 return -EINVAL;
 668                         }
 669                 }
 670
 671                 /* dst = mem */
 672                 _ctr2 &= ~CTR2_DREQ;
 673
 674                 break;
 675
 676         case DMA_MEM_TO_MEM:
 677                 /* Set source (memory) data width and burst */
 678                 init_dw = sdw;
 679                 init_bl_max = sbl_max;
 680                 sdw = stm32_dma3_get_max_dw(chan->max_burst, sap_max_dw, len, src_addr);
 681                 sbl_max = stm32_dma3_get_max_burst(len, sdw, chan->max_burst);
 682                 if (chan->config_set & STM32_DMA3_CFG_SET_DMA) {
 683                         sdw = min_t(u32, init_dw, sdw);
 684                         sbl_max = min_t(u32, init_bl_max,
 685                                         stm32_dma3_get_max_burst(len, sdw, chan->max_burst));
 686                 }
 687
 688                 /* Set destination (memory) data width and burst */
 689                 init_dw = ddw;
 690                 init_bl_max = dbl_max;
 691                 ddw = stm32_dma3_get_max_dw(chan->max_burst, dap_max_dw, len, dst_addr);
 692                 dbl_max = stm32_dma3_get_max_burst(len, ddw, chan->max_burst);
 693                 if (chan->config_set & STM32_DMA3_CFG_SET_DMA) {
 694                         ddw = min_t(u32, init_dw, ddw);
 695                         dbl_max = min_t(u32, init_bl_max,
 696                                         stm32_dma3_get_max_burst(len, ddw, chan->max_burst));
 697                 }
 698
 699                 _ctr1 |= FIELD_PREP(CTR1_SDW_LOG2, ilog2(sdw));
 700                 _ctr1 |= FIELD_PREP(CTR1_SBL_1, sbl_max - 1);
 701                 _ctr1 |= FIELD_PREP(CTR1_DDW_LOG2, ilog2(ddw));
 702                 _ctr1 |= FIELD_PREP(CTR1_DBL_1, dbl_max - 1);
 703
 704                 if (ddw != sdw) {
 705                         _ctr1 |= FIELD_PREP(CTR1_PAM, CTR1_PAM_PACK_UNPACK);
 706                         /* Should never reach this case as ddw is clamped down */
 707                         if (len & (ddw - 1)) {
 708                                 dev_err(chan2dev(chan),
 709                                         "Packing mode is enabled and len is not multiple of ddw");
 710                                 return -EINVAL;
 711                         }
 712                 }
 713
 714                 /* CTR2_REQSEL/DREQ/BREQ/PFREQ are ignored with CTR2_SWREQ=1 */
 715                 _ctr2 |= CTR2_SWREQ;
 716
 717                 break;
 718
 719         default:
 720                 dev_err(chan2dev(chan), "Direction %s not supported\n",
 721                         dmaengine_get_direction_text(dir));
 722                 return -EINVAL;
 723         }
 724
 725         *ccr |= FIELD_PREP(CCR_PRIO, FIELD_GET(STM32_DMA3_DT_PRIO, ch_conf));
 726         *ctr1 = _ctr1;
 727         *ctr2 = _ctr2;
 728
 729         dev_dbg(chan2dev(chan), "%s: sdw=%u bytes sbl=%u beats ddw=%u bytes dbl=%u beats\n",
 730                 __func__, sdw, sbl_max, ddw, dbl_max);
 731
 732         return 0;
 733 }
 734
 735 static void stm32_dma3_chan_start(struct stm32_dma3_chan *chan)
 736 {
 737         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
 738         struct virt_dma_desc *vdesc;
 739         struct stm32_dma3_hwdesc *hwdesc;
 740         u32 id = chan->id;
 741         u32 csr, ccr;
 742
 743         vdesc = vchan_next_desc(&chan->vchan);
 744         if (!vdesc) {
 745                 chan->swdesc = NULL;
 746                 return;
 747         }
 748         list_del(&vdesc->node);
 749
 750         chan->swdesc = to_stm32_dma3_swdesc(vdesc);
 751         hwdesc = chan->swdesc->lli[0].hwdesc;
 752
 753         stm32_dma3_chan_dump_hwdesc(chan, chan->swdesc);
 754
 755         writel_relaxed(chan->swdesc->ccr, ddata->base + STM32_DMA3_CCR(id));
 756         writel_relaxed(hwdesc->ctr1, ddata->base + STM32_DMA3_CTR1(id));
 757         writel_relaxed(hwdesc->ctr2, ddata->base + STM32_DMA3_CTR2(id));
 758         writel_relaxed(hwdesc->cbr1, ddata->base + STM32_DMA3_CBR1(id));
 759         writel_relaxed(hwdesc->csar, ddata->base + STM32_DMA3_CSAR(id));
 760         writel_relaxed(hwdesc->cdar, ddata->base + STM32_DMA3_CDAR(id));
 761         writel_relaxed(hwdesc->cllr, ddata->base + STM32_DMA3_CLLR(id));
 762
 763         /* Clear any pending interrupts */
 764         csr = readl_relaxed(ddata->base + STM32_DMA3_CSR(id));
 765         if (csr & CSR_ALL_F)
 766                 writel_relaxed(csr, ddata->base + STM32_DMA3_CFCR(id));
 767
 768         stm32_dma3_chan_dump_reg(chan);
 769
 770         ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(id));
 771         writel_relaxed(ccr | CCR_EN, ddata->base + STM32_DMA3_CCR(id));
 772
 773         chan->dma_status = DMA_IN_PROGRESS;
 774
 775         dev_dbg(chan2dev(chan), "vchan %pK: started\n", &chan->vchan);
 776 }
 777
 778 static int stm32_dma3_chan_suspend(struct stm32_dma3_chan *chan, bool susp)
 779 {
 780         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
 781         u32 csr, ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)) & ~CCR_EN;
 782         int ret = 0;
 783
 784         if (susp)
 785                 ccr |= CCR_SUSP;
 786         else
 787                 ccr &= ~CCR_SUSP;
 788
 789         writel_relaxed(ccr, ddata->base + STM32_DMA3_CCR(chan->id));
 790
 791         if (susp) {
 792                 ret = readl_relaxed_poll_timeout_atomic(ddata->base + STM32_DMA3_CSR(chan->id), csr,
 793                                                         csr & CSR_SUSPF, 1, 10);
 794                 if (!ret)
 795                         writel_relaxed(CFCR_SUSPF, ddata->base + STM32_DMA3_CFCR(chan->id));
 796
 797                 stm32_dma3_chan_dump_reg(chan);
 798         }
 799
 800         return ret;
 801 }
 802
 803 static void stm32_dma3_chan_reset(struct stm32_dma3_chan *chan)
 804 {
 805         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
 806         u32 ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)) & ~CCR_EN;
 807
 808         writel_relaxed(ccr |= CCR_RESET, ddata->base + STM32_DMA3_CCR(chan->id));
 809 }
 810
 811 static int stm32_dma3_chan_get_curr_hwdesc(struct stm32_dma3_swdesc *swdesc, u32 cllr, u32 *residue)
 812 {
 813         u32 i, lli_offset, next_lli_offset = cllr & CLLR_LA;
 814
 815         /* If cllr is null, it means it is either the last or single item */
 816         if (!cllr)
 817                 return swdesc->lli_size - 1;
 818
 819         /* In cyclic mode, go fast and first check we are not on the last item */
 820         if (swdesc->cyclic && next_lli_offset == (swdesc->lli[0].hwdesc_addr & CLLR_LA))
 821                 return swdesc->lli_size - 1;
 822
 823         /* As transfer is in progress, look backward from the last item */
 824         for (i = swdesc->lli_size - 1; i > 0; i--) {
 825                 *residue += FIELD_GET(CBR1_BNDT, swdesc->lli[i].hwdesc->cbr1);
 826                 lli_offset = swdesc->lli[i].hwdesc_addr & CLLR_LA;
 827                 if (lli_offset == next_lli_offset)
 828                         return i - 1;
 829         }
 830
 831         return -EINVAL;
 832 }
 833
 834 static void stm32_dma3_chan_set_residue(struct stm32_dma3_chan *chan,
 835                                         struct stm32_dma3_swdesc *swdesc,
 836                                         struct dma_tx_state *txstate)
 837 {
 838         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
 839         struct device *dev = chan2dev(chan);
 840         struct stm32_dma3_hwdesc *hwdesc;
 841         u32 residue, curr_lli, csr, cdar, cbr1, cllr, bndt, fifol;
 842         bool pack_unpack;
 843         int ret;
 844
 845         csr = readl_relaxed(ddata->base + STM32_DMA3_CSR(chan->id));
 846         if (!(csr & CSR_IDLEF) && chan->dma_status != DMA_PAUSED) {
 847                 /* Suspend current transfer to read registers for a snapshot */
 848                 writel_relaxed(swdesc->ccr | CCR_SUSP, ddata->base + STM32_DMA3_CCR(chan->id));
 849                 ret = readl_relaxed_poll_timeout_atomic(ddata->base + STM32_DMA3_CSR(chan->id), csr,
 850                                                         csr & (CSR_SUSPF | CSR_IDLEF), 1, 10);
 851
 852                 if (ret || ((csr & CSR_TCF) && (csr & CSR_IDLEF))) {
 853                         writel_relaxed(CFCR_SUSPF, ddata->base + STM32_DMA3_CFCR(chan->id));
 854                         writel_relaxed(swdesc->ccr, ddata->base + STM32_DMA3_CCR(chan->id));
 855                         if (ret)
 856                                 dev_err(dev, "Channel suspension timeout, csr=%08x\n", csr);
 857                 }
 858         }
 859
 860         /* If channel is still active (CSR_IDLEF is not set), can't get a reliable residue */
 861         if (!(csr & CSR_IDLEF))
 862                 dev_warn(dev, "Can't get residue: channel still active, csr=%08x\n", csr);
 863
 864         /*
 865          * If channel is not suspended, but Idle and Transfer Complete are set,
 866          * linked-list is over, no residue
 867          */
 868         if (!(csr & CSR_SUSPF) && (csr & CSR_TCF) && (csr & CSR_IDLEF))
 869                 return;
 870
 871         /* Read registers to have a snapshot */
 872         cllr = readl_relaxed(ddata->base + STM32_DMA3_CLLR(chan->id));
 873         cbr1 = readl_relaxed(ddata->base + STM32_DMA3_CBR1(chan->id));
 874         cdar = readl_relaxed(ddata->base + STM32_DMA3_CDAR(chan->id));
 875
 876         /* Resume current transfer */
 877         if (csr & CSR_SUSPF) {
 878                 writel_relaxed(CFCR_SUSPF, ddata->base + STM32_DMA3_CFCR(chan->id));
 879                 writel_relaxed(swdesc->ccr, ddata->base + STM32_DMA3_CCR(chan->id));
 880         }
 881
 882         /* Add current BNDT */
 883         bndt = FIELD_GET(CBR1_BNDT, cbr1);
 884         residue = bndt;
 885
 886         /* Get current hwdesc and cumulate residue of pending hwdesc BNDT */
 887         ret = stm32_dma3_chan_get_curr_hwdesc(swdesc, cllr, &residue);
 888         if (ret < 0) {
 889                 dev_err(chan2dev(chan), "Can't get residue: current hwdesc not found\n");
 890                 return;
 891         }
 892         curr_lli = ret;
 893
 894         /* Read current FIFO level - in units of programmed destination data width */
 895         hwdesc = swdesc->lli[curr_lli].hwdesc;
 896         fifol = FIELD_GET(CSR_FIFOL, csr) * (1 << FIELD_GET(CTR1_DDW_LOG2, hwdesc->ctr1));
 897         /* If the FIFO contains as many bytes as its size, it can't contain more */
 898         if (fifol == (1 << (chan->fifo_size + 1)))
 899                 goto skip_fifol_update;
 900
 901         /*
 902          * In case of PACKING (Destination burst length > Source burst length) or UNPACKING
 903          * (Source burst length > Destination burst length), bytes could be pending in the FIFO
 904          * (to be packed up to Destination burst length or unpacked into Destination burst length
 905          * chunks).
 906          * BNDT is not reliable, as it reflects the number of bytes read from the source but not the
 907          * number of bytes written to the destination.
 908          * FIFOL is also not sufficient, because it reflects the number of available write beats in
 909          * units of Destination data width but not the bytes not yet packed or unpacked.
 910          * In case of Destination increment DINC, it is possible to compute the number of bytes in
 911          * the FIFO:
 912          * fifol_in_bytes = bytes_read - bytes_written.
 913          */
 914         pack_unpack = !!(FIELD_GET(CTR1_PAM, hwdesc->ctr1) == CTR1_PAM_PACK_UNPACK);
 915         if (pack_unpack && (hwdesc->ctr1 & CTR1_DINC)) {
 916                 int bytes_read = FIELD_GET(CBR1_BNDT, hwdesc->cbr1) - bndt;
 917                 int bytes_written = cdar - hwdesc->cdar;
 918
 919                 if (bytes_read > 0)
 920                         fifol = bytes_read - bytes_written;
 921         }
 922
 923 skip_fifol_update:
 924         if (fifol) {
 925                 dev_dbg(chan2dev(chan), "%u byte(s) in the FIFO\n", fifol);
 926                 dma_set_in_flight_bytes(txstate, fifol);
 927                 /*
 928                  * Residue is already accurate for DMA_MEM_TO_DEV as BNDT reflects data read from
 929                  * the source memory buffer, so just need to add fifol to residue in case of
 930                  * DMA_DEV_TO_MEM transfer because these bytes are not yet written in destination
 931                  * memory buffer.
 932                  */
 933                 if (chan->dma_config.direction == DMA_DEV_TO_MEM)
 934                         residue += fifol;
 935         }
 936         dma_set_residue(txstate, residue);
 937 }
 938
 939 static int stm32_dma3_chan_stop(struct stm32_dma3_chan *chan)
 940 {
 941         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
 942         u32 ccr;
 943         int ret = 0;
 944
 945         chan->dma_status = DMA_COMPLETE;
 946
 947         /* Disable interrupts */
 948         ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id));
 949         writel_relaxed(ccr & ~(CCR_ALLIE | CCR_EN), ddata->base + STM32_DMA3_CCR(chan->id));
 950
 951         if (!(ccr & CCR_SUSP) && (ccr & CCR_EN)) {
 952                 /* Suspend the channel */
 953                 ret = stm32_dma3_chan_suspend(chan, true);
 954                 if (ret)
 955                         dev_warn(chan2dev(chan), "%s: timeout, data might be lost\n", __func__);
 956         }
 957
 958         /*
 959          * Reset the channel: this causes the reset of the FIFO and the reset of the channel
 960          * internal state, the reset of CCR_EN and CCR_SUSP bits.
 961          */
 962         stm32_dma3_chan_reset(chan);
 963
 964         return ret;
 965 }
 966
 967 static void stm32_dma3_chan_complete(struct stm32_dma3_chan *chan)
 968 {
 969         if (!chan->swdesc)
 970                 return;
 971
 972         vchan_cookie_complete(&chan->swdesc->vdesc);
 973         chan->swdesc = NULL;
 974         stm32_dma3_chan_start(chan);
 975 }
 976
 977 static irqreturn_t stm32_dma3_chan_irq(int irq, void *devid)
 978 {
 979         struct stm32_dma3_chan *chan = devid;
 980         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
 981         u32 misr, csr, ccr;
 982
 983         spin_lock(&chan->vchan.lock);
 984
 985         misr = readl_relaxed(ddata->base + STM32_DMA3_MISR);
 986         if (!(misr & MISR_MIS(chan->id))) {
 987                 spin_unlock(&chan->vchan.lock);
 988                 return IRQ_NONE;
 989         }
 990
 991         csr = readl_relaxed(ddata->base + STM32_DMA3_CSR(chan->id));
 992         ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)) & CCR_ALLIE;
 993
 994         if (csr & CSR_TCF && ccr & CCR_TCIE) {
 995                 if (chan->swdesc->cyclic)
 996                         vchan_cyclic_callback(&chan->swdesc->vdesc);
 997                 else
 998                         stm32_dma3_chan_complete(chan);
 999         }
1000
1001         if (csr & CSR_USEF && ccr & CCR_USEIE) {
1002                 dev_err(chan2dev(chan), "User setting error\n");
1003                 chan->dma_status = DMA_ERROR;
1004                 /* CCR.EN automatically cleared by HW */
1005                 stm32_dma3_check_user_setting(chan);
1006                 stm32_dma3_chan_reset(chan);
1007         }
1008
1009         if (csr & CSR_ULEF && ccr & CCR_ULEIE) {
1010                 dev_err(chan2dev(chan), "Update link transfer error\n");
1011                 chan->dma_status = DMA_ERROR;
1012                 /* CCR.EN automatically cleared by HW */
1013                 stm32_dma3_chan_reset(chan);
1014         }
1015
1016         if (csr & CSR_DTEF && ccr & CCR_DTEIE) {
1017                 dev_err(chan2dev(chan), "Data transfer error\n");
1018                 chan->dma_status = DMA_ERROR;
1019                 /* CCR.EN automatically cleared by HW */
1020                 stm32_dma3_chan_reset(chan);
1021         }
1022
1023         /*
1024          * Half Transfer Interrupt may be disabled but Half Transfer Flag can be set,
1025          * ensure HTF flag to be cleared, with other flags.
1026          */
1027         csr &= (ccr | CCR_HTIE);
1028
1029         if (csr)
1030                 writel_relaxed(csr, ddata->base + STM32_DMA3_CFCR(chan->id));
1031
1032         spin_unlock(&chan->vchan.lock);
1033
1034         return IRQ_HANDLED;
1035 }
1036
1037 static int stm32_dma3_alloc_chan_resources(struct dma_chan *c)
1038 {
1039         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1040         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
1041         u32 id = chan->id, csemcr, ccid;
1042         int ret;
1043
1044         ret = pm_runtime_resume_and_get(ddata->dma_dev.dev);
1045         if (ret < 0)
1046                 return ret;
1047
1048         /* Ensure the channel is free */
1049         if (chan->semaphore_mode &&
1050             readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(chan->id)) & CSEMCR_SEM_MUTEX) {
1051                 ret = -EBUSY;
1052                 goto err_put_sync;
1053         }
1054
1055         chan->lli_pool = dmam_pool_create(dev_name(&c->dev->device), c->device->dev,
1056                                           sizeof(struct stm32_dma3_hwdesc),
1057                                           __alignof__(struct stm32_dma3_hwdesc), SZ_64K);
1058         if (!chan->lli_pool) {
1059                 dev_err(chan2dev(chan), "Failed to create LLI pool\n");
1060                 ret = -ENOMEM;
1061                 goto err_put_sync;
1062         }
1063
1064         /* Take the channel semaphore */
1065         if (chan->semaphore_mode) {
1066                 writel_relaxed(CSEMCR_SEM_MUTEX, ddata->base + STM32_DMA3_CSEMCR(id));
1067                 csemcr = readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(id));
1068                 ccid = FIELD_GET(CSEMCR_SEM_CCID, csemcr);
1069                 /* Check that the channel is well taken */
1070                 if (ccid != CCIDCFGR_CID1) {
1071                         dev_err(chan2dev(chan), "Not under CID1 control (in-use by CID%d)\n", ccid);
1072                         ret = -EPERM;
1073                         goto err_pool_destroy;
1074                 }
1075                 dev_dbg(chan2dev(chan), "Under CID1 control (semcr=0x%08x)\n", csemcr);
1076         }
1077
1078         return 0;
1079
1080 err_pool_destroy:
1081         dmam_pool_destroy(chan->lli_pool);
1082         chan->lli_pool = NULL;
1083
1084 err_put_sync:
1085         pm_runtime_put_sync(ddata->dma_dev.dev);
1086
1087         return ret;
1088 }
1089
1090 static void stm32_dma3_free_chan_resources(struct dma_chan *c)
1091 {
1092         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1093         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
1094         unsigned long flags;
1095
1096         /* Ensure channel is in idle state */
1097         spin_lock_irqsave(&chan->vchan.lock, flags);
1098         stm32_dma3_chan_stop(chan);
1099         chan->swdesc = NULL;
1100         spin_unlock_irqrestore(&chan->vchan.lock, flags);
1101
1102         vchan_free_chan_resources(to_virt_chan(c));
1103
1104         dmam_pool_destroy(chan->lli_pool);
1105         chan->lli_pool = NULL;
1106
1107         /* Release the channel semaphore */
1108         if (chan->semaphore_mode)
1109                 writel_relaxed(0, ddata->base + STM32_DMA3_CSEMCR(chan->id));
1110
1111         pm_runtime_put_sync(ddata->dma_dev.dev);
1112
1113         /* Reset configuration */
1114         memset(&chan->dt_config, 0, sizeof(chan->dt_config));
1115         memset(&chan->dma_config, 0, sizeof(chan->dma_config));
1116         chan->config_set = 0;
1117 }
1118
1119 static void stm32_dma3_init_chan_config_for_memcpy(struct stm32_dma3_chan *chan,
1120                                                    dma_addr_t dst, dma_addr_t src)
1121 {
1122         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
1123         u32 dw = get_chan_max_dw(ddata->ports_max_dw[0], chan->max_burst); /* port 0 by default */
1124         u32 burst = chan->max_burst / dw;
1125
1126         /* Initialize dt_config if channel not pre-configured through DT */
1127         if (!(chan->config_set & STM32_DMA3_CFG_SET_DT)) {
1128                 chan->dt_config.ch_conf = FIELD_PREP(STM32_DMA3_DT_PRIO, CCR_PRIO_VERY_HIGH);
1129                 chan->dt_config.ch_conf |= FIELD_PREP(STM32_DMA3_DT_FIFO, chan->fifo_size);
1130                 chan->dt_config.tr_conf = STM32_DMA3_DT_SINC | STM32_DMA3_DT_DINC;
1131                 chan->dt_config.tr_conf |= FIELD_PREP(STM32_DMA3_DT_TCEM, CTR2_TCEM_CHANNEL);
1132         }
1133
1134         /* Initialize dma_config if dmaengine_slave_config() not used */
1135         if (!(chan->config_set & STM32_DMA3_CFG_SET_DMA)) {
1136                 chan->dma_config.src_addr_width = dw;
1137                 chan->dma_config.dst_addr_width = dw;
1138                 chan->dma_config.src_maxburst = burst;
1139                 chan->dma_config.dst_maxburst = burst;
1140                 chan->dma_config.src_addr = src;
1141                 chan->dma_config.dst_addr = dst;
1142         }
1143 }
1144
1145 static struct dma_async_tx_descriptor *stm32_dma3_prep_dma_memcpy(struct dma_chan *c,
1146                                                                   dma_addr_t dst, dma_addr_t src,
1147                                                                   size_t len, unsigned long flags)
1148 {
1149         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1150         struct stm32_dma3_swdesc *swdesc;
1151         size_t next_size, offset;
1152         u32 count, i, ctr1, ctr2;
1153
1154         count = DIV_ROUND_UP(len, STM32_DMA3_MAX_BLOCK_SIZE);
1155
1156         swdesc = stm32_dma3_chan_desc_alloc(chan, count);
1157         if (!swdesc)
1158                 return NULL;
1159
1160         if (chan->config_set != STM32_DMA3_CFG_SET_BOTH)
1161                 stm32_dma3_init_chan_config_for_memcpy(chan, dst, src);
1162
1163         for (i = 0, offset = 0; offset < len; i++, offset += next_size) {
1164                 size_t remaining;
1165                 int ret;
1166
1167                 remaining = len - offset;
1168                 next_size = min_t(size_t, remaining, STM32_DMA3_MAX_BLOCK_SIZE);
1169
1170                 ret = stm32_dma3_chan_prep_hw(chan, DMA_MEM_TO_MEM, &swdesc->ccr, &ctr1, &ctr2,
1171                                               src + offset, dst + offset, next_size);
1172                 if (ret)
1173                         goto err_desc_free;
1174
1175                 stm32_dma3_chan_prep_hwdesc(chan, swdesc, i, src + offset, dst + offset, next_size,
1176                                             ctr1, ctr2, next_size == remaining, false);
1177         }
1178
1179         /* Enable Errors interrupts */
1180         swdesc->ccr |= CCR_USEIE | CCR_ULEIE | CCR_DTEIE;
1181         /* Enable Transfer state interrupts */
1182         swdesc->ccr |= CCR_TCIE;
1183
1184         swdesc->cyclic = false;
1185
1186         return vchan_tx_prep(&chan->vchan, &swdesc->vdesc, flags);
1187
1188 err_desc_free:
1189         stm32_dma3_chan_desc_free(chan, swdesc);
1190
1191         return NULL;
1192 }
1193
1194 static struct dma_async_tx_descriptor *stm32_dma3_prep_slave_sg(struct dma_chan *c,
1195                                                                 struct scatterlist *sgl,
1196                                                                 unsigned int sg_len,
1197                                                                 enum dma_transfer_direction dir,
1198                                                                 unsigned long flags, void *context)
1199 {
1200         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1201         struct stm32_dma3_swdesc *swdesc;
1202         struct scatterlist *sg;
1203         size_t len;
1204         dma_addr_t sg_addr, dev_addr, src, dst;
1205         u32 i, j, count, ctr1, ctr2;
1206         int ret;
1207
1208         count = sg_len;
1209         for_each_sg(sgl, sg, sg_len, i) {
1210                 len = sg_dma_len(sg);
1211                 if (len > STM32_DMA3_MAX_BLOCK_SIZE)
1212                         count += DIV_ROUND_UP(len, STM32_DMA3_MAX_BLOCK_SIZE) - 1;
1213         }
1214
1215         swdesc = stm32_dma3_chan_desc_alloc(chan, count);
1216         if (!swdesc)
1217                 return NULL;
1218
1219         /* sg_len and i correspond to the initial sgl; count and j correspond to the hwdesc LL */
1220         j = 0;
1221         for_each_sg(sgl, sg, sg_len, i) {
1222                 sg_addr = sg_dma_address(sg);
1223                 dev_addr = (dir == DMA_MEM_TO_DEV) ? chan->dma_config.dst_addr :
1224                                                      chan->dma_config.src_addr;
1225                 len = sg_dma_len(sg);
1226
1227                 do {
1228                         size_t chunk = min_t(size_t, len, STM32_DMA3_MAX_BLOCK_SIZE);
1229
1230                         if (dir == DMA_MEM_TO_DEV) {
1231                                 src = sg_addr;
1232                                 dst = dev_addr;
1233
1234                                 ret = stm32_dma3_chan_prep_hw(chan, dir, &swdesc->ccr, &ctr1, &ctr2,
1235                                                               src, dst, chunk);
1236
1237                                 if (FIELD_GET(CTR1_DINC, ctr1))
1238                                         dev_addr += chunk;
1239                         } else { /* (dir == DMA_DEV_TO_MEM || dir == DMA_MEM_TO_MEM) */
1240                                 src = dev_addr;
1241                                 dst = sg_addr;
1242
1243                                 ret = stm32_dma3_chan_prep_hw(chan, dir, &swdesc->ccr, &ctr1, &ctr2,
1244                                                               src, dst, chunk);
1245
1246                                 if (FIELD_GET(CTR1_SINC, ctr1))
1247                                         dev_addr += chunk;
1248                         }
1249
1250                         if (ret)
1251                                 goto err_desc_free;
1252
1253                         stm32_dma3_chan_prep_hwdesc(chan, swdesc, j, src, dst, chunk,
1254                                                     ctr1, ctr2, j == (count - 1), false);
1255
1256                         sg_addr += chunk;
1257                         len -= chunk;
1258                         j++;
1259                 } while (len);
1260         }
1261
1262         /* Enable Error interrupts */
1263         swdesc->ccr |= CCR_USEIE | CCR_ULEIE | CCR_DTEIE;
1264         /* Enable Transfer state interrupts */
1265         swdesc->ccr |= CCR_TCIE;
1266
1267         swdesc->cyclic = false;
1268
1269         return vchan_tx_prep(&chan->vchan, &swdesc->vdesc, flags);
1270
1271 err_desc_free:
1272         stm32_dma3_chan_desc_free(chan, swdesc);
1273
1274         return NULL;
1275 }
1276
1277 static struct dma_async_tx_descriptor *stm32_dma3_prep_dma_cyclic(struct dma_chan *c,
1278                                                                   dma_addr_t buf_addr,
1279                                                                   size_t buf_len, size_t period_len,
1280                                                                   enum dma_transfer_direction dir,
1281                                                                   unsigned long flags)
1282 {
1283         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1284         struct stm32_dma3_swdesc *swdesc;
1285         dma_addr_t src, dst;
1286         u32 count, i, ctr1, ctr2;
1287         int ret;
1288
1289         if (!buf_len || !period_len || period_len > STM32_DMA3_MAX_BLOCK_SIZE) {
1290                 dev_err(chan2dev(chan), "Invalid buffer/period length\n");
1291                 return NULL;
1292         }
1293
1294         if (buf_len % period_len) {
1295                 dev_err(chan2dev(chan), "Buffer length not multiple of period length\n");
1296                 return NULL;
1297         }
1298
1299         count = buf_len / period_len;
1300         swdesc = stm32_dma3_chan_desc_alloc(chan, count);
1301         if (!swdesc)
1302                 return NULL;
1303
1304         if (dir == DMA_MEM_TO_DEV) {
1305                 src = buf_addr;
1306                 dst = chan->dma_config.dst_addr;
1307
1308                 ret = stm32_dma3_chan_prep_hw(chan, DMA_MEM_TO_DEV, &swdesc->ccr, &ctr1, &ctr2,
1309                                               src, dst, period_len);
1310         } else if (dir == DMA_DEV_TO_MEM) {
1311                 src = chan->dma_config.src_addr;
1312                 dst = buf_addr;
1313
1314                 ret = stm32_dma3_chan_prep_hw(chan, DMA_DEV_TO_MEM, &swdesc->ccr, &ctr1, &ctr2,
1315                                               src, dst, period_len);
1316         } else {
1317                 dev_err(chan2dev(chan), "Invalid direction\n");
1318                 ret = -EINVAL;
1319         }
1320
1321         if (ret)
1322                 goto err_desc_free;
1323
1324         for (i = 0; i < count; i++) {
1325                 if (dir == DMA_MEM_TO_DEV) {
1326                         src = buf_addr + i * period_len;
1327                         dst = chan->dma_config.dst_addr;
1328                 } else { /* (dir == DMA_DEV_TO_MEM) */
1329                         src = chan->dma_config.src_addr;
1330                         dst = buf_addr + i * period_len;
1331                 }
1332
1333                 stm32_dma3_chan_prep_hwdesc(chan, swdesc, i, src, dst, period_len,
1334                                             ctr1, ctr2, i == (count - 1), true);
1335         }
1336
1337         /* Enable Error interrupts */
1338         swdesc->ccr |= CCR_USEIE | CCR_ULEIE | CCR_DTEIE;
1339         /* Enable Transfer state interrupts */
1340         swdesc->ccr |= CCR_TCIE;
1341
1342         swdesc->cyclic = true;
1343
1344         return vchan_tx_prep(&chan->vchan, &swdesc->vdesc, flags);
1345
1346 err_desc_free:
1347         stm32_dma3_chan_desc_free(chan, swdesc);
1348
1349         return NULL;
1350 }
1351
1352 static void stm32_dma3_caps(struct dma_chan *c, struct dma_slave_caps *caps)
1353 {
1354         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1355
1356         if (!chan->fifo_size) {
1357                 caps->max_burst = 0;
1358                 caps->src_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1359                 caps->dst_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1360         } else {
1361                 /* Burst transfer should not exceed half of the fifo size */
1362                 caps->max_burst = chan->max_burst;
1363                 if (caps->max_burst < DMA_SLAVE_BUSWIDTH_8_BYTES) {
1364                         caps->src_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1365                         caps->dst_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1366                 }
1367         }
1368 }
1369
1370 static int stm32_dma3_config(struct dma_chan *c, struct dma_slave_config *config)
1371 {
1372         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1373
1374         memcpy(&chan->dma_config, config, sizeof(*config));
1375         chan->config_set |= STM32_DMA3_CFG_SET_DMA;
1376
1377         return 0;
1378 }
1379
1380 static int stm32_dma3_pause(struct dma_chan *c)
1381 {
1382         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1383         int ret;
1384
1385         ret = stm32_dma3_chan_suspend(chan, true);
1386         if (ret)
1387                 return ret;
1388
1389         chan->dma_status = DMA_PAUSED;
1390
1391         dev_dbg(chan2dev(chan), "vchan %pK: paused\n", &chan->vchan);
1392
1393         return 0;
1394 }
1395
1396 static int stm32_dma3_resume(struct dma_chan *c)
1397 {
1398         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1399
1400         stm32_dma3_chan_suspend(chan, false);
1401
1402         chan->dma_status = DMA_IN_PROGRESS;
1403
1404         dev_dbg(chan2dev(chan), "vchan %pK: resumed\n", &chan->vchan);
1405
1406         return 0;
1407 }
1408
1409 static int stm32_dma3_terminate_all(struct dma_chan *c)
1410 {
1411         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1412         unsigned long flags;
1413         LIST_HEAD(head);
1414
1415         spin_lock_irqsave(&chan->vchan.lock, flags);
1416
1417         if (chan->swdesc) {
1418                 vchan_terminate_vdesc(&chan->swdesc->vdesc);
1419                 chan->swdesc = NULL;
1420         }
1421
1422         stm32_dma3_chan_stop(chan);
1423
1424         vchan_get_all_descriptors(&chan->vchan, &head);
1425
1426         spin_unlock_irqrestore(&chan->vchan.lock, flags);
1427         vchan_dma_desc_free_list(&chan->vchan, &head);
1428
1429         dev_dbg(chan2dev(chan), "vchan %pK: terminated\n", &chan->vchan);
1430
1431         return 0;
1432 }
1433
1434 static void stm32_dma3_synchronize(struct dma_chan *c)
1435 {
1436         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1437
1438         vchan_synchronize(&chan->vchan);
1439 }
1440
1441 static enum dma_status stm32_dma3_tx_status(struct dma_chan *c, dma_cookie_t cookie,
1442                                             struct dma_tx_state *txstate)
1443 {
1444         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1445         struct stm32_dma3_swdesc *swdesc = NULL;
1446         enum dma_status status;
1447         unsigned long flags;
1448         struct virt_dma_desc *vd;
1449
1450         status = dma_cookie_status(c, cookie, txstate);
1451         if (status == DMA_COMPLETE)
1452                 return status;
1453
1454         if (!txstate)
1455                 return chan->dma_status;
1456
1457         spin_lock_irqsave(&chan->vchan.lock, flags);
1458
1459         vd = vchan_find_desc(&chan->vchan, cookie);
1460         if (vd)
1461                 swdesc = to_stm32_dma3_swdesc(vd);
1462         else if (chan->swdesc && chan->swdesc->vdesc.tx.cookie == cookie)
1463                 swdesc = chan->swdesc;
1464
1465         /* Get residue/in_flight_bytes only if a transfer is currently running (swdesc != NULL) */
1466         if (swdesc)
1467                 stm32_dma3_chan_set_residue(chan, swdesc, txstate);
1468
1469         spin_unlock_irqrestore(&chan->vchan.lock, flags);
1470
1471         return chan->dma_status;
1472 }
1473
1474 static void stm32_dma3_issue_pending(struct dma_chan *c)
1475 {
1476         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1477         unsigned long flags;
1478
1479         spin_lock_irqsave(&chan->vchan.lock, flags);
1480
1481         if (vchan_issue_pending(&chan->vchan) && !chan->swdesc) {
1482                 dev_dbg(chan2dev(chan), "vchan %pK: issued\n", &chan->vchan);
1483                 stm32_dma3_chan_start(chan);
1484         }
1485
1486         spin_unlock_irqrestore(&chan->vchan.lock, flags);
1487 }
1488
1489 static bool stm32_dma3_filter_fn(struct dma_chan *c, void *fn_param)
1490 {
1491         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1492         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
1493         struct stm32_dma3_dt_conf *conf = fn_param;
1494         u32 mask, semcr;
1495         int ret;
1496
1497         dev_dbg(c->device->dev, "%s(%s): req_line=%d ch_conf=%08x tr_conf=%08x\n",
1498                 __func__, dma_chan_name(c), conf->req_line, conf->ch_conf, conf->tr_conf);
1499
1500         if (!of_property_read_u32(c->device->dev->of_node, "dma-channel-mask", &mask))
1501                 if (!(mask & BIT(chan->id)))
1502                         return false;
1503
1504         ret = pm_runtime_resume_and_get(ddata->dma_dev.dev);
1505         if (ret < 0)
1506                 return false;
1507         semcr = readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(chan->id));
1508         pm_runtime_put_sync(ddata->dma_dev.dev);
1509
1510         /* Check if chan is free */
1511         if (semcr & CSEMCR_SEM_MUTEX)
1512                 return false;
1513
1514         /* Check if chan fifo fits well */
1515         if (FIELD_GET(STM32_DMA3_DT_FIFO, conf->ch_conf) != chan->fifo_size)
1516                 return false;
1517
1518         return true;
1519 }
1520
1521 static struct dma_chan *stm32_dma3_of_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma)
1522 {
1523         struct stm32_dma3_ddata *ddata = ofdma->of_dma_data;
1524         dma_cap_mask_t mask = ddata->dma_dev.cap_mask;
1525         struct stm32_dma3_dt_conf conf;
1526         struct stm32_dma3_chan *chan;
1527         struct dma_chan *c;
1528
1529         if (dma_spec->args_count < 3) {
1530                 dev_err(ddata->dma_dev.dev, "Invalid args count\n");
1531                 return NULL;
1532         }
1533
1534         conf.req_line = dma_spec->args[0];
1535         conf.ch_conf = dma_spec->args[1];
1536         conf.tr_conf = dma_spec->args[2];
1537
1538         if (conf.req_line >= ddata->dma_requests) {
1539                 dev_err(ddata->dma_dev.dev, "Invalid request line\n");
1540                 return NULL;
1541         }
1542
1543         /* Request dma channel among the generic dma controller list */
1544         c = dma_request_channel(mask, stm32_dma3_filter_fn, &conf);
1545         if (!c) {
1546                 dev_err(ddata->dma_dev.dev, "No suitable channel found\n");
1547                 return NULL;
1548         }
1549
1550         chan = to_stm32_dma3_chan(c);
1551         chan->dt_config = conf;
1552         chan->config_set |= STM32_DMA3_CFG_SET_DT;
1553
1554         return c;
1555 }
1556
1557 static u32 stm32_dma3_check_rif(struct stm32_dma3_ddata *ddata)
1558 {
1559         u32 chan_reserved, mask = 0, i, ccidcfgr, invalid_cid = 0;
1560
1561         /* Reserve Secure channels */
1562         chan_reserved = readl_relaxed(ddata->base + STM32_DMA3_SECCFGR);
1563
1564         /*
1565          * CID filtering must be configured to ensure that the DMA3 channel will inherit the CID of
1566          * the processor which is configuring and using the given channel.
1567          * In case CID filtering is not configured, dma-channel-mask property can be used to
1568          * specify available DMA channels to the kernel.
1569          */
1570         of_property_read_u32(ddata->dma_dev.dev->of_node, "dma-channel-mask", &mask);
1571
1572         /* Reserve !CID-filtered not in dma-channel-mask, static CID != CID1, CID1 not allowed */
1573         for (i = 0; i < ddata->dma_channels; i++) {
1574                 ccidcfgr = readl_relaxed(ddata->base + STM32_DMA3_CCIDCFGR(i));
1575
1576                 if (!(ccidcfgr & CCIDCFGR_CFEN)) { /* !CID-filtered */
1577                         invalid_cid |= BIT(i);
1578                         if (!(mask & BIT(i))) /* Not in dma-channel-mask */
1579                                 chan_reserved |= BIT(i);
1580                 } else { /* CID-filtered */
1581                         if (!(ccidcfgr & CCIDCFGR_SEM_EN)) { /* Static CID mode */
1582                                 if (FIELD_GET(CCIDCFGR_SCID, ccidcfgr) != CCIDCFGR_CID1)
1583                                         chan_reserved |= BIT(i);
1584                         } else { /* Semaphore mode */
1585                                 if (!FIELD_GET(CCIDCFGR_SEM_WLIST_CID1, ccidcfgr))
1586                                         chan_reserved |= BIT(i);
1587                                 ddata->chans[i].semaphore_mode = true;
1588                         }
1589                 }
1590                 dev_dbg(ddata->dma_dev.dev, "chan%d: %s mode, %s\n", i,
1591                         !(ccidcfgr & CCIDCFGR_CFEN) ? "!CID-filtered" :
1592                         ddata->chans[i].semaphore_mode ? "Semaphore" : "Static CID",
1593                         (chan_reserved & BIT(i)) ? "denied" :
1594                         mask & BIT(i) ? "force allowed" : "allowed");
1595         }
1596
1597         if (invalid_cid)
1598                 dev_warn(ddata->dma_dev.dev, "chan%*pbl have invalid CID configuration\n",
1599                          ddata->dma_channels, &invalid_cid);
1600
1601         return chan_reserved;
1602 }
1603
1604 static const struct of_device_id stm32_dma3_of_match[] = {
1605         { .compatible = "st,stm32mp25-dma3", },
1606         { /* sentinel */ },
1607 };
1608 MODULE_DEVICE_TABLE(of, stm32_dma3_of_match);
1609
1610 static int stm32_dma3_probe(struct platform_device *pdev)
1611 {
1612         struct device_node *np = pdev->dev.of_node;
1613         struct stm32_dma3_ddata *ddata;
1614         struct reset_control *reset;
1615         struct stm32_dma3_chan *chan;
1616         struct dma_device *dma_dev;
1617         u32 master_ports, chan_reserved, i, verr;
1618         u64 hwcfgr;
1619         int ret;
1620
1621         ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
1622         if (!ddata)
1623                 return -ENOMEM;
1624         platform_set_drvdata(pdev, ddata);
1625
1626         dma_dev = &ddata->dma_dev;
1627
1628         ddata->base = devm_platform_ioremap_resource(pdev, 0);
1629         if (IS_ERR(ddata->base))
1630                 return PTR_ERR(ddata->base);
1631
1632         ddata->clk = devm_clk_get(&pdev->dev, NULL);
1633         if (IS_ERR(ddata->clk))
1634                 return dev_err_probe(&pdev->dev, PTR_ERR(ddata->clk), "Failed to get clk\n");
1635
1636         reset = devm_reset_control_get_optional(&pdev->dev, NULL);
1637         if (IS_ERR(reset))
1638                 return dev_err_probe(&pdev->dev, PTR_ERR(reset), "Failed to get reset\n");
1639
1640         ret = clk_prepare_enable(ddata->clk);
1641         if (ret)
1642                 return dev_err_probe(&pdev->dev, ret, "Failed to enable clk\n");
1643
1644         reset_control_reset(reset);
1645
1646         INIT_LIST_HEAD(&dma_dev->channels);
1647
1648         dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);
1649         dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask);
1650         dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask);
1651         dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
1652         dma_dev->dev = &pdev->dev;
1653         /*
1654          * This controller supports up to 8-byte buswidth depending on the port used and the
1655          * channel, and can only access address at even boundaries, multiple of the buswidth.
1656          */
1657         dma_dev->copy_align = DMAENGINE_ALIGN_8_BYTES;
1658         dma_dev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1659                                    BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1660                                    BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1661                                    BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1662         dma_dev->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1663                                    BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1664                                    BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1665                                    BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1666         dma_dev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) | BIT(DMA_MEM_TO_MEM);
1667
1668         dma_dev->descriptor_reuse = true;
1669         dma_dev->max_sg_burst = STM32_DMA3_MAX_SEG_SIZE;
1670         dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1671         dma_dev->device_alloc_chan_resources = stm32_dma3_alloc_chan_resources;
1672         dma_dev->device_free_chan_resources = stm32_dma3_free_chan_resources;
1673         dma_dev->device_prep_dma_memcpy = stm32_dma3_prep_dma_memcpy;
1674         dma_dev->device_prep_slave_sg = stm32_dma3_prep_slave_sg;
1675         dma_dev->device_prep_dma_cyclic = stm32_dma3_prep_dma_cyclic;
1676         dma_dev->device_caps = stm32_dma3_caps;
1677         dma_dev->device_config = stm32_dma3_config;
1678         dma_dev->device_pause = stm32_dma3_pause;
1679         dma_dev->device_resume = stm32_dma3_resume;
1680         dma_dev->device_terminate_all = stm32_dma3_terminate_all;
1681         dma_dev->device_synchronize = stm32_dma3_synchronize;
1682         dma_dev->device_tx_status = stm32_dma3_tx_status;
1683         dma_dev->device_issue_pending = stm32_dma3_issue_pending;
1684
1685         /* if dma_channels is not modified, get it from hwcfgr1 */
1686         if (of_property_read_u32(np, "dma-channels", &ddata->dma_channels)) {
1687                 hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR1);
1688                 ddata->dma_channels = FIELD_GET(G_NUM_CHANNELS, hwcfgr);
1689         }
1690
1691         /* if dma_requests is not modified, get it from hwcfgr2 */
1692         if (of_property_read_u32(np, "dma-requests", &ddata->dma_requests)) {
1693                 hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR2);
1694                 ddata->dma_requests = FIELD_GET(G_MAX_REQ_ID, hwcfgr) + 1;
1695         }
1696
1697         /* G_MASTER_PORTS, G_M0_DATA_WIDTH_ENC, G_M1_DATA_WIDTH_ENC in HWCFGR1 */
1698         hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR1);
1699         master_ports = FIELD_GET(G_MASTER_PORTS, hwcfgr);
1700
1701         ddata->ports_max_dw[0] = FIELD_GET(G_M0_DATA_WIDTH_ENC, hwcfgr);
1702         if (master_ports == AXI64 || master_ports == AHB32) /* Single master port */
1703                 ddata->ports_max_dw[1] = DW_INVALID;
1704         else /* Dual master ports */
1705                 ddata->ports_max_dw[1] = FIELD_GET(G_M1_DATA_WIDTH_ENC, hwcfgr);
1706
1707         ddata->chans = devm_kcalloc(&pdev->dev, ddata->dma_channels, sizeof(*ddata->chans),
1708                                     GFP_KERNEL);
1709         if (!ddata->chans) {
1710                 ret = -ENOMEM;
1711                 goto err_clk_disable;
1712         }
1713
1714         chan_reserved = stm32_dma3_check_rif(ddata);
1715
1716         if (chan_reserved == GENMASK(ddata->dma_channels - 1, 0)) {
1717                 ret = -ENODEV;
1718                 dev_err_probe(&pdev->dev, ret, "No channel available, abort registration\n");
1719                 goto err_clk_disable;
1720         }
1721
1722         /* G_FIFO_SIZE x=0..7 in HWCFGR3 and G_FIFO_SIZE x=8..15 in HWCFGR4 */
1723         hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR3);
1724         hwcfgr |= ((u64)readl_relaxed(ddata->base + STM32_DMA3_HWCFGR4)) << 32;
1725
1726         for (i = 0; i < ddata->dma_channels; i++) {
1727                 if (chan_reserved & BIT(i))
1728                         continue;
1729
1730                 chan = &ddata->chans[i];
1731                 chan->id = i;
1732                 chan->fifo_size = get_chan_hwcfg(i, G_FIFO_SIZE(i), hwcfgr);
1733                 /* If chan->fifo_size > 0 then half of the fifo size, else no burst when no FIFO */
1734                 chan->max_burst = (chan->fifo_size) ? (1 << (chan->fifo_size + 1)) / 2 : 0;
1735         }
1736
1737         ret = dmaenginem_async_device_register(dma_dev);
1738         if (ret)
1739                 goto err_clk_disable;
1740
1741         for (i = 0; i < ddata->dma_channels; i++) {
1742                 char name[12];
1743
1744                 if (chan_reserved & BIT(i))
1745                         continue;
1746
1747                 chan = &ddata->chans[i];
1748                 snprintf(name, sizeof(name), "dma%dchan%d", ddata->dma_dev.dev_id, chan->id);
1749
1750                 chan->vchan.desc_free = stm32_dma3_chan_vdesc_free;
1751                 vchan_init(&chan->vchan, dma_dev);
1752
1753                 ret = dma_async_device_channel_register(&ddata->dma_dev, &chan->vchan.chan, name);
1754                 if (ret) {
1755                         dev_err_probe(&pdev->dev, ret, "Failed to register channel %s\n", name);
1756                         goto err_clk_disable;
1757                 }
1758
1759                 ret = platform_get_irq(pdev, i);
1760                 if (ret < 0)
1761                         goto err_clk_disable;
1762                 chan->irq = ret;
1763
1764                 ret = devm_request_irq(&pdev->dev, chan->irq, stm32_dma3_chan_irq, 0,
1765                                        dev_name(chan2dev(chan)), chan);
1766                 if (ret) {
1767                         dev_err_probe(&pdev->dev, ret, "Failed to request channel %s IRQ\n",
1768                                       dev_name(chan2dev(chan)));
1769                         goto err_clk_disable;
1770                 }
1771         }
1772
1773         ret = of_dma_controller_register(np, stm32_dma3_of_xlate, ddata);
1774         if (ret) {
1775                 dev_err_probe(&pdev->dev, ret, "Failed to register controller\n");
1776                 goto err_clk_disable;
1777         }
1778
1779         verr = readl_relaxed(ddata->base + STM32_DMA3_VERR);
1780
1781         pm_runtime_set_active(&pdev->dev);
1782         pm_runtime_enable(&pdev->dev);
1783         pm_runtime_get_noresume(&pdev->dev);
1784         pm_runtime_put(&pdev->dev);
1785
1786         dev_info(&pdev->dev, "STM32 DMA3 registered rev:%lu.%lu\n",
1787                  FIELD_GET(VERR_MAJREV, verr), FIELD_GET(VERR_MINREV, verr));
1788
1789         return 0;
1790
1791 err_clk_disable:
1792         clk_disable_unprepare(ddata->clk);
1793
1794         return ret;
1795 }
1796
1797 static void stm32_dma3_remove(struct platform_device *pdev)
1798 {
1799         pm_runtime_disable(&pdev->dev);
1800 }
1801
1802 static int stm32_dma3_runtime_suspend(struct device *dev)
1803 {
1804         struct stm32_dma3_ddata *ddata = dev_get_drvdata(dev);
1805
1806         clk_disable_unprepare(ddata->clk);
1807
1808         return 0;
1809 }
1810
1811 static int stm32_dma3_runtime_resume(struct device *dev)
1812 {
1813         struct stm32_dma3_ddata *ddata = dev_get_drvdata(dev);
1814         int ret;
1815
1816         ret = clk_prepare_enable(ddata->clk);
1817         if (ret)
1818                 dev_err(dev, "Failed to enable clk: %d\n", ret);
1819
1820         return ret;
1821 }
1822
1823 static const struct dev_pm_ops stm32_dma3_pm_ops = {
1824         SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)
1825         RUNTIME_PM_OPS(stm32_dma3_runtime_suspend, stm32_dma3_runtime_resume, NULL)
1826 };
1827
1828 static struct platform_driver stm32_dma3_driver = {
1829         .probe = stm32_dma3_probe,
1830         .remove_new = stm32_dma3_remove,
1831         .driver = {
1832                 .name = "stm32-dma3",
1833                 .of_match_table = stm32_dma3_of_match,
1834                 .pm = pm_ptr(&stm32_dma3_pm_ops),
1835         },
1836 };
1837
1838 static int __init stm32_dma3_init(void)
1839 {
1840         return platform_driver_register(&stm32_dma3_driver);
1841 }
1842
1843 subsys_initcall(stm32_dma3_init);
1844
1845 MODULE_DESCRIPTION("STM32 DMA3 controller driver");
1846 MODULE_AUTHOR("Amelie Delaunay <[email protected]>");
1847 MODULE_LICENSE("GPL");