drivers/dma/stm32/stm32-dma3.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * STM32 DMA3 controller driver
   4  *
   5  * Copyright (C) STMicroelectronics 2024
   6  * Author(s): Amelie Delaunay <[email protected]>
   7  */
   8
   9 #include <linux/bitfield.h>
  10 #include <linux/clk.h>
  11 #include <linux/dma-mapping.h>
  12 #include <linux/dmaengine.h>
  13 #include <linux/dmapool.h>
  14 #include <linux/init.h>
  15 #include <linux/iopoll.h>
  16 #include <linux/list.h>
  17 #include <linux/module.h>
  18 #include <linux/of_dma.h>
  19 #include <linux/platform_device.h>
  20 #include <linux/pm_runtime.h>
  21 #include <linux/reset.h>
  22 #include <linux/slab.h>
  23
  24 #include "../virt-dma.h"
  25
  26 #define STM32_DMA3_SECCFGR              0x00
  27 #define STM32_DMA3_PRIVCFGR             0x04
  28 #define STM32_DMA3_RCFGLOCKR            0x08
  29 #define STM32_DMA3_MISR                 0x0c
  30 #define STM32_DMA3_SMISR                0x10
  31
  32 #define STM32_DMA3_CLBAR(x)             (0x50 + 0x80 * (x))
  33 #define STM32_DMA3_CCIDCFGR(x)          (0x54 + 0x80 * (x))
  34 #define STM32_DMA3_CSEMCR(x)            (0x58 + 0x80 * (x))
  35 #define STM32_DMA3_CFCR(x)              (0x5c + 0x80 * (x))
  36 #define STM32_DMA3_CSR(x)               (0x60 + 0x80 * (x))
  37 #define STM32_DMA3_CCR(x)               (0x64 + 0x80 * (x))
  38 #define STM32_DMA3_CTR1(x)              (0x90 + 0x80 * (x))
  39 #define STM32_DMA3_CTR2(x)              (0x94 + 0x80 * (x))
  40 #define STM32_DMA3_CBR1(x)              (0x98 + 0x80 * (x))
  41 #define STM32_DMA3_CSAR(x)              (0x9c + 0x80 * (x))
  42 #define STM32_DMA3_CDAR(x)              (0xa0 + 0x80 * (x))
  43 #define STM32_DMA3_CLLR(x)              (0xcc + 0x80 * (x))
  44
  45 #define STM32_DMA3_HWCFGR13             0xfc0 /* G_PER_CTRL(X) x=8..15 */
  46 #define STM32_DMA3_HWCFGR12             0xfc4 /* G_PER_CTRL(X) x=0..7 */
  47 #define STM32_DMA3_HWCFGR4              0xfe4 /* G_FIFO_SIZE(X) x=8..15 */
  48 #define STM32_DMA3_HWCFGR3              0xfe8 /* G_FIFO_SIZE(X) x=0..7 */
  49 #define STM32_DMA3_HWCFGR2              0xfec /* G_MAX_REQ_ID */
  50 #define STM32_DMA3_HWCFGR1              0xff0 /* G_MASTER_PORTS, G_NUM_CHANNELS, G_Mx_DATA_WIDTH */
  51 #define STM32_DMA3_VERR                 0xff4
  52
  53 /* SECCFGR DMA secure configuration register */
  54 #define SECCFGR_SEC(x)                  BIT(x)
  55
  56 /* MISR DMA non-secure/secure masked interrupt status register */
  57 #define MISR_MIS(x)                     BIT(x)
  58
  59 /* CxLBAR DMA channel x linked_list base address register */
  60 #define CLBAR_LBA                       GENMASK(31, 16)
  61
  62 /* CxCIDCFGR DMA channel x CID register */
  63 #define CCIDCFGR_CFEN                   BIT(0)
  64 #define CCIDCFGR_SEM_EN                 BIT(1)
  65 #define CCIDCFGR_SCID                   GENMASK(5, 4)
  66 #define CCIDCFGR_SEM_WLIST_CID0         BIT(16)
  67 #define CCIDCFGR_SEM_WLIST_CID1         BIT(17)
  68 #define CCIDCFGR_SEM_WLIST_CID2         BIT(18)
  69
  70 enum ccidcfgr_cid {
  71         CCIDCFGR_CID0,
  72         CCIDCFGR_CID1,
  73         CCIDCFGR_CID2,
  74 };
  75
  76 /* CxSEMCR DMA channel x semaphore control register */
  77 #define CSEMCR_SEM_MUTEX                BIT(0)
  78 #define CSEMCR_SEM_CCID                 GENMASK(5, 4)
  79
  80 /* CxFCR DMA channel x flag clear register */
  81 #define CFCR_TCF                        BIT(8)
  82 #define CFCR_HTF                        BIT(9)
  83 #define CFCR_DTEF                       BIT(10)
  84 #define CFCR_ULEF                       BIT(11)
  85 #define CFCR_USEF                       BIT(12)
  86 #define CFCR_SUSPF                      BIT(13)
  87
  88 /* CxSR DMA channel x status register */
  89 #define CSR_IDLEF                       BIT(0)
  90 #define CSR_TCF                         BIT(8)
  91 #define CSR_HTF                         BIT(9)
  92 #define CSR_DTEF                        BIT(10)
  93 #define CSR_ULEF                        BIT(11)
  94 #define CSR_USEF                        BIT(12)
  95 #define CSR_SUSPF                       BIT(13)
  96 #define CSR_ALL_F                       GENMASK(13, 8)
  97 #define CSR_FIFOL                       GENMASK(24, 16)
  98
  99 /* CxCR DMA channel x control register */
 100 #define CCR_EN                          BIT(0)
 101 #define CCR_RESET                       BIT(1)
 102 #define CCR_SUSP                        BIT(2)
 103 #define CCR_TCIE                        BIT(8)
 104 #define CCR_HTIE                        BIT(9)
 105 #define CCR_DTEIE                       BIT(10)
 106 #define CCR_ULEIE                       BIT(11)
 107 #define CCR_USEIE                       BIT(12)
 108 #define CCR_SUSPIE                      BIT(13)
 109 #define CCR_ALLIE                       GENMASK(13, 8)
 110 #define CCR_LSM                         BIT(16)
 111 #define CCR_LAP                         BIT(17)
 112 #define CCR_PRIO                        GENMASK(23, 22)
 113
 114 enum ccr_prio {
 115         CCR_PRIO_LOW,
 116         CCR_PRIO_MID,
 117         CCR_PRIO_HIGH,
 118         CCR_PRIO_VERY_HIGH,
 119 };
 120
 121 /* CxTR1 DMA channel x transfer register 1 */
 122 #define CTR1_SINC                       BIT(3)
 123 #define CTR1_SBL_1                      GENMASK(9, 4)
 124 #define CTR1_DINC                       BIT(19)
 125 #define CTR1_DBL_1                      GENMASK(25, 20)
 126 #define CTR1_SDW_LOG2                   GENMASK(1, 0)
 127 #define CTR1_PAM                        GENMASK(12, 11)
 128 #define CTR1_SAP                        BIT(14)
 129 #define CTR1_DDW_LOG2                   GENMASK(17, 16)
 130 #define CTR1_DAP                        BIT(30)
 131
 132 enum ctr1_dw {
 133         CTR1_DW_BYTE,
 134         CTR1_DW_HWORD,
 135         CTR1_DW_WORD,
 136         CTR1_DW_DWORD, /* Depends on HWCFGR1.G_M0_DATA_WIDTH_ENC and .G_M1_DATA_WIDTH_ENC */
 137 };
 138
 139 enum ctr1_pam {
 140         CTR1_PAM_0S_LT,         /* if DDW > SDW, padded with 0s else left-truncated */
 141         CTR1_PAM_SE_RT,         /* if DDW > SDW, sign extended else right-truncated */
 142         CTR1_PAM_PACK_UNPACK,   /* FIFO queued */
 143 };
 144
 145 /* CxTR2 DMA channel x transfer register 2 */
 146 #define CTR2_REQSEL                     GENMASK(7, 0)
 147 #define CTR2_SWREQ                      BIT(9)
 148 #define CTR2_DREQ                       BIT(10)
 149 #define CTR2_BREQ                       BIT(11)
 150 #define CTR2_PFREQ                      BIT(12)
 151 #define CTR2_TCEM                       GENMASK(31, 30)
 152
 153 enum ctr2_tcem {
 154         CTR2_TCEM_BLOCK,
 155         CTR2_TCEM_REPEAT_BLOCK,
 156         CTR2_TCEM_LLI,
 157         CTR2_TCEM_CHANNEL,
 158 };
 159
 160 /* CxBR1 DMA channel x block register 1 */
 161 #define CBR1_BNDT                       GENMASK(15, 0)
 162
 163 /* CxLLR DMA channel x linked-list address register */
 164 #define CLLR_LA                         GENMASK(15, 2)
 165 #define CLLR_ULL                        BIT(16)
 166 #define CLLR_UDA                        BIT(27)
 167 #define CLLR_USA                        BIT(28)
 168 #define CLLR_UB1                        BIT(29)
 169 #define CLLR_UT2                        BIT(30)
 170 #define CLLR_UT1                        BIT(31)
 171
 172 /* HWCFGR13 DMA hardware configuration register 13 x=8..15 */
 173 /* HWCFGR12 DMA hardware configuration register 12 x=0..7 */
 174 #define G_PER_CTRL(x)                   (ULL(0x1) << (4 * (x)))
 175
 176 /* HWCFGR4 DMA hardware configuration register 4 x=8..15 */
 177 /* HWCFGR3 DMA hardware configuration register 3 x=0..7 */
 178 #define G_FIFO_SIZE(x)                  (ULL(0x7) << (4 * (x)))
 179
 180 #define get_chan_hwcfg(x, mask, reg)    (((reg) & (mask)) >> (4 * (x)))
 181
 182 /* HWCFGR2 DMA hardware configuration register 2 */
 183 #define G_MAX_REQ_ID                    GENMASK(7, 0)
 184
 185 /* HWCFGR1 DMA hardware configuration register 1 */
 186 #define G_MASTER_PORTS                  GENMASK(2, 0)
 187 #define G_NUM_CHANNELS                  GENMASK(12, 8)
 188 #define G_M0_DATA_WIDTH_ENC             GENMASK(25, 24)
 189 #define G_M1_DATA_WIDTH_ENC             GENMASK(29, 28)
 190
 191 enum stm32_dma3_master_ports {
 192         AXI64,          /* 1x AXI: 64-bit port 0 */
 193         AHB32,          /* 1x AHB: 32-bit port 0 */
 194         AHB32_AHB32,    /* 2x AHB: 32-bit port 0 and 32-bit port 1 */
 195         AXI64_AHB32,    /* 1x AXI 64-bit port 0 and 1x AHB 32-bit port 1 */
 196         AXI64_AXI64,    /* 2x AXI: 64-bit port 0 and 64-bit port 1 */
 197         AXI128_AHB32,   /* 1x AXI 128-bit port 0 and 1x AHB 32-bit port 1 */
 198 };
 199
 200 enum stm32_dma3_port_data_width {
 201         DW_32,          /* 32-bit, for AHB */
 202         DW_64,          /* 64-bit, for AXI */
 203         DW_128,         /* 128-bit, for AXI */
 204         DW_INVALID,
 205 };
 206
 207 /* VERR DMA version register */
 208 #define VERR_MINREV                     GENMASK(3, 0)
 209 #define VERR_MAJREV                     GENMASK(7, 4)
 210
 211 /* Device tree */
 212 /* struct stm32_dma3_dt_conf */
 213 /* .ch_conf */
 214 #define STM32_DMA3_DT_PRIO              GENMASK(1, 0) /* CCR_PRIO */
 215 #define STM32_DMA3_DT_FIFO              GENMASK(7, 4)
 216 /* .tr_conf */
 217 #define STM32_DMA3_DT_SINC              BIT(0) /* CTR1_SINC */
 218 #define STM32_DMA3_DT_SAP               BIT(1) /* CTR1_SAP */
 219 #define STM32_DMA3_DT_DINC              BIT(4) /* CTR1_DINC */
 220 #define STM32_DMA3_DT_DAP               BIT(5) /* CTR1_DAP */
 221 #define STM32_DMA3_DT_BREQ              BIT(8) /* CTR2_BREQ */
 222 #define STM32_DMA3_DT_PFREQ             BIT(9) /* CTR2_PFREQ */
 223 #define STM32_DMA3_DT_TCEM              GENMASK(13, 12) /* CTR2_TCEM */
 224 #define STM32_DMA3_DT_NOPACK            BIT(16) /* CTR1_PAM */
 225 #define STM32_DMA3_DT_NOREFACT          BIT(17)
 226
 227 /* struct stm32_dma3_chan .config_set bitfield */
 228 #define STM32_DMA3_CFG_SET_DT           BIT(0)
 229 #define STM32_DMA3_CFG_SET_DMA          BIT(1)
 230 #define STM32_DMA3_CFG_SET_BOTH         (STM32_DMA3_CFG_SET_DT | STM32_DMA3_CFG_SET_DMA)
 231
 232 #define STM32_DMA3_MAX_BLOCK_SIZE       ALIGN_DOWN(CBR1_BNDT, 64)
 233 #define STM32_DMA3_MAX_BURST_LEN        (1 + min_t(u32, FIELD_MAX(CTR1_SBL_1), \
 234                                                         FIELD_MAX(CTR1_DBL_1)))
 235 #define port_is_ahb(maxdw)              ({ typeof(maxdw) (_maxdw) = (maxdw); \
 236                                            ((_maxdw) != DW_INVALID) && ((_maxdw) == DW_32); })
 237 #define port_is_axi(maxdw)              ({ typeof(maxdw) (_maxdw) = (maxdw); \
 238                                            ((_maxdw) != DW_INVALID) && ((_maxdw) != DW_32); })
 239 #define get_chan_max_dw(maxdw, maxburst)((port_is_ahb(maxdw) ||                      \
 240                                           (maxburst) < DMA_SLAVE_BUSWIDTH_8_BYTES) ? \
 241                                          DMA_SLAVE_BUSWIDTH_4_BYTES : DMA_SLAVE_BUSWIDTH_8_BYTES)
 242
 243 /* Static linked-list data structure (depends on update bits UT1/UT2/UB1/USA/UDA/ULL) */
 244 struct stm32_dma3_hwdesc {
 245         u32 ctr1;
 246         u32 ctr2;
 247         u32 cbr1;
 248         u32 csar;
 249         u32 cdar;
 250         u32 cllr;
 251 } __packed __aligned(32);
 252
 253 /*
 254  * CLLR_LA / sizeof(struct stm32_dma3_hwdesc) represents the number of hdwdesc that can be addressed
 255  * by the pointer to the next linked-list data structure. The __aligned forces the 32-byte
 256  * alignment. So use hardcoded 32. Multiplied by the max block size of each item, it represents
 257  * the sg size limitation.
 258  */
 259 #define STM32_DMA3_MAX_SEG_SIZE         ((CLLR_LA / 32) * STM32_DMA3_MAX_BLOCK_SIZE)
 260
 261 /*
 262  * Linked-list items
 263  */
 264 struct stm32_dma3_lli {
 265         struct stm32_dma3_hwdesc *hwdesc;
 266         dma_addr_t hwdesc_addr;
 267 };
 268
 269 struct stm32_dma3_swdesc {
 270         struct virt_dma_desc vdesc;
 271         u32 ccr;
 272         bool cyclic;
 273         u32 lli_size;
 274         struct stm32_dma3_lli lli[] __counted_by(lli_size);
 275 };
 276
 277 struct stm32_dma3_dt_conf {
 278         u32 ch_id;
 279         u32 req_line;
 280         u32 ch_conf;
 281         u32 tr_conf;
 282 };
 283
 284 struct stm32_dma3_chan {
 285         struct virt_dma_chan vchan;
 286         u32 id;
 287         int irq;
 288         u32 fifo_size;
 289         u32 max_burst;
 290         bool semaphore_mode;
 291         struct stm32_dma3_dt_conf dt_config;
 292         struct dma_slave_config dma_config;
 293         u8 config_set;
 294         struct dma_pool *lli_pool;
 295         struct stm32_dma3_swdesc *swdesc;
 296         enum ctr2_tcem tcem;
 297         u32 dma_status;
 298 };
 299
 300 struct stm32_dma3_pdata {
 301         u32 axi_max_burst_len;
 302 };
 303
 304 struct stm32_dma3_ddata {
 305         struct dma_device dma_dev;
 306         void __iomem *base;
 307         struct clk *clk;
 308         struct stm32_dma3_chan *chans;
 309         u32 dma_channels;
 310         u32 dma_requests;
 311         enum stm32_dma3_port_data_width ports_max_dw[2];
 312         u32 axi_max_burst_len;
 313 };
 314
 315 static inline struct stm32_dma3_ddata *to_stm32_dma3_ddata(struct stm32_dma3_chan *chan)
 316 {
 317         return container_of(chan->vchan.chan.device, struct stm32_dma3_ddata, dma_dev);
 318 }
 319
 320 static inline struct stm32_dma3_chan *to_stm32_dma3_chan(struct dma_chan *c)
 321 {
 322         return container_of(c, struct stm32_dma3_chan, vchan.chan);
 323 }
 324
 325 static inline struct stm32_dma3_swdesc *to_stm32_dma3_swdesc(struct virt_dma_desc *vdesc)
 326 {
 327         return container_of(vdesc, struct stm32_dma3_swdesc, vdesc);
 328 }
 329
 330 static struct device *chan2dev(struct stm32_dma3_chan *chan)
 331 {
 332         return &chan->vchan.chan.dev->device;
 333 }
 334
 335 static void stm32_dma3_chan_dump_reg(struct stm32_dma3_chan *chan)
 336 {
 337         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
 338         struct device *dev = chan2dev(chan);
 339         u32 id = chan->id, offset;
 340
 341         offset = STM32_DMA3_SECCFGR;
 342         dev_dbg(dev, "SECCFGR(0x%03x): %08x\n", offset, readl_relaxed(ddata->base + offset));
 343         offset = STM32_DMA3_PRIVCFGR;
 344         dev_dbg(dev, "PRIVCFGR(0x%03x): %08x\n", offset, readl_relaxed(ddata->base + offset));
 345         offset = STM32_DMA3_CCIDCFGR(id);
 346         dev_dbg(dev, "C%dCIDCFGR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
 347         offset = STM32_DMA3_CSEMCR(id);
 348         dev_dbg(dev, "C%dSEMCR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
 349         offset = STM32_DMA3_CSR(id);
 350         dev_dbg(dev, "C%dSR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
 351         offset = STM32_DMA3_CCR(id);
 352         dev_dbg(dev, "C%dCR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
 353         offset = STM32_DMA3_CTR1(id);
 354         dev_dbg(dev, "C%dTR1(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
 355         offset = STM32_DMA3_CTR2(id);
 356         dev_dbg(dev, "C%dTR2(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
 357         offset = STM32_DMA3_CBR1(id);
 358         dev_dbg(dev, "C%dBR1(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
 359         offset = STM32_DMA3_CSAR(id);
 360         dev_dbg(dev, "C%dSAR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
 361         offset = STM32_DMA3_CDAR(id);
 362         dev_dbg(dev, "C%dDAR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
 363         offset = STM32_DMA3_CLLR(id);
 364         dev_dbg(dev, "C%dLLR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
 365         offset = STM32_DMA3_CLBAR(id);
 366         dev_dbg(dev, "C%dLBAR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
 367 }
 368
 369 static void stm32_dma3_chan_dump_hwdesc(struct stm32_dma3_chan *chan,
 370                                         struct stm32_dma3_swdesc *swdesc)
 371 {
 372         struct stm32_dma3_hwdesc *hwdesc;
 373         int i;
 374
 375         for (i = 0; i < swdesc->lli_size; i++) {
 376                 hwdesc = swdesc->lli[i].hwdesc;
 377                 if (i)
 378                         dev_dbg(chan2dev(chan), "V\n");
 379                 dev_dbg(chan2dev(chan), "[%d]@%pad\n", i, &swdesc->lli[i].hwdesc_addr);
 380                 dev_dbg(chan2dev(chan), "| C%dTR1: %08x\n", chan->id, hwdesc->ctr1);
 381                 dev_dbg(chan2dev(chan), "| C%dTR2: %08x\n", chan->id, hwdesc->ctr2);
 382                 dev_dbg(chan2dev(chan), "| C%dBR1: %08x\n", chan->id, hwdesc->cbr1);
 383                 dev_dbg(chan2dev(chan), "| C%dSAR: %08x\n", chan->id, hwdesc->csar);
 384                 dev_dbg(chan2dev(chan), "| C%dDAR: %08x\n", chan->id, hwdesc->cdar);
 385                 dev_dbg(chan2dev(chan), "| C%dLLR: %08x\n", chan->id, hwdesc->cllr);
 386         }
 387
 388         if (swdesc->cyclic) {
 389                 dev_dbg(chan2dev(chan), "|\n");
 390                 dev_dbg(chan2dev(chan), "-->[0]@%pad\n", &swdesc->lli[0].hwdesc_addr);
 391         } else {
 392                 dev_dbg(chan2dev(chan), "X\n");
 393         }
 394 }
 395
 396 static struct stm32_dma3_swdesc *stm32_dma3_chan_desc_alloc(struct stm32_dma3_chan *chan, u32 count)
 397 {
 398         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
 399         struct stm32_dma3_swdesc *swdesc;
 400         int i;
 401
 402         /*
 403          * If the memory to be allocated for the number of hwdesc (6 u32 members but 32-bytes
 404          * aligned) is greater than the maximum address of CLLR_LA, then the last items can't be
 405          * addressed, so abort the allocation.
 406          */
 407         if ((count * 32) > CLLR_LA) {
 408                 dev_err(chan2dev(chan), "Transfer is too big (> %luB)\n", STM32_DMA3_MAX_SEG_SIZE);
 409                 return NULL;
 410         }
 411
 412         swdesc = kzalloc(struct_size(swdesc, lli, count), GFP_NOWAIT);
 413         if (!swdesc)
 414                 return NULL;
 415         swdesc->lli_size = count;
 416
 417         for (i = 0; i < count; i++) {
 418                 swdesc->lli[i].hwdesc = dma_pool_zalloc(chan->lli_pool, GFP_NOWAIT,
 419                                                         &swdesc->lli[i].hwdesc_addr);
 420                 if (!swdesc->lli[i].hwdesc)
 421                         goto err_pool_free;
 422         }
 423         swdesc->ccr = 0;
 424
 425         /* Set LL base address */
 426         writel_relaxed(swdesc->lli[0].hwdesc_addr & CLBAR_LBA,
 427                        ddata->base + STM32_DMA3_CLBAR(chan->id));
 428
 429         /* Set LL allocated port */
 430         swdesc->ccr &= ~CCR_LAP;
 431
 432         return swdesc;
 433
 434 err_pool_free:
 435         dev_err(chan2dev(chan), "Failed to alloc descriptors\n");
 436         while (--i >= 0)
 437                 dma_pool_free(chan->lli_pool, swdesc->lli[i].hwdesc, swdesc->lli[i].hwdesc_addr);
 438         kfree(swdesc);
 439
 440         return NULL;
 441 }
 442
 443 static void stm32_dma3_chan_desc_free(struct stm32_dma3_chan *chan,
 444                                       struct stm32_dma3_swdesc *swdesc)
 445 {
 446         int i;
 447
 448         for (i = 0; i < swdesc->lli_size; i++)
 449                 dma_pool_free(chan->lli_pool, swdesc->lli[i].hwdesc, swdesc->lli[i].hwdesc_addr);
 450
 451         kfree(swdesc);
 452 }
 453
 454 static void stm32_dma3_chan_vdesc_free(struct virt_dma_desc *vdesc)
 455 {
 456         struct stm32_dma3_swdesc *swdesc = to_stm32_dma3_swdesc(vdesc);
 457         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(vdesc->tx.chan);
 458
 459         stm32_dma3_chan_desc_free(chan, swdesc);
 460 }
 461
 462 static void stm32_dma3_check_user_setting(struct stm32_dma3_chan *chan)
 463 {
 464         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
 465         struct device *dev = chan2dev(chan);
 466         u32 ctr1 = readl_relaxed(ddata->base + STM32_DMA3_CTR1(chan->id));
 467         u32 cbr1 = readl_relaxed(ddata->base + STM32_DMA3_CBR1(chan->id));
 468         u32 csar = readl_relaxed(ddata->base + STM32_DMA3_CSAR(chan->id));
 469         u32 cdar = readl_relaxed(ddata->base + STM32_DMA3_CDAR(chan->id));
 470         u32 cllr = readl_relaxed(ddata->base + STM32_DMA3_CLLR(chan->id));
 471         u32 bndt = FIELD_GET(CBR1_BNDT, cbr1);
 472         u32 sdw = 1 << FIELD_GET(CTR1_SDW_LOG2, ctr1);
 473         u32 ddw = 1 << FIELD_GET(CTR1_DDW_LOG2, ctr1);
 474         u32 sap = FIELD_GET(CTR1_SAP, ctr1);
 475         u32 dap = FIELD_GET(CTR1_DAP, ctr1);
 476
 477         if (!bndt && !FIELD_GET(CLLR_UB1, cllr))
 478                 dev_err(dev, "null source block size and no update of this value\n");
 479         if (bndt % sdw)
 480                 dev_err(dev, "source block size not multiple of src data width\n");
 481         if (FIELD_GET(CTR1_PAM, ctr1) == CTR1_PAM_PACK_UNPACK && bndt % ddw)
 482                 dev_err(dev, "(un)packing mode w/ src block size not multiple of dst data width\n");
 483         if (csar % sdw)
 484                 dev_err(dev, "unaligned source address not multiple of src data width\n");
 485         if (cdar % ddw)
 486                 dev_err(dev, "unaligned destination address not multiple of dst data width\n");
 487         if (sdw == DMA_SLAVE_BUSWIDTH_8_BYTES && port_is_ahb(ddata->ports_max_dw[sap]))
 488                 dev_err(dev, "double-word source data width not supported on port %u\n", sap);
 489         if (ddw == DMA_SLAVE_BUSWIDTH_8_BYTES && port_is_ahb(ddata->ports_max_dw[dap]))
 490                 dev_err(dev, "double-word destination data width not supported on port %u\n", dap);
 491 }
 492
 493 static void stm32_dma3_chan_prep_hwdesc(struct stm32_dma3_chan *chan,
 494                                         struct stm32_dma3_swdesc *swdesc,
 495                                         u32 curr, dma_addr_t src, dma_addr_t dst, u32 len,
 496                                         u32 ctr1, u32 ctr2, bool is_last, bool is_cyclic)
 497 {
 498         struct stm32_dma3_hwdesc *hwdesc;
 499         dma_addr_t next_lli;
 500         u32 next = curr + 1;
 501
 502         hwdesc = swdesc->lli[curr].hwdesc;
 503         hwdesc->ctr1 = ctr1;
 504         hwdesc->ctr2 = ctr2;
 505         hwdesc->cbr1 = FIELD_PREP(CBR1_BNDT, len);
 506         hwdesc->csar = src;
 507         hwdesc->cdar = dst;
 508
 509         if (is_last) {
 510                 if (is_cyclic)
 511                         next_lli = swdesc->lli[0].hwdesc_addr;
 512                 else
 513                         next_lli = 0;
 514         } else {
 515                 next_lli = swdesc->lli[next].hwdesc_addr;
 516         }
 517
 518         hwdesc->cllr = 0;
 519         if (next_lli) {
 520                 hwdesc->cllr |= CLLR_UT1 | CLLR_UT2 | CLLR_UB1;
 521                 hwdesc->cllr |= CLLR_USA | CLLR_UDA | CLLR_ULL;
 522                 hwdesc->cllr |= (next_lli & CLLR_LA);
 523         }
 524
 525         /*
 526          * Make sure to flush the CPU's write buffers so that the descriptors are ready to be read
 527          * by DMA3. By explicitly using a write memory barrier here, instead of doing it with writel
 528          * to enable the channel, we avoid an unnecessary barrier in the case where the descriptors
 529          * are reused (DMA_CTRL_REUSE).
 530          */
 531         if (is_last)
 532                 dma_wmb();
 533 }
 534
 535 static enum dma_slave_buswidth stm32_dma3_get_max_dw(u32 chan_max_burst,
 536                                                      enum stm32_dma3_port_data_width port_max_dw,
 537                                                      u32 len, dma_addr_t addr)
 538 {
 539         enum dma_slave_buswidth max_dw = get_chan_max_dw(port_max_dw, chan_max_burst);
 540
 541         /* len and addr must be a multiple of dw */
 542         return 1 << __ffs(len | addr | max_dw);
 543 }
 544
 545 static u32 stm32_dma3_get_max_burst(u32 len, enum dma_slave_buswidth dw,
 546                                     u32 chan_max_burst, u32 bus_max_burst)
 547 {
 548         u32 max_burst = chan_max_burst ? chan_max_burst / dw : 1;
 549
 550         /* len is a multiple of dw, so if len is < chan_max_burst, shorten burst */
 551         if (len < chan_max_burst)
 552                 max_burst = len / dw;
 553
 554         /*
 555          * HW doesn't modify the burst if burst size <= half of the fifo size.
 556          * If len is not a multiple of burst size, last burst is shortened by HW.
 557          * Take care of maximum burst supported on interconnect bus.
 558          */
 559         return min_t(u32, max_burst, bus_max_burst);
 560 }
 561
 562 static int stm32_dma3_chan_prep_hw(struct stm32_dma3_chan *chan, enum dma_transfer_direction dir,
 563                                    u32 *ccr, u32 *ctr1, u32 *ctr2,
 564                                    dma_addr_t src_addr, dma_addr_t dst_addr, u32 len)
 565 {
 566         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
 567         struct dma_device dma_device = ddata->dma_dev;
 568         u32 src_max_burst = STM32_DMA3_MAX_BURST_LEN, dst_max_burst = STM32_DMA3_MAX_BURST_LEN;
 569         u32 sdw, ddw, sbl_max, dbl_max, tcem, init_dw, init_bl_max;
 570         u32 _ctr1 = 0, _ctr2 = 0;
 571         u32 ch_conf = chan->dt_config.ch_conf;
 572         u32 tr_conf = chan->dt_config.tr_conf;
 573         u32 sap = FIELD_GET(STM32_DMA3_DT_SAP, tr_conf), sap_max_dw;
 574         u32 dap = FIELD_GET(STM32_DMA3_DT_DAP, tr_conf), dap_max_dw;
 575
 576         dev_dbg(chan2dev(chan), "%s from %pad to %pad\n",
 577                 dmaengine_get_direction_text(dir), &src_addr, &dst_addr);
 578
 579         sdw = chan->dma_config.src_addr_width ? : get_chan_max_dw(sap, chan->max_burst);
 580         ddw = chan->dma_config.dst_addr_width ? : get_chan_max_dw(dap, chan->max_burst);
 581         sbl_max = chan->dma_config.src_maxburst ? : 1;
 582         dbl_max = chan->dma_config.dst_maxburst ? : 1;
 583
 584         /* Following conditions would raise User Setting Error interrupt */
 585         if (!(dma_device.src_addr_widths & BIT(sdw)) || !(dma_device.dst_addr_widths & BIT(ddw))) {
 586                 dev_err(chan2dev(chan), "Bus width (src=%u, dst=%u) not supported\n", sdw, ddw);
 587                 return -EINVAL;
 588         }
 589
 590         if (ddata->ports_max_dw[1] == DW_INVALID && (sap || dap)) {
 591                 dev_err(chan2dev(chan), "Only one master port, port 1 is not supported\n");
 592                 return -EINVAL;
 593         }
 594
 595         sap_max_dw = ddata->ports_max_dw[sap];
 596         dap_max_dw = ddata->ports_max_dw[dap];
 597         if ((port_is_ahb(sap_max_dw) && sdw == DMA_SLAVE_BUSWIDTH_8_BYTES) ||
 598             (port_is_ahb(dap_max_dw) && ddw == DMA_SLAVE_BUSWIDTH_8_BYTES)) {
 599                 dev_err(chan2dev(chan),
 600                         "8 bytes buswidth (src=%u, dst=%u) not supported on port (sap=%u, dap=%u\n",
 601                         sdw, ddw, sap, dap);
 602                 return -EINVAL;
 603         }
 604
 605         if (FIELD_GET(STM32_DMA3_DT_SINC, tr_conf))
 606                 _ctr1 |= CTR1_SINC;
 607         if (sap)
 608                 _ctr1 |= CTR1_SAP;
 609         if (port_is_axi(sap_max_dw)) /* AXI - apply axi maximum burst limitation */
 610                 src_max_burst = ddata->axi_max_burst_len;
 611         if (FIELD_GET(STM32_DMA3_DT_DINC, tr_conf))
 612                 _ctr1 |= CTR1_DINC;
 613         if (dap)
 614                 _ctr1 |= CTR1_DAP;
 615         if (port_is_axi(dap_max_dw)) /* AXI - apply axi maximum burst limitation */
 616                 dst_max_burst = ddata->axi_max_burst_len;
 617
 618         _ctr2 |= FIELD_PREP(CTR2_REQSEL, chan->dt_config.req_line) & ~CTR2_SWREQ;
 619         if (FIELD_GET(STM32_DMA3_DT_BREQ, tr_conf))
 620                 _ctr2 |= CTR2_BREQ;
 621         if (dir == DMA_DEV_TO_MEM && FIELD_GET(STM32_DMA3_DT_PFREQ, tr_conf))
 622                 _ctr2 |= CTR2_PFREQ;
 623         tcem = FIELD_GET(STM32_DMA3_DT_TCEM, tr_conf);
 624         _ctr2 |= FIELD_PREP(CTR2_TCEM, tcem);
 625
 626         /* Store TCEM to know on which event TC flag occurred */
 627         chan->tcem = tcem;
 628         /* Store direction for residue computation */
 629         chan->dma_config.direction = dir;
 630
 631         switch (dir) {
 632         case DMA_MEM_TO_DEV:
 633                 /* Set destination (device) data width and burst */
 634                 ddw = min_t(u32, ddw, stm32_dma3_get_max_dw(chan->max_burst, dap_max_dw,
 635                                                             len, dst_addr));
 636                 dbl_max = min_t(u32, dbl_max, stm32_dma3_get_max_burst(len, ddw, chan->max_burst,
 637                                                                        dst_max_burst));
 638
 639                 /* Set source (memory) data width and burst */
 640                 sdw = stm32_dma3_get_max_dw(chan->max_burst, sap_max_dw, len, src_addr);
 641                 sbl_max = stm32_dma3_get_max_burst(len, sdw, chan->max_burst, src_max_burst);
 642                 if (!!FIELD_GET(STM32_DMA3_DT_NOPACK, tr_conf)) {
 643                         sdw = ddw;
 644                         sbl_max = dbl_max;
 645                 }
 646
 647                 _ctr1 |= FIELD_PREP(CTR1_SDW_LOG2, ilog2(sdw));
 648                 _ctr1 |= FIELD_PREP(CTR1_SBL_1, sbl_max - 1);
 649                 _ctr1 |= FIELD_PREP(CTR1_DDW_LOG2, ilog2(ddw));
 650                 _ctr1 |= FIELD_PREP(CTR1_DBL_1, dbl_max - 1);
 651
 652                 if (ddw != sdw) {
 653                         _ctr1 |= FIELD_PREP(CTR1_PAM, CTR1_PAM_PACK_UNPACK);
 654                         /* Should never reach this case as ddw is clamped down */
 655                         if (len & (ddw - 1)) {
 656                                 dev_err(chan2dev(chan),
 657                                         "Packing mode is enabled and len is not multiple of ddw");
 658                                 return -EINVAL;
 659                         }
 660                 }
 661
 662                 /* dst = dev */
 663                 _ctr2 |= CTR2_DREQ;
 664
 665                 break;
 666
 667         case DMA_DEV_TO_MEM:
 668                 /* Set source (device) data width and burst */
 669                 sdw = min_t(u32, sdw, stm32_dma3_get_max_dw(chan->max_burst, sap_max_dw,
 670                                                             len, src_addr));
 671                 sbl_max = min_t(u32, sbl_max, stm32_dma3_get_max_burst(len, sdw, chan->max_burst,
 672                                                                        src_max_burst));
 673
 674                 /* Set destination (memory) data width and burst */
 675                 ddw = stm32_dma3_get_max_dw(chan->max_burst, dap_max_dw, len, dst_addr);
 676                 dbl_max = stm32_dma3_get_max_burst(len, ddw, chan->max_burst, dst_max_burst);
 677                 if (!!FIELD_GET(STM32_DMA3_DT_NOPACK, tr_conf) ||
 678                     ((_ctr2 & CTR2_PFREQ) && ddw > sdw)) { /* Packing to wider ddw not supported */
 679                         ddw = sdw;
 680                         dbl_max = sbl_max;
 681                 }
 682
 683                 _ctr1 |= FIELD_PREP(CTR1_SDW_LOG2, ilog2(sdw));
 684                 _ctr1 |= FIELD_PREP(CTR1_SBL_1, sbl_max - 1);
 685                 _ctr1 |= FIELD_PREP(CTR1_DDW_LOG2, ilog2(ddw));
 686                 _ctr1 |= FIELD_PREP(CTR1_DBL_1, dbl_max - 1);
 687
 688                 if (ddw != sdw) {
 689                         _ctr1 |= FIELD_PREP(CTR1_PAM, CTR1_PAM_PACK_UNPACK);
 690                         /* Should never reach this case as ddw is clamped down */
 691                         if (len & (ddw - 1)) {
 692                                 dev_err(chan2dev(chan),
 693                                         "Packing mode is enabled and len is not multiple of ddw\n");
 694                                 return -EINVAL;
 695                         }
 696                 }
 697
 698                 /* dst = mem */
 699                 _ctr2 &= ~CTR2_DREQ;
 700
 701                 break;
 702
 703         case DMA_MEM_TO_MEM:
 704                 /* Set source (memory) data width and burst */
 705                 init_dw = sdw;
 706                 init_bl_max = sbl_max;
 707                 sdw = stm32_dma3_get_max_dw(chan->max_burst, sap_max_dw, len, src_addr);
 708                 sbl_max = stm32_dma3_get_max_burst(len, sdw, chan->max_burst, src_max_burst);
 709                 if (chan->config_set & STM32_DMA3_CFG_SET_DMA) {
 710                         sdw = min_t(u32, init_dw, sdw);
 711                         sbl_max = min_t(u32, init_bl_max, stm32_dma3_get_max_burst(len, sdw,
 712                                                                                    chan->max_burst,
 713                                                                                    src_max_burst));
 714                 }
 715
 716                 /* Set destination (memory) data width and burst */
 717                 init_dw = ddw;
 718                 init_bl_max = dbl_max;
 719                 ddw = stm32_dma3_get_max_dw(chan->max_burst, dap_max_dw, len, dst_addr);
 720                 dbl_max = stm32_dma3_get_max_burst(len, ddw, chan->max_burst, dst_max_burst);
 721                 if (chan->config_set & STM32_DMA3_CFG_SET_DMA) {
 722                         ddw = min_t(u32, init_dw, ddw);
 723                         dbl_max = min_t(u32, init_bl_max, stm32_dma3_get_max_burst(len, ddw,
 724                                                                                    chan->max_burst,
 725                                                                                    dst_max_burst));
 726                 }
 727
 728                 _ctr1 |= FIELD_PREP(CTR1_SDW_LOG2, ilog2(sdw));
 729                 _ctr1 |= FIELD_PREP(CTR1_SBL_1, sbl_max - 1);
 730                 _ctr1 |= FIELD_PREP(CTR1_DDW_LOG2, ilog2(ddw));
 731                 _ctr1 |= FIELD_PREP(CTR1_DBL_1, dbl_max - 1);
 732
 733                 if (ddw != sdw) {
 734                         _ctr1 |= FIELD_PREP(CTR1_PAM, CTR1_PAM_PACK_UNPACK);
 735                         /* Should never reach this case as ddw is clamped down */
 736                         if (len & (ddw - 1)) {
 737                                 dev_err(chan2dev(chan),
 738                                         "Packing mode is enabled and len is not multiple of ddw");
 739                                 return -EINVAL;
 740                         }
 741                 }
 742
 743                 /* CTR2_REQSEL/DREQ/BREQ/PFREQ are ignored with CTR2_SWREQ=1 */
 744                 _ctr2 |= CTR2_SWREQ;
 745
 746                 break;
 747
 748         default:
 749                 dev_err(chan2dev(chan), "Direction %s not supported\n",
 750                         dmaengine_get_direction_text(dir));
 751                 return -EINVAL;
 752         }
 753
 754         *ccr |= FIELD_PREP(CCR_PRIO, FIELD_GET(STM32_DMA3_DT_PRIO, ch_conf));
 755         *ctr1 = _ctr1;
 756         *ctr2 = _ctr2;
 757
 758         dev_dbg(chan2dev(chan), "%s: sdw=%u bytes sbl=%u beats ddw=%u bytes dbl=%u beats\n",
 759                 __func__, sdw, sbl_max, ddw, dbl_max);
 760
 761         return 0;
 762 }
 763
 764 static void stm32_dma3_chan_start(struct stm32_dma3_chan *chan)
 765 {
 766         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
 767         struct virt_dma_desc *vdesc;
 768         struct stm32_dma3_hwdesc *hwdesc;
 769         u32 id = chan->id;
 770         u32 csr, ccr;
 771
 772         vdesc = vchan_next_desc(&chan->vchan);
 773         if (!vdesc) {
 774                 chan->swdesc = NULL;
 775                 return;
 776         }
 777         list_del(&vdesc->node);
 778
 779         chan->swdesc = to_stm32_dma3_swdesc(vdesc);
 780         hwdesc = chan->swdesc->lli[0].hwdesc;
 781
 782         stm32_dma3_chan_dump_hwdesc(chan, chan->swdesc);
 783
 784         writel_relaxed(chan->swdesc->ccr, ddata->base + STM32_DMA3_CCR(id));
 785         writel_relaxed(hwdesc->ctr1, ddata->base + STM32_DMA3_CTR1(id));
 786         writel_relaxed(hwdesc->ctr2, ddata->base + STM32_DMA3_CTR2(id));
 787         writel_relaxed(hwdesc->cbr1, ddata->base + STM32_DMA3_CBR1(id));
 788         writel_relaxed(hwdesc->csar, ddata->base + STM32_DMA3_CSAR(id));
 789         writel_relaxed(hwdesc->cdar, ddata->base + STM32_DMA3_CDAR(id));
 790         writel_relaxed(hwdesc->cllr, ddata->base + STM32_DMA3_CLLR(id));
 791
 792         /* Clear any pending interrupts */
 793         csr = readl_relaxed(ddata->base + STM32_DMA3_CSR(id));
 794         if (csr & CSR_ALL_F)
 795                 writel_relaxed(csr, ddata->base + STM32_DMA3_CFCR(id));
 796
 797         stm32_dma3_chan_dump_reg(chan);
 798
 799         ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(id));
 800         writel_relaxed(ccr | CCR_EN, ddata->base + STM32_DMA3_CCR(id));
 801
 802         chan->dma_status = DMA_IN_PROGRESS;
 803
 804         dev_dbg(chan2dev(chan), "vchan %pK: started\n", &chan->vchan);
 805 }
 806
 807 static int stm32_dma3_chan_suspend(struct stm32_dma3_chan *chan, bool susp)
 808 {
 809         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
 810         u32 csr, ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)) & ~CCR_EN;
 811         int ret = 0;
 812
 813         if (susp)
 814                 ccr |= CCR_SUSP;
 815         else
 816                 ccr &= ~CCR_SUSP;
 817
 818         writel_relaxed(ccr, ddata->base + STM32_DMA3_CCR(chan->id));
 819
 820         if (susp) {
 821                 ret = readl_relaxed_poll_timeout_atomic(ddata->base + STM32_DMA3_CSR(chan->id), csr,
 822                                                         csr & CSR_SUSPF, 1, 10);
 823                 if (!ret)
 824                         writel_relaxed(CFCR_SUSPF, ddata->base + STM32_DMA3_CFCR(chan->id));
 825
 826                 stm32_dma3_chan_dump_reg(chan);
 827         }
 828
 829         return ret;
 830 }
 831
 832 static void stm32_dma3_chan_reset(struct stm32_dma3_chan *chan)
 833 {
 834         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
 835         u32 ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)) & ~CCR_EN;
 836
 837         writel_relaxed(ccr |= CCR_RESET, ddata->base + STM32_DMA3_CCR(chan->id));
 838 }
 839
 840 static int stm32_dma3_chan_get_curr_hwdesc(struct stm32_dma3_swdesc *swdesc, u32 cllr, u32 *residue)
 841 {
 842         u32 i, lli_offset, next_lli_offset = cllr & CLLR_LA;
 843
 844         /* If cllr is null, it means it is either the last or single item */
 845         if (!cllr)
 846                 return swdesc->lli_size - 1;
 847
 848         /* In cyclic mode, go fast and first check we are not on the last item */
 849         if (swdesc->cyclic && next_lli_offset == (swdesc->lli[0].hwdesc_addr & CLLR_LA))
 850                 return swdesc->lli_size - 1;
 851
 852         /* As transfer is in progress, look backward from the last item */
 853         for (i = swdesc->lli_size - 1; i > 0; i--) {
 854                 *residue += FIELD_GET(CBR1_BNDT, swdesc->lli[i].hwdesc->cbr1);
 855                 lli_offset = swdesc->lli[i].hwdesc_addr & CLLR_LA;
 856                 if (lli_offset == next_lli_offset)
 857                         return i - 1;
 858         }
 859
 860         return -EINVAL;
 861 }
 862
 863 static void stm32_dma3_chan_set_residue(struct stm32_dma3_chan *chan,
 864                                         struct stm32_dma3_swdesc *swdesc,
 865                                         struct dma_tx_state *txstate)
 866 {
 867         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
 868         struct device *dev = chan2dev(chan);
 869         struct stm32_dma3_hwdesc *hwdesc;
 870         u32 residue, curr_lli, csr, cdar, cbr1, cllr, bndt, fifol;
 871         bool pack_unpack;
 872         int ret;
 873
 874         csr = readl_relaxed(ddata->base + STM32_DMA3_CSR(chan->id));
 875         if (!(csr & CSR_IDLEF) && chan->dma_status != DMA_PAUSED) {
 876                 /* Suspend current transfer to read registers for a snapshot */
 877                 writel_relaxed(swdesc->ccr | CCR_SUSP, ddata->base + STM32_DMA3_CCR(chan->id));
 878                 ret = readl_relaxed_poll_timeout_atomic(ddata->base + STM32_DMA3_CSR(chan->id), csr,
 879                                                         csr & (CSR_SUSPF | CSR_IDLEF), 1, 10);
 880
 881                 if (ret || ((csr & CSR_TCF) && (csr & CSR_IDLEF))) {
 882                         writel_relaxed(CFCR_SUSPF, ddata->base + STM32_DMA3_CFCR(chan->id));
 883                         writel_relaxed(swdesc->ccr, ddata->base + STM32_DMA3_CCR(chan->id));
 884                         if (ret)
 885                                 dev_err(dev, "Channel suspension timeout, csr=%08x\n", csr);
 886                 }
 887         }
 888
 889         /* If channel is still active (CSR_IDLEF is not set), can't get a reliable residue */
 890         if (!(csr & CSR_IDLEF))
 891                 dev_warn(dev, "Can't get residue: channel still active, csr=%08x\n", csr);
 892
 893         /*
 894          * If channel is not suspended, but Idle and Transfer Complete are set,
 895          * linked-list is over, no residue
 896          */
 897         if (!(csr & CSR_SUSPF) && (csr & CSR_TCF) && (csr & CSR_IDLEF))
 898                 return;
 899
 900         /* Read registers to have a snapshot */
 901         cllr = readl_relaxed(ddata->base + STM32_DMA3_CLLR(chan->id));
 902         cbr1 = readl_relaxed(ddata->base + STM32_DMA3_CBR1(chan->id));
 903         cdar = readl_relaxed(ddata->base + STM32_DMA3_CDAR(chan->id));
 904
 905         /* Resume current transfer */
 906         if (csr & CSR_SUSPF) {
 907                 writel_relaxed(CFCR_SUSPF, ddata->base + STM32_DMA3_CFCR(chan->id));
 908                 writel_relaxed(swdesc->ccr, ddata->base + STM32_DMA3_CCR(chan->id));
 909         }
 910
 911         /* Add current BNDT */
 912         bndt = FIELD_GET(CBR1_BNDT, cbr1);
 913         residue = bndt;
 914
 915         /* Get current hwdesc and cumulate residue of pending hwdesc BNDT */
 916         ret = stm32_dma3_chan_get_curr_hwdesc(swdesc, cllr, &residue);
 917         if (ret < 0) {
 918                 dev_err(chan2dev(chan), "Can't get residue: current hwdesc not found\n");
 919                 return;
 920         }
 921         curr_lli = ret;
 922
 923         /* Read current FIFO level - in units of programmed destination data width */
 924         hwdesc = swdesc->lli[curr_lli].hwdesc;
 925         fifol = FIELD_GET(CSR_FIFOL, csr) * (1 << FIELD_GET(CTR1_DDW_LOG2, hwdesc->ctr1));
 926         /* If the FIFO contains as many bytes as its size, it can't contain more */
 927         if (fifol == (1 << (chan->fifo_size + 1)))
 928                 goto skip_fifol_update;
 929
 930         /*
 931          * In case of PACKING (Destination burst length > Source burst length) or UNPACKING
 932          * (Source burst length > Destination burst length), bytes could be pending in the FIFO
 933          * (to be packed up to Destination burst length or unpacked into Destination burst length
 934          * chunks).
 935          * BNDT is not reliable, as it reflects the number of bytes read from the source but not the
 936          * number of bytes written to the destination.
 937          * FIFOL is also not sufficient, because it reflects the number of available write beats in
 938          * units of Destination data width but not the bytes not yet packed or unpacked.
 939          * In case of Destination increment DINC, it is possible to compute the number of bytes in
 940          * the FIFO:
 941          * fifol_in_bytes = bytes_read - bytes_written.
 942          */
 943         pack_unpack = !!(FIELD_GET(CTR1_PAM, hwdesc->ctr1) == CTR1_PAM_PACK_UNPACK);
 944         if (pack_unpack && (hwdesc->ctr1 & CTR1_DINC)) {
 945                 int bytes_read = FIELD_GET(CBR1_BNDT, hwdesc->cbr1) - bndt;
 946                 int bytes_written = cdar - hwdesc->cdar;
 947
 948                 if (bytes_read > 0)
 949                         fifol = bytes_read - bytes_written;
 950         }
 951
 952 skip_fifol_update:
 953         if (fifol) {
 954                 dev_dbg(chan2dev(chan), "%u byte(s) in the FIFO\n", fifol);
 955                 dma_set_in_flight_bytes(txstate, fifol);
 956                 /*
 957                  * Residue is already accurate for DMA_MEM_TO_DEV as BNDT reflects data read from
 958                  * the source memory buffer, so just need to add fifol to residue in case of
 959                  * DMA_DEV_TO_MEM transfer because these bytes are not yet written in destination
 960                  * memory buffer.
 961                  */
 962                 if (chan->dma_config.direction == DMA_DEV_TO_MEM)
 963                         residue += fifol;
 964         }
 965         dma_set_residue(txstate, residue);
 966 }
 967
 968 static int stm32_dma3_chan_stop(struct stm32_dma3_chan *chan)
 969 {
 970         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
 971         u32 ccr;
 972         int ret = 0;
 973
 974         chan->dma_status = DMA_COMPLETE;
 975
 976         /* Disable interrupts */
 977         ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id));
 978         writel_relaxed(ccr & ~(CCR_ALLIE | CCR_EN), ddata->base + STM32_DMA3_CCR(chan->id));
 979
 980         if (!(ccr & CCR_SUSP) && (ccr & CCR_EN)) {
 981                 /* Suspend the channel */
 982                 ret = stm32_dma3_chan_suspend(chan, true);
 983                 if (ret)
 984                         dev_warn(chan2dev(chan), "%s: timeout, data might be lost\n", __func__);
 985         }
 986
 987         /*
 988          * Reset the channel: this causes the reset of the FIFO and the reset of the channel
 989          * internal state, the reset of CCR_EN and CCR_SUSP bits.
 990          */
 991         stm32_dma3_chan_reset(chan);
 992
 993         return ret;
 994 }
 995
 996 static void stm32_dma3_chan_complete(struct stm32_dma3_chan *chan)
 997 {
 998         if (!chan->swdesc)
 999                 return;
1000
1001         vchan_cookie_complete(&chan->swdesc->vdesc);
1002         chan->swdesc = NULL;
1003         stm32_dma3_chan_start(chan);
1004 }
1005
1006 static irqreturn_t stm32_dma3_chan_irq(int irq, void *devid)
1007 {
1008         struct stm32_dma3_chan *chan = devid;
1009         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
1010         u32 misr, csr, ccr;
1011
1012         spin_lock(&chan->vchan.lock);
1013
1014         misr = readl_relaxed(ddata->base + STM32_DMA3_MISR);
1015         if (!(misr & MISR_MIS(chan->id))) {
1016                 spin_unlock(&chan->vchan.lock);
1017                 return IRQ_NONE;
1018         }
1019
1020         csr = readl_relaxed(ddata->base + STM32_DMA3_CSR(chan->id));
1021         ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)) & CCR_ALLIE;
1022
1023         if (csr & CSR_TCF && ccr & CCR_TCIE) {
1024                 if (chan->swdesc->cyclic)
1025                         vchan_cyclic_callback(&chan->swdesc->vdesc);
1026                 else
1027                         stm32_dma3_chan_complete(chan);
1028         }
1029
1030         if (csr & CSR_USEF && ccr & CCR_USEIE) {
1031                 dev_err(chan2dev(chan), "User setting error\n");
1032                 chan->dma_status = DMA_ERROR;
1033                 /* CCR.EN automatically cleared by HW */
1034                 stm32_dma3_check_user_setting(chan);
1035                 stm32_dma3_chan_reset(chan);
1036         }
1037
1038         if (csr & CSR_ULEF && ccr & CCR_ULEIE) {
1039                 dev_err(chan2dev(chan), "Update link transfer error\n");
1040                 chan->dma_status = DMA_ERROR;
1041                 /* CCR.EN automatically cleared by HW */
1042                 stm32_dma3_chan_reset(chan);
1043         }
1044
1045         if (csr & CSR_DTEF && ccr & CCR_DTEIE) {
1046                 dev_err(chan2dev(chan), "Data transfer error\n");
1047                 chan->dma_status = DMA_ERROR;
1048                 /* CCR.EN automatically cleared by HW */
1049                 stm32_dma3_chan_reset(chan);
1050         }
1051
1052         /*
1053          * Half Transfer Interrupt may be disabled but Half Transfer Flag can be set,
1054          * ensure HTF flag to be cleared, with other flags.
1055          */
1056         csr &= (ccr | CCR_HTIE);
1057
1058         if (csr)
1059                 writel_relaxed(csr, ddata->base + STM32_DMA3_CFCR(chan->id));
1060
1061         spin_unlock(&chan->vchan.lock);
1062
1063         return IRQ_HANDLED;
1064 }
1065
1066 static int stm32_dma3_alloc_chan_resources(struct dma_chan *c)
1067 {
1068         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1069         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
1070         u32 id = chan->id, csemcr, ccid;
1071         int ret;
1072
1073         ret = pm_runtime_resume_and_get(ddata->dma_dev.dev);
1074         if (ret < 0)
1075                 return ret;
1076
1077         /* Ensure the channel is free */
1078         if (chan->semaphore_mode &&
1079             readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(chan->id)) & CSEMCR_SEM_MUTEX) {
1080                 ret = -EBUSY;
1081                 goto err_put_sync;
1082         }
1083
1084         chan->lli_pool = dmam_pool_create(dev_name(&c->dev->device), c->device->dev,
1085                                           sizeof(struct stm32_dma3_hwdesc),
1086                                           __alignof__(struct stm32_dma3_hwdesc), SZ_64K);
1087         if (!chan->lli_pool) {
1088                 dev_err(chan2dev(chan), "Failed to create LLI pool\n");
1089                 ret = -ENOMEM;
1090                 goto err_put_sync;
1091         }
1092
1093         /* Take the channel semaphore */
1094         if (chan->semaphore_mode) {
1095                 writel_relaxed(CSEMCR_SEM_MUTEX, ddata->base + STM32_DMA3_CSEMCR(id));
1096                 csemcr = readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(id));
1097                 ccid = FIELD_GET(CSEMCR_SEM_CCID, csemcr);
1098                 /* Check that the channel is well taken */
1099                 if (ccid != CCIDCFGR_CID1) {
1100                         dev_err(chan2dev(chan), "Not under CID1 control (in-use by CID%d)\n", ccid);
1101                         ret = -EPERM;
1102                         goto err_pool_destroy;
1103                 }
1104                 dev_dbg(chan2dev(chan), "Under CID1 control (semcr=0x%08x)\n", csemcr);
1105         }
1106
1107         return 0;
1108
1109 err_pool_destroy:
1110         dmam_pool_destroy(chan->lli_pool);
1111         chan->lli_pool = NULL;
1112
1113 err_put_sync:
1114         pm_runtime_put_sync(ddata->dma_dev.dev);
1115
1116         return ret;
1117 }
1118
1119 static void stm32_dma3_free_chan_resources(struct dma_chan *c)
1120 {
1121         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1122         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
1123         unsigned long flags;
1124
1125         /* Ensure channel is in idle state */
1126         spin_lock_irqsave(&chan->vchan.lock, flags);
1127         stm32_dma3_chan_stop(chan);
1128         chan->swdesc = NULL;
1129         spin_unlock_irqrestore(&chan->vchan.lock, flags);
1130
1131         vchan_free_chan_resources(to_virt_chan(c));
1132
1133         dmam_pool_destroy(chan->lli_pool);
1134         chan->lli_pool = NULL;
1135
1136         /* Release the channel semaphore */
1137         if (chan->semaphore_mode)
1138                 writel_relaxed(0, ddata->base + STM32_DMA3_CSEMCR(chan->id));
1139
1140         pm_runtime_put_sync(ddata->dma_dev.dev);
1141
1142         /* Reset configuration */
1143         memset(&chan->dt_config, 0, sizeof(chan->dt_config));
1144         memset(&chan->dma_config, 0, sizeof(chan->dma_config));
1145         chan->config_set = 0;
1146 }
1147
1148 static u32 stm32_dma3_get_ll_count(struct stm32_dma3_chan *chan, size_t len, bool prevent_refactor)
1149 {
1150         u32 count;
1151
1152         if (prevent_refactor)
1153                 return DIV_ROUND_UP(len, STM32_DMA3_MAX_BLOCK_SIZE);
1154
1155         count = len / STM32_DMA3_MAX_BLOCK_SIZE;
1156         len -= (len / STM32_DMA3_MAX_BLOCK_SIZE) * STM32_DMA3_MAX_BLOCK_SIZE;
1157
1158         if (len >= chan->max_burst) {
1159                 count += 1; /* len < STM32_DMA3_MAX_BLOCK_SIZE here, so it fits in one item */
1160                 len -= (len / chan->max_burst) * chan->max_burst;
1161         }
1162
1163         /* Unaligned remainder fits in one extra item */
1164         if (len > 0)
1165                 count += 1;
1166
1167         return count;
1168 }
1169
1170 static void stm32_dma3_init_chan_config_for_memcpy(struct stm32_dma3_chan *chan,
1171                                                    dma_addr_t dst, dma_addr_t src)
1172 {
1173         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
1174         u32 dw = get_chan_max_dw(ddata->ports_max_dw[0], chan->max_burst); /* port 0 by default */
1175         u32 burst = chan->max_burst / dw;
1176
1177         /* Initialize dt_config if channel not pre-configured through DT */
1178         if (!(chan->config_set & STM32_DMA3_CFG_SET_DT)) {
1179                 chan->dt_config.ch_conf = FIELD_PREP(STM32_DMA3_DT_PRIO, CCR_PRIO_VERY_HIGH);
1180                 chan->dt_config.ch_conf |= FIELD_PREP(STM32_DMA3_DT_FIFO, chan->fifo_size);
1181                 chan->dt_config.tr_conf = STM32_DMA3_DT_SINC | STM32_DMA3_DT_DINC;
1182                 chan->dt_config.tr_conf |= FIELD_PREP(STM32_DMA3_DT_TCEM, CTR2_TCEM_CHANNEL);
1183         }
1184
1185         /* Initialize dma_config if dmaengine_slave_config() not used */
1186         if (!(chan->config_set & STM32_DMA3_CFG_SET_DMA)) {
1187                 chan->dma_config.src_addr_width = dw;
1188                 chan->dma_config.dst_addr_width = dw;
1189                 chan->dma_config.src_maxburst = burst;
1190                 chan->dma_config.dst_maxburst = burst;
1191                 chan->dma_config.src_addr = src;
1192                 chan->dma_config.dst_addr = dst;
1193         }
1194 }
1195
1196 static struct dma_async_tx_descriptor *stm32_dma3_prep_dma_memcpy(struct dma_chan *c,
1197                                                                   dma_addr_t dst, dma_addr_t src,
1198                                                                   size_t len, unsigned long flags)
1199 {
1200         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1201         struct stm32_dma3_swdesc *swdesc;
1202         size_t next_size, offset;
1203         u32 count, i, ctr1, ctr2;
1204         bool prevent_refactor = !!FIELD_GET(STM32_DMA3_DT_NOPACK, chan->dt_config.tr_conf) ||
1205                                 !!FIELD_GET(STM32_DMA3_DT_NOREFACT, chan->dt_config.tr_conf);
1206
1207         count = stm32_dma3_get_ll_count(chan, len, prevent_refactor);
1208
1209         swdesc = stm32_dma3_chan_desc_alloc(chan, count);
1210         if (!swdesc)
1211                 return NULL;
1212
1213         if (chan->config_set != STM32_DMA3_CFG_SET_BOTH)
1214                 stm32_dma3_init_chan_config_for_memcpy(chan, dst, src);
1215
1216         for (i = 0, offset = 0; offset < len; i++, offset += next_size) {
1217                 size_t remaining;
1218                 int ret;
1219
1220                 remaining = len - offset;
1221                 next_size = min_t(size_t, remaining, STM32_DMA3_MAX_BLOCK_SIZE);
1222
1223                 if (!prevent_refactor &&
1224                     (next_size < STM32_DMA3_MAX_BLOCK_SIZE && next_size >= chan->max_burst))
1225                         next_size = chan->max_burst * (remaining / chan->max_burst);
1226
1227                 ret = stm32_dma3_chan_prep_hw(chan, DMA_MEM_TO_MEM, &swdesc->ccr, &ctr1, &ctr2,
1228                                               src + offset, dst + offset, next_size);
1229                 if (ret)
1230                         goto err_desc_free;
1231
1232                 stm32_dma3_chan_prep_hwdesc(chan, swdesc, i, src + offset, dst + offset, next_size,
1233                                             ctr1, ctr2, next_size == remaining, false);
1234         }
1235
1236         /* Enable Errors interrupts */
1237         swdesc->ccr |= CCR_USEIE | CCR_ULEIE | CCR_DTEIE;
1238         /* Enable Transfer state interrupts */
1239         swdesc->ccr |= CCR_TCIE;
1240
1241         swdesc->cyclic = false;
1242
1243         return vchan_tx_prep(&chan->vchan, &swdesc->vdesc, flags);
1244
1245 err_desc_free:
1246         stm32_dma3_chan_desc_free(chan, swdesc);
1247
1248         return NULL;
1249 }
1250
1251 static struct dma_async_tx_descriptor *stm32_dma3_prep_slave_sg(struct dma_chan *c,
1252                                                                 struct scatterlist *sgl,
1253                                                                 unsigned int sg_len,
1254                                                                 enum dma_transfer_direction dir,
1255                                                                 unsigned long flags, void *context)
1256 {
1257         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1258         struct stm32_dma3_swdesc *swdesc;
1259         struct scatterlist *sg;
1260         size_t len;
1261         dma_addr_t sg_addr, dev_addr, src, dst;
1262         u32 i, j, count, ctr1, ctr2;
1263         bool prevent_refactor = !!FIELD_GET(STM32_DMA3_DT_NOPACK, chan->dt_config.tr_conf) ||
1264                                 !!FIELD_GET(STM32_DMA3_DT_NOREFACT, chan->dt_config.tr_conf);
1265         int ret;
1266
1267         count = 0;
1268         for_each_sg(sgl, sg, sg_len, i)
1269                 count += stm32_dma3_get_ll_count(chan, sg_dma_len(sg), prevent_refactor);
1270
1271         swdesc = stm32_dma3_chan_desc_alloc(chan, count);
1272         if (!swdesc)
1273                 return NULL;
1274
1275         /* sg_len and i correspond to the initial sgl; count and j correspond to the hwdesc LL */
1276         j = 0;
1277         for_each_sg(sgl, sg, sg_len, i) {
1278                 sg_addr = sg_dma_address(sg);
1279                 dev_addr = (dir == DMA_MEM_TO_DEV) ? chan->dma_config.dst_addr :
1280                                                      chan->dma_config.src_addr;
1281                 len = sg_dma_len(sg);
1282
1283                 do {
1284                         size_t chunk = min_t(size_t, len, STM32_DMA3_MAX_BLOCK_SIZE);
1285
1286                         if (!prevent_refactor &&
1287                             (chunk < STM32_DMA3_MAX_BLOCK_SIZE && chunk >= chan->max_burst))
1288                                 chunk = chan->max_burst * (len / chan->max_burst);
1289
1290                         if (dir == DMA_MEM_TO_DEV) {
1291                                 src = sg_addr;
1292                                 dst = dev_addr;
1293
1294                                 ret = stm32_dma3_chan_prep_hw(chan, dir, &swdesc->ccr, &ctr1, &ctr2,
1295                                                               src, dst, chunk);
1296
1297                                 if (FIELD_GET(CTR1_DINC, ctr1))
1298                                         dev_addr += chunk;
1299                         } else { /* (dir == DMA_DEV_TO_MEM || dir == DMA_MEM_TO_MEM) */
1300                                 src = dev_addr;
1301                                 dst = sg_addr;
1302
1303                                 ret = stm32_dma3_chan_prep_hw(chan, dir, &swdesc->ccr, &ctr1, &ctr2,
1304                                                               src, dst, chunk);
1305
1306                                 if (FIELD_GET(CTR1_SINC, ctr1))
1307                                         dev_addr += chunk;
1308                         }
1309
1310                         if (ret)
1311                                 goto err_desc_free;
1312
1313                         stm32_dma3_chan_prep_hwdesc(chan, swdesc, j, src, dst, chunk,
1314                                                     ctr1, ctr2, j == (count - 1), false);
1315
1316                         sg_addr += chunk;
1317                         len -= chunk;
1318                         j++;
1319                 } while (len);
1320         }
1321
1322         if (count != sg_len && chan->tcem != CTR2_TCEM_CHANNEL)
1323                 dev_warn(chan2dev(chan), "Linked-list refactored, %d items instead of %d\n",
1324                          count, sg_len);
1325
1326         /* Enable Error interrupts */
1327         swdesc->ccr |= CCR_USEIE | CCR_ULEIE | CCR_DTEIE;
1328         /* Enable Transfer state interrupts */
1329         swdesc->ccr |= CCR_TCIE;
1330
1331         swdesc->cyclic = false;
1332
1333         return vchan_tx_prep(&chan->vchan, &swdesc->vdesc, flags);
1334
1335 err_desc_free:
1336         stm32_dma3_chan_desc_free(chan, swdesc);
1337
1338         return NULL;
1339 }
1340
1341 static struct dma_async_tx_descriptor *stm32_dma3_prep_dma_cyclic(struct dma_chan *c,
1342                                                                   dma_addr_t buf_addr,
1343                                                                   size_t buf_len, size_t period_len,
1344                                                                   enum dma_transfer_direction dir,
1345                                                                   unsigned long flags)
1346 {
1347         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1348         struct stm32_dma3_swdesc *swdesc;
1349         dma_addr_t src, dst;
1350         u32 count, i, ctr1, ctr2;
1351         int ret;
1352
1353         if (!buf_len || !period_len || period_len > STM32_DMA3_MAX_BLOCK_SIZE) {
1354                 dev_err(chan2dev(chan), "Invalid buffer/period length\n");
1355                 return NULL;
1356         }
1357
1358         if (buf_len % period_len) {
1359                 dev_err(chan2dev(chan), "Buffer length not multiple of period length\n");
1360                 return NULL;
1361         }
1362
1363         count = buf_len / period_len;
1364         swdesc = stm32_dma3_chan_desc_alloc(chan, count);
1365         if (!swdesc)
1366                 return NULL;
1367
1368         if (dir == DMA_MEM_TO_DEV) {
1369                 src = buf_addr;
1370                 dst = chan->dma_config.dst_addr;
1371
1372                 ret = stm32_dma3_chan_prep_hw(chan, DMA_MEM_TO_DEV, &swdesc->ccr, &ctr1, &ctr2,
1373                                               src, dst, period_len);
1374         } else if (dir == DMA_DEV_TO_MEM) {
1375                 src = chan->dma_config.src_addr;
1376                 dst = buf_addr;
1377
1378                 ret = stm32_dma3_chan_prep_hw(chan, DMA_DEV_TO_MEM, &swdesc->ccr, &ctr1, &ctr2,
1379                                               src, dst, period_len);
1380         } else {
1381                 dev_err(chan2dev(chan), "Invalid direction\n");
1382                 ret = -EINVAL;
1383         }
1384
1385         if (ret)
1386                 goto err_desc_free;
1387
1388         for (i = 0; i < count; i++) {
1389                 if (dir == DMA_MEM_TO_DEV) {
1390                         src = buf_addr + i * period_len;
1391                         dst = chan->dma_config.dst_addr;
1392                 } else { /* (dir == DMA_DEV_TO_MEM) */
1393                         src = chan->dma_config.src_addr;
1394                         dst = buf_addr + i * period_len;
1395                 }
1396
1397                 stm32_dma3_chan_prep_hwdesc(chan, swdesc, i, src, dst, period_len,
1398                                             ctr1, ctr2, i == (count - 1), true);
1399         }
1400
1401         /* Enable Error interrupts */
1402         swdesc->ccr |= CCR_USEIE | CCR_ULEIE | CCR_DTEIE;
1403         /* Enable Transfer state interrupts */
1404         swdesc->ccr |= CCR_TCIE;
1405
1406         swdesc->cyclic = true;
1407
1408         return vchan_tx_prep(&chan->vchan, &swdesc->vdesc, flags);
1409
1410 err_desc_free:
1411         stm32_dma3_chan_desc_free(chan, swdesc);
1412
1413         return NULL;
1414 }
1415
1416 static void stm32_dma3_caps(struct dma_chan *c, struct dma_slave_caps *caps)
1417 {
1418         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1419
1420         if (!chan->fifo_size) {
1421                 caps->max_burst = 0;
1422                 caps->src_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1423                 caps->dst_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1424         } else {
1425                 /* Burst transfer should not exceed half of the fifo size */
1426                 caps->max_burst = chan->max_burst;
1427                 if (caps->max_burst < DMA_SLAVE_BUSWIDTH_8_BYTES) {
1428                         caps->src_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1429                         caps->dst_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1430                 }
1431         }
1432 }
1433
1434 static int stm32_dma3_config(struct dma_chan *c, struct dma_slave_config *config)
1435 {
1436         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1437
1438         memcpy(&chan->dma_config, config, sizeof(*config));
1439         chan->config_set |= STM32_DMA3_CFG_SET_DMA;
1440
1441         return 0;
1442 }
1443
1444 static int stm32_dma3_pause(struct dma_chan *c)
1445 {
1446         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1447         int ret;
1448
1449         ret = stm32_dma3_chan_suspend(chan, true);
1450         if (ret)
1451                 return ret;
1452
1453         chan->dma_status = DMA_PAUSED;
1454
1455         dev_dbg(chan2dev(chan), "vchan %pK: paused\n", &chan->vchan);
1456
1457         return 0;
1458 }
1459
1460 static int stm32_dma3_resume(struct dma_chan *c)
1461 {
1462         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1463
1464         stm32_dma3_chan_suspend(chan, false);
1465
1466         chan->dma_status = DMA_IN_PROGRESS;
1467
1468         dev_dbg(chan2dev(chan), "vchan %pK: resumed\n", &chan->vchan);
1469
1470         return 0;
1471 }
1472
1473 static int stm32_dma3_terminate_all(struct dma_chan *c)
1474 {
1475         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1476         unsigned long flags;
1477         LIST_HEAD(head);
1478
1479         spin_lock_irqsave(&chan->vchan.lock, flags);
1480
1481         if (chan->swdesc) {
1482                 vchan_terminate_vdesc(&chan->swdesc->vdesc);
1483                 chan->swdesc = NULL;
1484         }
1485
1486         stm32_dma3_chan_stop(chan);
1487
1488         vchan_get_all_descriptors(&chan->vchan, &head);
1489
1490         spin_unlock_irqrestore(&chan->vchan.lock, flags);
1491         vchan_dma_desc_free_list(&chan->vchan, &head);
1492
1493         dev_dbg(chan2dev(chan), "vchan %pK: terminated\n", &chan->vchan);
1494
1495         return 0;
1496 }
1497
1498 static void stm32_dma3_synchronize(struct dma_chan *c)
1499 {
1500         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1501
1502         vchan_synchronize(&chan->vchan);
1503 }
1504
1505 static enum dma_status stm32_dma3_tx_status(struct dma_chan *c, dma_cookie_t cookie,
1506                                             struct dma_tx_state *txstate)
1507 {
1508         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1509         struct stm32_dma3_swdesc *swdesc = NULL;
1510         enum dma_status status;
1511         unsigned long flags;
1512         struct virt_dma_desc *vd;
1513
1514         status = dma_cookie_status(c, cookie, txstate);
1515         if (status == DMA_COMPLETE)
1516                 return status;
1517
1518         if (!txstate)
1519                 return chan->dma_status;
1520
1521         spin_lock_irqsave(&chan->vchan.lock, flags);
1522
1523         vd = vchan_find_desc(&chan->vchan, cookie);
1524         if (vd)
1525                 swdesc = to_stm32_dma3_swdesc(vd);
1526         else if (chan->swdesc && chan->swdesc->vdesc.tx.cookie == cookie)
1527                 swdesc = chan->swdesc;
1528
1529         /* Get residue/in_flight_bytes only if a transfer is currently running (swdesc != NULL) */
1530         if (swdesc)
1531                 stm32_dma3_chan_set_residue(chan, swdesc, txstate);
1532
1533         spin_unlock_irqrestore(&chan->vchan.lock, flags);
1534
1535         return chan->dma_status;
1536 }
1537
1538 static void stm32_dma3_issue_pending(struct dma_chan *c)
1539 {
1540         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1541         unsigned long flags;
1542
1543         spin_lock_irqsave(&chan->vchan.lock, flags);
1544
1545         if (vchan_issue_pending(&chan->vchan) && !chan->swdesc) {
1546                 dev_dbg(chan2dev(chan), "vchan %pK: issued\n", &chan->vchan);
1547                 stm32_dma3_chan_start(chan);
1548         }
1549
1550         spin_unlock_irqrestore(&chan->vchan.lock, flags);
1551 }
1552
1553 static bool stm32_dma3_filter_fn(struct dma_chan *c, void *fn_param)
1554 {
1555         struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1556         struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
1557         struct stm32_dma3_dt_conf *conf = fn_param;
1558         u32 mask, semcr;
1559         int ret;
1560
1561         dev_dbg(c->device->dev, "%s(%s): req_line=%d ch_conf=%08x tr_conf=%08x\n",
1562                 __func__, dma_chan_name(c), conf->req_line, conf->ch_conf, conf->tr_conf);
1563
1564         if (!of_property_read_u32(c->device->dev->of_node, "dma-channel-mask", &mask))
1565                 if (!(mask & BIT(chan->id)))
1566                         return false;
1567
1568         ret = pm_runtime_resume_and_get(ddata->dma_dev.dev);
1569         if (ret < 0)
1570                 return false;
1571         semcr = readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(chan->id));
1572         pm_runtime_put_sync(ddata->dma_dev.dev);
1573
1574         /* Check if chan is free */
1575         if (semcr & CSEMCR_SEM_MUTEX)
1576                 return false;
1577
1578         /* Check if chan fifo fits well */
1579         if (FIELD_GET(STM32_DMA3_DT_FIFO, conf->ch_conf) != chan->fifo_size)
1580                 return false;
1581
1582         return true;
1583 }
1584
1585 static struct dma_chan *stm32_dma3_of_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma)
1586 {
1587         struct stm32_dma3_ddata *ddata = ofdma->of_dma_data;
1588         dma_cap_mask_t mask = ddata->dma_dev.cap_mask;
1589         struct stm32_dma3_dt_conf conf;
1590         struct stm32_dma3_chan *chan;
1591         struct dma_chan *c;
1592
1593         if (dma_spec->args_count < 3) {
1594                 dev_err(ddata->dma_dev.dev, "Invalid args count\n");
1595                 return NULL;
1596         }
1597
1598         conf.req_line = dma_spec->args[0];
1599         conf.ch_conf = dma_spec->args[1];
1600         conf.tr_conf = dma_spec->args[2];
1601
1602         if (conf.req_line >= ddata->dma_requests) {
1603                 dev_err(ddata->dma_dev.dev, "Invalid request line\n");
1604                 return NULL;
1605         }
1606
1607         /* Request dma channel among the generic dma controller list */
1608         c = dma_request_channel(mask, stm32_dma3_filter_fn, &conf);
1609         if (!c) {
1610                 dev_err(ddata->dma_dev.dev, "No suitable channel found\n");
1611                 return NULL;
1612         }
1613
1614         chan = to_stm32_dma3_chan(c);
1615         chan->dt_config = conf;
1616         chan->config_set |= STM32_DMA3_CFG_SET_DT;
1617
1618         return c;
1619 }
1620
1621 static u32 stm32_dma3_check_rif(struct stm32_dma3_ddata *ddata)
1622 {
1623         u32 chan_reserved, mask = 0, i, ccidcfgr, invalid_cid = 0;
1624
1625         /* Reserve Secure channels */
1626         chan_reserved = readl_relaxed(ddata->base + STM32_DMA3_SECCFGR);
1627
1628         /*
1629          * CID filtering must be configured to ensure that the DMA3 channel will inherit the CID of
1630          * the processor which is configuring and using the given channel.
1631          * In case CID filtering is not configured, dma-channel-mask property can be used to
1632          * specify available DMA channels to the kernel.
1633          */
1634         of_property_read_u32(ddata->dma_dev.dev->of_node, "dma-channel-mask", &mask);
1635
1636         /* Reserve !CID-filtered not in dma-channel-mask, static CID != CID1, CID1 not allowed */
1637         for (i = 0; i < ddata->dma_channels; i++) {
1638                 ccidcfgr = readl_relaxed(ddata->base + STM32_DMA3_CCIDCFGR(i));
1639
1640                 if (!(ccidcfgr & CCIDCFGR_CFEN)) { /* !CID-filtered */
1641                         invalid_cid |= BIT(i);
1642                         if (!(mask & BIT(i))) /* Not in dma-channel-mask */
1643                                 chan_reserved |= BIT(i);
1644                 } else { /* CID-filtered */
1645                         if (!(ccidcfgr & CCIDCFGR_SEM_EN)) { /* Static CID mode */
1646                                 if (FIELD_GET(CCIDCFGR_SCID, ccidcfgr) != CCIDCFGR_CID1)
1647                                         chan_reserved |= BIT(i);
1648                         } else { /* Semaphore mode */
1649                                 if (!FIELD_GET(CCIDCFGR_SEM_WLIST_CID1, ccidcfgr))
1650                                         chan_reserved |= BIT(i);
1651                                 ddata->chans[i].semaphore_mode = true;
1652                         }
1653                 }
1654                 dev_dbg(ddata->dma_dev.dev, "chan%d: %s mode, %s\n", i,
1655                         !(ccidcfgr & CCIDCFGR_CFEN) ? "!CID-filtered" :
1656                         ddata->chans[i].semaphore_mode ? "Semaphore" : "Static CID",
1657                         (chan_reserved & BIT(i)) ? "denied" :
1658                         mask & BIT(i) ? "force allowed" : "allowed");
1659         }
1660
1661         if (invalid_cid)
1662                 dev_warn(ddata->dma_dev.dev, "chan%*pbl have invalid CID configuration\n",
1663                          ddata->dma_channels, &invalid_cid);
1664
1665         return chan_reserved;
1666 }
1667
1668 static struct stm32_dma3_pdata stm32mp25_pdata = {
1669         .axi_max_burst_len = 16,
1670 };
1671
1672 static const struct of_device_id stm32_dma3_of_match[] = {
1673         { .compatible = "st,stm32mp25-dma3", .data = &stm32mp25_pdata, },
1674         { /* sentinel */ },
1675 };
1676 MODULE_DEVICE_TABLE(of, stm32_dma3_of_match);
1677
1678 static int stm32_dma3_probe(struct platform_device *pdev)
1679 {
1680         struct device_node *np = pdev->dev.of_node;
1681         const struct stm32_dma3_pdata *pdata;
1682         struct stm32_dma3_ddata *ddata;
1683         struct reset_control *reset;
1684         struct stm32_dma3_chan *chan;
1685         struct dma_device *dma_dev;
1686         u32 master_ports, chan_reserved, i, verr;
1687         u64 hwcfgr;
1688         int ret;
1689
1690         ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
1691         if (!ddata)
1692                 return -ENOMEM;
1693         platform_set_drvdata(pdev, ddata);
1694
1695         dma_dev = &ddata->dma_dev;
1696
1697         ddata->base = devm_platform_ioremap_resource(pdev, 0);
1698         if (IS_ERR(ddata->base))
1699                 return PTR_ERR(ddata->base);
1700
1701         ddata->clk = devm_clk_get(&pdev->dev, NULL);
1702         if (IS_ERR(ddata->clk))
1703                 return dev_err_probe(&pdev->dev, PTR_ERR(ddata->clk), "Failed to get clk\n");
1704
1705         reset = devm_reset_control_get_optional(&pdev->dev, NULL);
1706         if (IS_ERR(reset))
1707                 return dev_err_probe(&pdev->dev, PTR_ERR(reset), "Failed to get reset\n");
1708
1709         ret = clk_prepare_enable(ddata->clk);
1710         if (ret)
1711                 return dev_err_probe(&pdev->dev, ret, "Failed to enable clk\n");
1712
1713         reset_control_reset(reset);
1714
1715         INIT_LIST_HEAD(&dma_dev->channels);
1716
1717         dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);
1718         dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask);
1719         dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask);
1720         dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
1721         dma_dev->dev = &pdev->dev;
1722         /*
1723          * This controller supports up to 8-byte buswidth depending on the port used and the
1724          * channel, and can only access address at even boundaries, multiple of the buswidth.
1725          */
1726         dma_dev->copy_align = DMAENGINE_ALIGN_8_BYTES;
1727         dma_dev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1728                                    BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1729                                    BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1730                                    BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1731         dma_dev->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1732                                    BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1733                                    BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1734                                    BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1735         dma_dev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) | BIT(DMA_MEM_TO_MEM);
1736
1737         dma_dev->descriptor_reuse = true;
1738         dma_dev->max_sg_burst = STM32_DMA3_MAX_SEG_SIZE;
1739         dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1740         dma_dev->device_alloc_chan_resources = stm32_dma3_alloc_chan_resources;
1741         dma_dev->device_free_chan_resources = stm32_dma3_free_chan_resources;
1742         dma_dev->device_prep_dma_memcpy = stm32_dma3_prep_dma_memcpy;
1743         dma_dev->device_prep_slave_sg = stm32_dma3_prep_slave_sg;
1744         dma_dev->device_prep_dma_cyclic = stm32_dma3_prep_dma_cyclic;
1745         dma_dev->device_caps = stm32_dma3_caps;
1746         dma_dev->device_config = stm32_dma3_config;
1747         dma_dev->device_pause = stm32_dma3_pause;
1748         dma_dev->device_resume = stm32_dma3_resume;
1749         dma_dev->device_terminate_all = stm32_dma3_terminate_all;
1750         dma_dev->device_synchronize = stm32_dma3_synchronize;
1751         dma_dev->device_tx_status = stm32_dma3_tx_status;
1752         dma_dev->device_issue_pending = stm32_dma3_issue_pending;
1753
1754         /* if dma_channels is not modified, get it from hwcfgr1 */
1755         if (of_property_read_u32(np, "dma-channels", &ddata->dma_channels)) {
1756                 hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR1);
1757                 ddata->dma_channels = FIELD_GET(G_NUM_CHANNELS, hwcfgr);
1758         }
1759
1760         /* if dma_requests is not modified, get it from hwcfgr2 */
1761         if (of_property_read_u32(np, "dma-requests", &ddata->dma_requests)) {
1762                 hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR2);
1763                 ddata->dma_requests = FIELD_GET(G_MAX_REQ_ID, hwcfgr) + 1;
1764         }
1765
1766         /* G_MASTER_PORTS, G_M0_DATA_WIDTH_ENC, G_M1_DATA_WIDTH_ENC in HWCFGR1 */
1767         hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR1);
1768         master_ports = FIELD_GET(G_MASTER_PORTS, hwcfgr);
1769
1770         ddata->ports_max_dw[0] = FIELD_GET(G_M0_DATA_WIDTH_ENC, hwcfgr);
1771         if (master_ports == AXI64 || master_ports == AHB32) /* Single master port */
1772                 ddata->ports_max_dw[1] = DW_INVALID;
1773         else /* Dual master ports */
1774                 ddata->ports_max_dw[1] = FIELD_GET(G_M1_DATA_WIDTH_ENC, hwcfgr);
1775
1776         /* axi_max_burst_len is optional, if not defined, use STM32_DMA3_MAX_BURST_LEN  */
1777         ddata->axi_max_burst_len = STM32_DMA3_MAX_BURST_LEN;
1778         pdata = device_get_match_data(&pdev->dev);
1779         if (pdata && pdata->axi_max_burst_len) {
1780                 ddata->axi_max_burst_len = min_t(u32, pdata->axi_max_burst_len,
1781                                                  STM32_DMA3_MAX_BURST_LEN);
1782                 dev_dbg(&pdev->dev, "Burst is limited to %u beats through AXI port\n",
1783                         ddata->axi_max_burst_len);
1784         }
1785
1786         ddata->chans = devm_kcalloc(&pdev->dev, ddata->dma_channels, sizeof(*ddata->chans),
1787                                     GFP_KERNEL);
1788         if (!ddata->chans) {
1789                 ret = -ENOMEM;
1790                 goto err_clk_disable;
1791         }
1792
1793         chan_reserved = stm32_dma3_check_rif(ddata);
1794
1795         if (chan_reserved == GENMASK(ddata->dma_channels - 1, 0)) {
1796                 ret = -ENODEV;
1797                 dev_err_probe(&pdev->dev, ret, "No channel available, abort registration\n");
1798                 goto err_clk_disable;
1799         }
1800
1801         /* G_FIFO_SIZE x=0..7 in HWCFGR3 and G_FIFO_SIZE x=8..15 in HWCFGR4 */
1802         hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR3);
1803         hwcfgr |= ((u64)readl_relaxed(ddata->base + STM32_DMA3_HWCFGR4)) << 32;
1804
1805         for (i = 0; i < ddata->dma_channels; i++) {
1806                 if (chan_reserved & BIT(i))
1807                         continue;
1808
1809                 chan = &ddata->chans[i];
1810                 chan->id = i;
1811                 chan->fifo_size = get_chan_hwcfg(i, G_FIFO_SIZE(i), hwcfgr);
1812                 /* If chan->fifo_size > 0 then half of the fifo size, else no burst when no FIFO */
1813                 chan->max_burst = (chan->fifo_size) ? (1 << (chan->fifo_size + 1)) / 2 : 0;
1814         }
1815
1816         ret = dmaenginem_async_device_register(dma_dev);
1817         if (ret)
1818                 goto err_clk_disable;
1819
1820         for (i = 0; i < ddata->dma_channels; i++) {
1821                 char name[12];
1822
1823                 if (chan_reserved & BIT(i))
1824                         continue;
1825
1826                 chan = &ddata->chans[i];
1827                 snprintf(name, sizeof(name), "dma%dchan%d", ddata->dma_dev.dev_id, chan->id);
1828
1829                 chan->vchan.desc_free = stm32_dma3_chan_vdesc_free;
1830                 vchan_init(&chan->vchan, dma_dev);
1831
1832                 ret = dma_async_device_channel_register(&ddata->dma_dev, &chan->vchan.chan, name);
1833                 if (ret) {
1834                         dev_err_probe(&pdev->dev, ret, "Failed to register channel %s\n", name);
1835                         goto err_clk_disable;
1836                 }
1837
1838                 ret = platform_get_irq(pdev, i);
1839                 if (ret < 0)
1840                         goto err_clk_disable;
1841                 chan->irq = ret;
1842
1843                 ret = devm_request_irq(&pdev->dev, chan->irq, stm32_dma3_chan_irq, 0,
1844                                        dev_name(chan2dev(chan)), chan);
1845                 if (ret) {
1846                         dev_err_probe(&pdev->dev, ret, "Failed to request channel %s IRQ\n",
1847                                       dev_name(chan2dev(chan)));
1848                         goto err_clk_disable;
1849                 }
1850         }
1851
1852         ret = of_dma_controller_register(np, stm32_dma3_of_xlate, ddata);
1853         if (ret) {
1854                 dev_err_probe(&pdev->dev, ret, "Failed to register controller\n");
1855                 goto err_clk_disable;
1856         }
1857
1858         verr = readl_relaxed(ddata->base + STM32_DMA3_VERR);
1859
1860         pm_runtime_set_active(&pdev->dev);
1861         pm_runtime_enable(&pdev->dev);
1862         pm_runtime_get_noresume(&pdev->dev);
1863         pm_runtime_put(&pdev->dev);
1864
1865         dev_info(&pdev->dev, "STM32 DMA3 registered rev:%lu.%lu\n",
1866                  FIELD_GET(VERR_MAJREV, verr), FIELD_GET(VERR_MINREV, verr));
1867
1868         return 0;
1869
1870 err_clk_disable:
1871         clk_disable_unprepare(ddata->clk);
1872
1873         return ret;
1874 }
1875
1876 static void stm32_dma3_remove(struct platform_device *pdev)
1877 {
1878         pm_runtime_disable(&pdev->dev);
1879 }
1880
1881 static int stm32_dma3_runtime_suspend(struct device *dev)
1882 {
1883         struct stm32_dma3_ddata *ddata = dev_get_drvdata(dev);
1884
1885         clk_disable_unprepare(ddata->clk);
1886
1887         return 0;
1888 }
1889
1890 static int stm32_dma3_runtime_resume(struct device *dev)
1891 {
1892         struct stm32_dma3_ddata *ddata = dev_get_drvdata(dev);
1893         int ret;
1894
1895         ret = clk_prepare_enable(ddata->clk);
1896         if (ret)
1897                 dev_err(dev, "Failed to enable clk: %d\n", ret);
1898
1899         return ret;
1900 }
1901
1902 static const struct dev_pm_ops stm32_dma3_pm_ops = {
1903         SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)
1904         RUNTIME_PM_OPS(stm32_dma3_runtime_suspend, stm32_dma3_runtime_resume, NULL)
1905 };
1906
1907 static struct platform_driver stm32_dma3_driver = {
1908         .probe = stm32_dma3_probe,
1909         .remove = stm32_dma3_remove,
1910         .driver = {
1911                 .name = "stm32-dma3",
1912                 .of_match_table = stm32_dma3_of_match,
1913                 .pm = pm_ptr(&stm32_dma3_pm_ops),
1914         },
1915 };
1916
1917 static int __init stm32_dma3_init(void)
1918 {
1919         return platform_driver_register(&stm32_dma3_driver);
1920 }
1921
1922 subsys_initcall(stm32_dma3_init);
1923
1924 MODULE_DESCRIPTION("STM32 DMA3 controller driver");
1925 MODULE_AUTHOR("Amelie Delaunay <[email protected]>");
1926 MODULE_LICENSE("GPL");