drivers/net/ethernet/mediatek/mtk_wed.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (C) 2021 Felix Fietkau <[email protected]> */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/platform_device.h>
   6 #include <linux/slab.h>
   7 #include <linux/module.h>
   8 #include <linux/bitfield.h>
   9 #include <linux/dma-mapping.h>
  10 #include <linux/skbuff.h>
  11 #include <linux/of_platform.h>
  12 #include <linux/of_address.h>
  13 #include <linux/of_reserved_mem.h>
  14 #include <linux/mfd/syscon.h>
  15 #include <linux/debugfs.h>
  16 #include <linux/soc/mediatek/mtk_wed.h>
  17 #include <net/flow_offload.h>
  18 #include <net/pkt_cls.h>
  19 #include "mtk_eth_soc.h"
  20 #include "mtk_wed.h"
  21 #include "mtk_ppe.h"
  22 #include "mtk_wed_wo.h"
  23
  24 #define MTK_PCIE_BASE(n)                (0x1a143000 + (n) * 0x2000)
  25
  26 #define MTK_WED_PKT_SIZE                1920
  27 #define MTK_WED_BUF_SIZE                2048
  28 #define MTK_WED_PAGE_BUF_SIZE           128
  29 #define MTK_WED_BUF_PER_PAGE            (PAGE_SIZE / 2048)
  30 #define MTK_WED_RX_BUF_PER_PAGE         (PAGE_SIZE / MTK_WED_PAGE_BUF_SIZE)
  31 #define MTK_WED_RX_RING_SIZE            1536
  32 #define MTK_WED_RX_PG_BM_CNT            8192
  33 #define MTK_WED_AMSDU_BUF_SIZE          (PAGE_SIZE << 4)
  34 #define MTK_WED_AMSDU_NPAGES            32
  35
  36 #define MTK_WED_TX_RING_SIZE            2048
  37 #define MTK_WED_WDMA_RING_SIZE          1024
  38 #define MTK_WED_MAX_GROUP_SIZE          0x100
  39 #define MTK_WED_VLD_GROUP_SIZE          0x40
  40 #define MTK_WED_PER_GROUP_PKT           128
  41
  42 #define MTK_WED_FBUF_SIZE               128
  43 #define MTK_WED_MIOD_CNT                16
  44 #define MTK_WED_FB_CMD_CNT              1024
  45 #define MTK_WED_RRO_QUE_CNT             8192
  46 #define MTK_WED_MIOD_ENTRY_CNT          128
  47
  48 #define MTK_WED_TX_BM_DMA_SIZE          65536
  49 #define MTK_WED_TX_BM_PKT_CNT           32768
  50
  51 static struct mtk_wed_hw *hw_list[3];
  52 static DEFINE_MUTEX(hw_lock);
  53
  54 struct mtk_wed_flow_block_priv {
  55         struct mtk_wed_hw *hw;
  56         struct net_device *dev;
  57 };
  58
  59 static const struct mtk_wed_soc_data mt7622_data = {
  60         .regmap = {
  61                 .tx_bm_tkid             = 0x088,
  62                 .wpdma_rx_ring0         = 0x770,
  63                 .reset_idx_tx_mask      = GENMASK(3, 0),
  64                 .reset_idx_rx_mask      = GENMASK(17, 16),
  65         },
  66         .tx_ring_desc_size = sizeof(struct mtk_wdma_desc),
  67         .wdma_desc_size = sizeof(struct mtk_wdma_desc),
  68 };
  69
  70 static const struct mtk_wed_soc_data mt7986_data = {
  71         .regmap = {
  72                 .tx_bm_tkid             = 0x0c8,
  73                 .wpdma_rx_ring0         = 0x770,
  74                 .reset_idx_tx_mask      = GENMASK(1, 0),
  75                 .reset_idx_rx_mask      = GENMASK(7, 6),
  76         },
  77         .tx_ring_desc_size = sizeof(struct mtk_wdma_desc),
  78         .wdma_desc_size = 2 * sizeof(struct mtk_wdma_desc),
  79 };
  80
  81 static const struct mtk_wed_soc_data mt7988_data = {
  82         .regmap = {
  83                 .tx_bm_tkid             = 0x0c8,
  84                 .wpdma_rx_ring0         = 0x7d0,
  85                 .reset_idx_tx_mask      = GENMASK(1, 0),
  86                 .reset_idx_rx_mask      = GENMASK(7, 6),
  87         },
  88         .tx_ring_desc_size = sizeof(struct mtk_wed_bm_desc),
  89         .wdma_desc_size = 2 * sizeof(struct mtk_wdma_desc),
  90 };
  91
  92 static void
  93 wed_m32(struct mtk_wed_device *dev, u32 reg, u32 mask, u32 val)
  94 {
  95         regmap_update_bits(dev->hw->regs, reg, mask | val, val);
  96 }
  97
  98 static void
  99 wed_set(struct mtk_wed_device *dev, u32 reg, u32 mask)
 100 {
 101         return wed_m32(dev, reg, 0, mask);
 102 }
 103
 104 static void
 105 wed_clr(struct mtk_wed_device *dev, u32 reg, u32 mask)
 106 {
 107         return wed_m32(dev, reg, mask, 0);
 108 }
 109
 110 static void
 111 wdma_m32(struct mtk_wed_device *dev, u32 reg, u32 mask, u32 val)
 112 {
 113         wdma_w32(dev, reg, (wdma_r32(dev, reg) & ~mask) | val);
 114 }
 115
 116 static void
 117 wdma_set(struct mtk_wed_device *dev, u32 reg, u32 mask)
 118 {
 119         wdma_m32(dev, reg, 0, mask);
 120 }
 121
 122 static void
 123 wdma_clr(struct mtk_wed_device *dev, u32 reg, u32 mask)
 124 {
 125         wdma_m32(dev, reg, mask, 0);
 126 }
 127
 128 static u32
 129 wifi_r32(struct mtk_wed_device *dev, u32 reg)
 130 {
 131         return readl(dev->wlan.base + reg);
 132 }
 133
 134 static void
 135 wifi_w32(struct mtk_wed_device *dev, u32 reg, u32 val)
 136 {
 137         writel(val, dev->wlan.base + reg);
 138 }
 139
 140 static u32
 141 mtk_wed_read_reset(struct mtk_wed_device *dev)
 142 {
 143         return wed_r32(dev, MTK_WED_RESET);
 144 }
 145
 146 static u32
 147 mtk_wdma_read_reset(struct mtk_wed_device *dev)
 148 {
 149         return wdma_r32(dev, MTK_WDMA_GLO_CFG);
 150 }
 151
 152 static void
 153 mtk_wdma_v3_rx_reset(struct mtk_wed_device *dev)
 154 {
 155         u32 status;
 156
 157         if (!mtk_wed_is_v3_or_greater(dev->hw))
 158                 return;
 159
 160         wdma_clr(dev, MTK_WDMA_PREF_TX_CFG, MTK_WDMA_PREF_TX_CFG_PREF_EN);
 161         wdma_clr(dev, MTK_WDMA_PREF_RX_CFG, MTK_WDMA_PREF_RX_CFG_PREF_EN);
 162
 163         if (read_poll_timeout(wdma_r32, status,
 164                               !(status & MTK_WDMA_PREF_TX_CFG_PREF_BUSY),
 165                               0, 10000, false, dev, MTK_WDMA_PREF_TX_CFG))
 166                 dev_err(dev->hw->dev, "rx reset failed\n");
 167
 168         if (read_poll_timeout(wdma_r32, status,
 169                               !(status & MTK_WDMA_PREF_RX_CFG_PREF_BUSY),
 170                               0, 10000, false, dev, MTK_WDMA_PREF_RX_CFG))
 171                 dev_err(dev->hw->dev, "rx reset failed\n");
 172
 173         wdma_clr(dev, MTK_WDMA_WRBK_TX_CFG, MTK_WDMA_WRBK_TX_CFG_WRBK_EN);
 174         wdma_clr(dev, MTK_WDMA_WRBK_RX_CFG, MTK_WDMA_WRBK_RX_CFG_WRBK_EN);
 175
 176         if (read_poll_timeout(wdma_r32, status,
 177                               !(status & MTK_WDMA_WRBK_TX_CFG_WRBK_BUSY),
 178                               0, 10000, false, dev, MTK_WDMA_WRBK_TX_CFG))
 179                 dev_err(dev->hw->dev, "rx reset failed\n");
 180
 181         if (read_poll_timeout(wdma_r32, status,
 182                               !(status & MTK_WDMA_WRBK_RX_CFG_WRBK_BUSY),
 183                               0, 10000, false, dev, MTK_WDMA_WRBK_RX_CFG))
 184                 dev_err(dev->hw->dev, "rx reset failed\n");
 185
 186         /* prefetch FIFO */
 187         wdma_w32(dev, MTK_WDMA_PREF_RX_FIFO_CFG,
 188                  MTK_WDMA_PREF_RX_FIFO_CFG_RING0_CLEAR |
 189                  MTK_WDMA_PREF_RX_FIFO_CFG_RING1_CLEAR);
 190         wdma_clr(dev, MTK_WDMA_PREF_RX_FIFO_CFG,
 191                  MTK_WDMA_PREF_RX_FIFO_CFG_RING0_CLEAR |
 192                  MTK_WDMA_PREF_RX_FIFO_CFG_RING1_CLEAR);
 193
 194         /* core FIFO */
 195         wdma_w32(dev, MTK_WDMA_XDMA_RX_FIFO_CFG,
 196                  MTK_WDMA_XDMA_RX_FIFO_CFG_RX_PAR_FIFO_CLEAR |
 197                  MTK_WDMA_XDMA_RX_FIFO_CFG_RX_CMD_FIFO_CLEAR |
 198                  MTK_WDMA_XDMA_RX_FIFO_CFG_RX_DMAD_FIFO_CLEAR |
 199                  MTK_WDMA_XDMA_RX_FIFO_CFG_RX_ARR_FIFO_CLEAR |
 200                  MTK_WDMA_XDMA_RX_FIFO_CFG_RX_LEN_FIFO_CLEAR |
 201                  MTK_WDMA_XDMA_RX_FIFO_CFG_RX_WID_FIFO_CLEAR |
 202                  MTK_WDMA_XDMA_RX_FIFO_CFG_RX_BID_FIFO_CLEAR);
 203         wdma_clr(dev, MTK_WDMA_XDMA_RX_FIFO_CFG,
 204                  MTK_WDMA_XDMA_RX_FIFO_CFG_RX_PAR_FIFO_CLEAR |
 205                  MTK_WDMA_XDMA_RX_FIFO_CFG_RX_CMD_FIFO_CLEAR |
 206                  MTK_WDMA_XDMA_RX_FIFO_CFG_RX_DMAD_FIFO_CLEAR |
 207                  MTK_WDMA_XDMA_RX_FIFO_CFG_RX_ARR_FIFO_CLEAR |
 208                  MTK_WDMA_XDMA_RX_FIFO_CFG_RX_LEN_FIFO_CLEAR |
 209                  MTK_WDMA_XDMA_RX_FIFO_CFG_RX_WID_FIFO_CLEAR |
 210                  MTK_WDMA_XDMA_RX_FIFO_CFG_RX_BID_FIFO_CLEAR);
 211
 212         /* writeback FIFO */
 213         wdma_w32(dev, MTK_WDMA_WRBK_RX_FIFO_CFG(0),
 214                  MTK_WDMA_WRBK_RX_FIFO_CFG_RING_CLEAR);
 215         wdma_w32(dev, MTK_WDMA_WRBK_RX_FIFO_CFG(1),
 216                  MTK_WDMA_WRBK_RX_FIFO_CFG_RING_CLEAR);
 217
 218         wdma_clr(dev, MTK_WDMA_WRBK_RX_FIFO_CFG(0),
 219                  MTK_WDMA_WRBK_RX_FIFO_CFG_RING_CLEAR);
 220         wdma_clr(dev, MTK_WDMA_WRBK_RX_FIFO_CFG(1),
 221                  MTK_WDMA_WRBK_RX_FIFO_CFG_RING_CLEAR);
 222
 223         /* prefetch ring status */
 224         wdma_w32(dev, MTK_WDMA_PREF_SIDX_CFG,
 225                  MTK_WDMA_PREF_SIDX_CFG_RX_RING_CLEAR);
 226         wdma_clr(dev, MTK_WDMA_PREF_SIDX_CFG,
 227                  MTK_WDMA_PREF_SIDX_CFG_RX_RING_CLEAR);
 228
 229         /* writeback ring status */
 230         wdma_w32(dev, MTK_WDMA_WRBK_SIDX_CFG,
 231                  MTK_WDMA_WRBK_SIDX_CFG_RX_RING_CLEAR);
 232         wdma_clr(dev, MTK_WDMA_WRBK_SIDX_CFG,
 233                  MTK_WDMA_WRBK_SIDX_CFG_RX_RING_CLEAR);
 234 }
 235
 236 static int
 237 mtk_wdma_rx_reset(struct mtk_wed_device *dev)
 238 {
 239         u32 status, mask = MTK_WDMA_GLO_CFG_RX_DMA_BUSY;
 240         int i, ret;
 241
 242         wdma_clr(dev, MTK_WDMA_GLO_CFG, MTK_WDMA_GLO_CFG_RX_DMA_EN);
 243         ret = readx_poll_timeout(mtk_wdma_read_reset, dev, status,
 244                                  !(status & mask), 0, 10000);
 245         if (ret)
 246                 dev_err(dev->hw->dev, "rx reset failed\n");
 247
 248         mtk_wdma_v3_rx_reset(dev);
 249         wdma_w32(dev, MTK_WDMA_RESET_IDX, MTK_WDMA_RESET_IDX_RX);
 250         wdma_w32(dev, MTK_WDMA_RESET_IDX, 0);
 251
 252         for (i = 0; i < ARRAY_SIZE(dev->rx_wdma); i++) {
 253                 if (dev->rx_wdma[i].desc)
 254                         continue;
 255
 256                 wdma_w32(dev,
 257                          MTK_WDMA_RING_RX(i) + MTK_WED_RING_OFS_CPU_IDX, 0);
 258         }
 259
 260         return ret;
 261 }
 262
 263 static u32
 264 mtk_wed_check_busy(struct mtk_wed_device *dev, u32 reg, u32 mask)
 265 {
 266         return !!(wed_r32(dev, reg) & mask);
 267 }
 268
 269 static int
 270 mtk_wed_poll_busy(struct mtk_wed_device *dev, u32 reg, u32 mask)
 271 {
 272         int sleep = 15000;
 273         int timeout = 100 * sleep;
 274         u32 val;
 275
 276         return read_poll_timeout(mtk_wed_check_busy, val, !val, sleep,
 277                                  timeout, false, dev, reg, mask);
 278 }
 279
 280 static void
 281 mtk_wdma_v3_tx_reset(struct mtk_wed_device *dev)
 282 {
 283         u32 status;
 284
 285         if (!mtk_wed_is_v3_or_greater(dev->hw))
 286                 return;
 287
 288         wdma_clr(dev, MTK_WDMA_PREF_TX_CFG, MTK_WDMA_PREF_TX_CFG_PREF_EN);
 289         wdma_clr(dev, MTK_WDMA_PREF_RX_CFG, MTK_WDMA_PREF_RX_CFG_PREF_EN);
 290
 291         if (read_poll_timeout(wdma_r32, status,
 292                               !(status & MTK_WDMA_PREF_TX_CFG_PREF_BUSY),
 293                               0, 10000, false, dev, MTK_WDMA_PREF_TX_CFG))
 294                 dev_err(dev->hw->dev, "tx reset failed\n");
 295
 296         if (read_poll_timeout(wdma_r32, status,
 297                               !(status & MTK_WDMA_PREF_RX_CFG_PREF_BUSY),
 298                               0, 10000, false, dev, MTK_WDMA_PREF_RX_CFG))
 299                 dev_err(dev->hw->dev, "tx reset failed\n");
 300
 301         wdma_clr(dev, MTK_WDMA_WRBK_TX_CFG, MTK_WDMA_WRBK_TX_CFG_WRBK_EN);
 302         wdma_clr(dev, MTK_WDMA_WRBK_RX_CFG, MTK_WDMA_WRBK_RX_CFG_WRBK_EN);
 303
 304         if (read_poll_timeout(wdma_r32, status,
 305                               !(status & MTK_WDMA_WRBK_TX_CFG_WRBK_BUSY),
 306                               0, 10000, false, dev, MTK_WDMA_WRBK_TX_CFG))
 307                 dev_err(dev->hw->dev, "tx reset failed\n");
 308
 309         if (read_poll_timeout(wdma_r32, status,
 310                               !(status & MTK_WDMA_WRBK_RX_CFG_WRBK_BUSY),
 311                               0, 10000, false, dev, MTK_WDMA_WRBK_RX_CFG))
 312                 dev_err(dev->hw->dev, "tx reset failed\n");
 313
 314         /* prefetch FIFO */
 315         wdma_w32(dev, MTK_WDMA_PREF_TX_FIFO_CFG,
 316                  MTK_WDMA_PREF_TX_FIFO_CFG_RING0_CLEAR |
 317                  MTK_WDMA_PREF_TX_FIFO_CFG_RING1_CLEAR);
 318         wdma_clr(dev, MTK_WDMA_PREF_TX_FIFO_CFG,
 319                  MTK_WDMA_PREF_TX_FIFO_CFG_RING0_CLEAR |
 320                  MTK_WDMA_PREF_TX_FIFO_CFG_RING1_CLEAR);
 321
 322         /* core FIFO */
 323         wdma_w32(dev, MTK_WDMA_XDMA_TX_FIFO_CFG,
 324                  MTK_WDMA_XDMA_TX_FIFO_CFG_TX_PAR_FIFO_CLEAR |
 325                  MTK_WDMA_XDMA_TX_FIFO_CFG_TX_CMD_FIFO_CLEAR |
 326                  MTK_WDMA_XDMA_TX_FIFO_CFG_TX_DMAD_FIFO_CLEAR |
 327                  MTK_WDMA_XDMA_TX_FIFO_CFG_TX_ARR_FIFO_CLEAR);
 328         wdma_clr(dev, MTK_WDMA_XDMA_TX_FIFO_CFG,
 329                  MTK_WDMA_XDMA_TX_FIFO_CFG_TX_PAR_FIFO_CLEAR |
 330                  MTK_WDMA_XDMA_TX_FIFO_CFG_TX_CMD_FIFO_CLEAR |
 331                  MTK_WDMA_XDMA_TX_FIFO_CFG_TX_DMAD_FIFO_CLEAR |
 332                  MTK_WDMA_XDMA_TX_FIFO_CFG_TX_ARR_FIFO_CLEAR);
 333
 334         /* writeback FIFO */
 335         wdma_w32(dev, MTK_WDMA_WRBK_TX_FIFO_CFG(0),
 336                  MTK_WDMA_WRBK_TX_FIFO_CFG_RING_CLEAR);
 337         wdma_w32(dev, MTK_WDMA_WRBK_TX_FIFO_CFG(1),
 338                  MTK_WDMA_WRBK_TX_FIFO_CFG_RING_CLEAR);
 339
 340         wdma_clr(dev, MTK_WDMA_WRBK_TX_FIFO_CFG(0),
 341                  MTK_WDMA_WRBK_TX_FIFO_CFG_RING_CLEAR);
 342         wdma_clr(dev, MTK_WDMA_WRBK_TX_FIFO_CFG(1),
 343                  MTK_WDMA_WRBK_TX_FIFO_CFG_RING_CLEAR);
 344
 345         /* prefetch ring status */
 346         wdma_w32(dev, MTK_WDMA_PREF_SIDX_CFG,
 347                  MTK_WDMA_PREF_SIDX_CFG_TX_RING_CLEAR);
 348         wdma_clr(dev, MTK_WDMA_PREF_SIDX_CFG,
 349                  MTK_WDMA_PREF_SIDX_CFG_TX_RING_CLEAR);
 350
 351         /* writeback ring status */
 352         wdma_w32(dev, MTK_WDMA_WRBK_SIDX_CFG,
 353                  MTK_WDMA_WRBK_SIDX_CFG_TX_RING_CLEAR);
 354         wdma_clr(dev, MTK_WDMA_WRBK_SIDX_CFG,
 355                  MTK_WDMA_WRBK_SIDX_CFG_TX_RING_CLEAR);
 356 }
 357
 358 static void
 359 mtk_wdma_tx_reset(struct mtk_wed_device *dev)
 360 {
 361         u32 status, mask = MTK_WDMA_GLO_CFG_TX_DMA_BUSY;
 362         int i;
 363
 364         wdma_clr(dev, MTK_WDMA_GLO_CFG, MTK_WDMA_GLO_CFG_TX_DMA_EN);
 365         if (readx_poll_timeout(mtk_wdma_read_reset, dev, status,
 366                                !(status & mask), 0, 10000))
 367                 dev_err(dev->hw->dev, "tx reset failed\n");
 368
 369         mtk_wdma_v3_tx_reset(dev);
 370         wdma_w32(dev, MTK_WDMA_RESET_IDX, MTK_WDMA_RESET_IDX_TX);
 371         wdma_w32(dev, MTK_WDMA_RESET_IDX, 0);
 372
 373         for (i = 0; i < ARRAY_SIZE(dev->tx_wdma); i++)
 374                 wdma_w32(dev,
 375                          MTK_WDMA_RING_TX(i) + MTK_WED_RING_OFS_CPU_IDX, 0);
 376 }
 377
 378 static void
 379 mtk_wed_reset(struct mtk_wed_device *dev, u32 mask)
 380 {
 381         u32 status;
 382
 383         wed_w32(dev, MTK_WED_RESET, mask);
 384         if (readx_poll_timeout(mtk_wed_read_reset, dev, status,
 385                                !(status & mask), 0, 1000))
 386                 WARN_ON_ONCE(1);
 387 }
 388
 389 static u32
 390 mtk_wed_wo_read_status(struct mtk_wed_device *dev)
 391 {
 392         return wed_r32(dev, MTK_WED_SCR0 + 4 * MTK_WED_DUMMY_CR_WO_STATUS);
 393 }
 394
 395 static void
 396 mtk_wed_wo_reset(struct mtk_wed_device *dev)
 397 {
 398         struct mtk_wed_wo *wo = dev->hw->wed_wo;
 399         u8 state = MTK_WED_WO_STATE_DISABLE;
 400         void __iomem *reg;
 401         u32 val;
 402
 403         mtk_wdma_tx_reset(dev);
 404         mtk_wed_reset(dev, MTK_WED_RESET_WED);
 405
 406         if (mtk_wed_mcu_send_msg(wo, MTK_WED_MODULE_ID_WO,
 407                                  MTK_WED_WO_CMD_CHANGE_STATE, &state,
 408                                  sizeof(state), false))
 409                 return;
 410
 411         if (readx_poll_timeout(mtk_wed_wo_read_status, dev, val,
 412                                val == MTK_WED_WOIF_DISABLE_DONE,
 413                                100, MTK_WOCPU_TIMEOUT))
 414                 dev_err(dev->hw->dev, "failed to disable wed-wo\n");
 415
 416         reg = ioremap(MTK_WED_WO_CPU_MCUSYS_RESET_ADDR, 4);
 417
 418         val = readl(reg);
 419         switch (dev->hw->index) {
 420         case 0:
 421                 val |= MTK_WED_WO_CPU_WO0_MCUSYS_RESET_MASK;
 422                 writel(val, reg);
 423                 val &= ~MTK_WED_WO_CPU_WO0_MCUSYS_RESET_MASK;
 424                 writel(val, reg);
 425                 break;
 426         case 1:
 427                 val |= MTK_WED_WO_CPU_WO1_MCUSYS_RESET_MASK;
 428                 writel(val, reg);
 429                 val &= ~MTK_WED_WO_CPU_WO1_MCUSYS_RESET_MASK;
 430                 writel(val, reg);
 431                 break;
 432         default:
 433                 break;
 434         }
 435         iounmap(reg);
 436 }
 437
 438 void mtk_wed_fe_reset(void)
 439 {
 440         int i;
 441
 442         mutex_lock(&hw_lock);
 443
 444         for (i = 0; i < ARRAY_SIZE(hw_list); i++) {
 445                 struct mtk_wed_hw *hw = hw_list[i];
 446                 struct mtk_wed_device *dev;
 447                 int err;
 448
 449                 if (!hw)
 450                         break;
 451
 452                 dev = hw->wed_dev;
 453                 if (!dev || !dev->wlan.reset)
 454                         continue;
 455
 456                 /* reset callback blocks until WLAN reset is completed */
 457                 err = dev->wlan.reset(dev);
 458                 if (err)
 459                         dev_err(dev->dev, "wlan reset failed: %d\n", err);
 460         }
 461
 462         mutex_unlock(&hw_lock);
 463 }
 464
 465 void mtk_wed_fe_reset_complete(void)
 466 {
 467         int i;
 468
 469         mutex_lock(&hw_lock);
 470
 471         for (i = 0; i < ARRAY_SIZE(hw_list); i++) {
 472                 struct mtk_wed_hw *hw = hw_list[i];
 473                 struct mtk_wed_device *dev;
 474
 475                 if (!hw)
 476                         break;
 477
 478                 dev = hw->wed_dev;
 479                 if (!dev || !dev->wlan.reset_complete)
 480                         continue;
 481
 482                 dev->wlan.reset_complete(dev);
 483         }
 484
 485         mutex_unlock(&hw_lock);
 486 }
 487
 488 static struct mtk_wed_hw *
 489 mtk_wed_assign(struct mtk_wed_device *dev)
 490 {
 491         struct mtk_wed_hw *hw;
 492         int i;
 493
 494         if (dev->wlan.bus_type == MTK_WED_BUS_PCIE) {
 495                 hw = hw_list[pci_domain_nr(dev->wlan.pci_dev->bus)];
 496                 if (!hw)
 497                         return NULL;
 498
 499                 if (!hw->wed_dev)
 500                         goto out;
 501
 502                 if (mtk_wed_is_v1(hw))
 503                         return NULL;
 504
 505                 /* MT7986 WED devices do not have any pcie slot restrictions */
 506         }
 507         /* MT7986 PCIE or AXI */
 508         for (i = 0; i < ARRAY_SIZE(hw_list); i++) {
 509                 hw = hw_list[i];
 510                 if (hw && !hw->wed_dev)
 511                         goto out;
 512         }
 513
 514         return NULL;
 515
 516 out:
 517         hw->wed_dev = dev;
 518         return hw;
 519 }
 520
 521 static int
 522 mtk_wed_amsdu_buffer_alloc(struct mtk_wed_device *dev)
 523 {
 524         struct mtk_wed_hw *hw = dev->hw;
 525         struct mtk_wed_amsdu *wed_amsdu;
 526         int i;
 527
 528         if (!mtk_wed_is_v3_or_greater(hw))
 529                 return 0;
 530
 531         wed_amsdu = devm_kcalloc(hw->dev, MTK_WED_AMSDU_NPAGES,
 532                                  sizeof(*wed_amsdu), GFP_KERNEL);
 533         if (!wed_amsdu)
 534                 return -ENOMEM;
 535
 536         for (i = 0; i < MTK_WED_AMSDU_NPAGES; i++) {
 537                 void *ptr;
 538
 539                 /* each segment is 64K */
 540                 ptr = (void *)__get_free_pages(GFP_KERNEL | __GFP_NOWARN |
 541                                                __GFP_ZERO | __GFP_COMP |
 542                                                GFP_DMA32,
 543                                                get_order(MTK_WED_AMSDU_BUF_SIZE));
 544                 if (!ptr)
 545                         goto error;
 546
 547                 wed_amsdu[i].txd = ptr;
 548                 wed_amsdu[i].txd_phy = dma_map_single(hw->dev, ptr,
 549                                                       MTK_WED_AMSDU_BUF_SIZE,
 550                                                       DMA_TO_DEVICE);
 551                 if (dma_mapping_error(hw->dev, wed_amsdu[i].txd_phy))
 552                         goto error;
 553         }
 554         dev->hw->wed_amsdu = wed_amsdu;
 555
 556         return 0;
 557
 558 error:
 559         for (i--; i >= 0; i--)
 560                 dma_unmap_single(hw->dev, wed_amsdu[i].txd_phy,
 561                                  MTK_WED_AMSDU_BUF_SIZE, DMA_TO_DEVICE);
 562         return -ENOMEM;
 563 }
 564
 565 static void
 566 mtk_wed_amsdu_free_buffer(struct mtk_wed_device *dev)
 567 {
 568         struct mtk_wed_amsdu *wed_amsdu = dev->hw->wed_amsdu;
 569         int i;
 570
 571         if (!wed_amsdu)
 572                 return;
 573
 574         for (i = 0; i < MTK_WED_AMSDU_NPAGES; i++) {
 575                 dma_unmap_single(dev->hw->dev, wed_amsdu[i].txd_phy,
 576                                  MTK_WED_AMSDU_BUF_SIZE, DMA_TO_DEVICE);
 577                 free_pages((unsigned long)wed_amsdu[i].txd,
 578                            get_order(MTK_WED_AMSDU_BUF_SIZE));
 579         }
 580 }
 581
 582 static int
 583 mtk_wed_amsdu_init(struct mtk_wed_device *dev)
 584 {
 585         struct mtk_wed_amsdu *wed_amsdu = dev->hw->wed_amsdu;
 586         int i, ret;
 587
 588         if (!wed_amsdu)
 589                 return 0;
 590
 591         for (i = 0; i < MTK_WED_AMSDU_NPAGES; i++)
 592                 wed_w32(dev, MTK_WED_AMSDU_HIFTXD_BASE_L(i),
 593                         wed_amsdu[i].txd_phy);
 594
 595         /* init all sta parameter */
 596         wed_w32(dev, MTK_WED_AMSDU_STA_INFO_INIT, MTK_WED_AMSDU_STA_RMVL |
 597                 MTK_WED_AMSDU_STA_WTBL_HDRT_MODE |
 598                 FIELD_PREP(MTK_WED_AMSDU_STA_MAX_AMSDU_LEN,
 599                            dev->wlan.amsdu_max_len >> 8) |
 600                 FIELD_PREP(MTK_WED_AMSDU_STA_MAX_AMSDU_NUM,
 601                            dev->wlan.amsdu_max_subframes));
 602
 603         wed_w32(dev, MTK_WED_AMSDU_STA_INFO, MTK_WED_AMSDU_STA_INFO_DO_INIT);
 604
 605         ret = mtk_wed_poll_busy(dev, MTK_WED_AMSDU_STA_INFO,
 606                                 MTK_WED_AMSDU_STA_INFO_DO_INIT);
 607         if (ret) {
 608                 dev_err(dev->hw->dev, "amsdu initialization failed\n");
 609                 return ret;
 610         }
 611
 612         /* init partial amsdu offload txd src */
 613         wed_set(dev, MTK_WED_AMSDU_HIFTXD_CFG,
 614                 FIELD_PREP(MTK_WED_AMSDU_HIFTXD_SRC, dev->hw->index));
 615
 616         /* init qmem */
 617         wed_set(dev, MTK_WED_AMSDU_PSE, MTK_WED_AMSDU_PSE_RESET);
 618         ret = mtk_wed_poll_busy(dev, MTK_WED_MON_AMSDU_QMEM_STS1, BIT(29));
 619         if (ret) {
 620                 pr_info("%s: amsdu qmem initialization failed\n", __func__);
 621                 return ret;
 622         }
 623
 624         /* eagle E1 PCIE1 tx ring 22 flow control issue */
 625         if (dev->wlan.id == 0x7991)
 626                 wed_clr(dev, MTK_WED_AMSDU_FIFO, MTK_WED_AMSDU_IS_PRIOR0_RING);
 627
 628         wed_set(dev, MTK_WED_CTRL, MTK_WED_CTRL_TX_AMSDU_EN);
 629
 630         return 0;
 631 }
 632
 633 static int
 634 mtk_wed_tx_buffer_alloc(struct mtk_wed_device *dev)
 635 {
 636         u32 desc_size = dev->hw->soc->tx_ring_desc_size;
 637         int i, page_idx = 0, n_pages, ring_size;
 638         int token = dev->wlan.token_start;
 639         struct mtk_wed_buf *page_list;
 640         dma_addr_t desc_phys;
 641         void *desc_ptr;
 642
 643         if (!mtk_wed_is_v3_or_greater(dev->hw)) {
 644                 ring_size = dev->wlan.nbuf & ~(MTK_WED_BUF_PER_PAGE - 1);
 645                 dev->tx_buf_ring.size = ring_size;
 646         } else {
 647                 dev->tx_buf_ring.size = MTK_WED_TX_BM_DMA_SIZE;
 648                 ring_size = MTK_WED_TX_BM_PKT_CNT;
 649         }
 650         n_pages = dev->tx_buf_ring.size / MTK_WED_BUF_PER_PAGE;
 651
 652         page_list = kcalloc(n_pages, sizeof(*page_list), GFP_KERNEL);
 653         if (!page_list)
 654                 return -ENOMEM;
 655
 656         dev->tx_buf_ring.pages = page_list;
 657
 658         desc_ptr = dma_alloc_coherent(dev->hw->dev,
 659                                       dev->tx_buf_ring.size * desc_size,
 660                                       &desc_phys, GFP_KERNEL);
 661         if (!desc_ptr)
 662                 return -ENOMEM;
 663
 664         dev->tx_buf_ring.desc = desc_ptr;
 665         dev->tx_buf_ring.desc_phys = desc_phys;
 666
 667         for (i = 0; i < ring_size; i += MTK_WED_BUF_PER_PAGE) {
 668                 dma_addr_t page_phys, buf_phys;
 669                 struct page *page;
 670                 void *buf;
 671                 int s;
 672
 673                 page = __dev_alloc_page(GFP_KERNEL);
 674                 if (!page)
 675                         return -ENOMEM;
 676
 677                 page_phys = dma_map_page(dev->hw->dev, page, 0, PAGE_SIZE,
 678                                          DMA_BIDIRECTIONAL);
 679                 if (dma_mapping_error(dev->hw->dev, page_phys)) {
 680                         __free_page(page);
 681                         return -ENOMEM;
 682                 }
 683
 684                 page_list[page_idx].p = page;
 685                 page_list[page_idx++].phy_addr = page_phys;
 686                 dma_sync_single_for_cpu(dev->hw->dev, page_phys, PAGE_SIZE,
 687                                         DMA_BIDIRECTIONAL);
 688
 689                 buf = page_to_virt(page);
 690                 buf_phys = page_phys;
 691
 692                 for (s = 0; s < MTK_WED_BUF_PER_PAGE; s++) {
 693                         struct mtk_wdma_desc *desc = desc_ptr;
 694                         u32 ctrl;
 695
 696                         desc->buf0 = cpu_to_le32(buf_phys);
 697                         if (!mtk_wed_is_v3_or_greater(dev->hw)) {
 698                                 u32 txd_size;
 699
 700                                 txd_size = dev->wlan.init_buf(buf, buf_phys,
 701                                                               token++);
 702                                 desc->buf1 = cpu_to_le32(buf_phys + txd_size);
 703                                 ctrl = FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN0, txd_size);
 704                                 if (mtk_wed_is_v1(dev->hw))
 705                                         ctrl |= MTK_WDMA_DESC_CTRL_LAST_SEG1 |
 706                                                 FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN1,
 707                                                            MTK_WED_BUF_SIZE - txd_size);
 708                                 else
 709                                         ctrl |= MTK_WDMA_DESC_CTRL_LAST_SEG0 |
 710                                                 FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN1_V2,
 711                                                            MTK_WED_BUF_SIZE - txd_size);
 712                                 desc->info = 0;
 713                         } else {
 714                                 ctrl = token << 16 | TX_DMA_PREP_ADDR64(buf_phys);
 715                         }
 716                         desc->ctrl = cpu_to_le32(ctrl);
 717
 718                         desc_ptr += desc_size;
 719                         buf += MTK_WED_BUF_SIZE;
 720                         buf_phys += MTK_WED_BUF_SIZE;
 721                 }
 722
 723                 dma_sync_single_for_device(dev->hw->dev, page_phys, PAGE_SIZE,
 724                                            DMA_BIDIRECTIONAL);
 725         }
 726
 727         return 0;
 728 }
 729
 730 static void
 731 mtk_wed_free_tx_buffer(struct mtk_wed_device *dev)
 732 {
 733         struct mtk_wed_buf *page_list = dev->tx_buf_ring.pages;
 734         struct mtk_wed_hw *hw = dev->hw;
 735         int i, page_idx = 0;
 736
 737         if (!page_list)
 738                 return;
 739
 740         if (!dev->tx_buf_ring.desc)
 741                 goto free_pagelist;
 742
 743         for (i = 0; i < dev->tx_buf_ring.size; i += MTK_WED_BUF_PER_PAGE) {
 744                 dma_addr_t page_phy = page_list[page_idx].phy_addr;
 745                 void *page = page_list[page_idx++].p;
 746
 747                 if (!page)
 748                         break;
 749
 750                 dma_unmap_page(dev->hw->dev, page_phy, PAGE_SIZE,
 751                                DMA_BIDIRECTIONAL);
 752                 __free_page(page);
 753         }
 754
 755         dma_free_coherent(dev->hw->dev,
 756                           dev->tx_buf_ring.size * hw->soc->tx_ring_desc_size,
 757                           dev->tx_buf_ring.desc,
 758                           dev->tx_buf_ring.desc_phys);
 759
 760 free_pagelist:
 761         kfree(page_list);
 762 }
 763
 764 static int
 765 mtk_wed_hwrro_buffer_alloc(struct mtk_wed_device *dev)
 766 {
 767         int n_pages = MTK_WED_RX_PG_BM_CNT / MTK_WED_RX_BUF_PER_PAGE;
 768         struct mtk_wed_buf *page_list;
 769         struct mtk_wed_bm_desc *desc;
 770         dma_addr_t desc_phys;
 771         int i, page_idx = 0;
 772
 773         if (!dev->wlan.hw_rro)
 774                 return 0;
 775
 776         page_list = kcalloc(n_pages, sizeof(*page_list), GFP_KERNEL);
 777         if (!page_list)
 778                 return -ENOMEM;
 779
 780         dev->hw_rro.size = dev->wlan.rx_nbuf & ~(MTK_WED_BUF_PER_PAGE - 1);
 781         dev->hw_rro.pages = page_list;
 782         desc = dma_alloc_coherent(dev->hw->dev,
 783                                   dev->wlan.rx_nbuf * sizeof(*desc),
 784                                   &desc_phys, GFP_KERNEL);
 785         if (!desc)
 786                 return -ENOMEM;
 787
 788         dev->hw_rro.desc = desc;
 789         dev->hw_rro.desc_phys = desc_phys;
 790
 791         for (i = 0; i < MTK_WED_RX_PG_BM_CNT; i += MTK_WED_RX_BUF_PER_PAGE) {
 792                 dma_addr_t page_phys, buf_phys;
 793                 struct page *page;
 794                 int s;
 795
 796                 page = __dev_alloc_page(GFP_KERNEL);
 797                 if (!page)
 798                         return -ENOMEM;
 799
 800                 page_phys = dma_map_page(dev->hw->dev, page, 0, PAGE_SIZE,
 801                                          DMA_BIDIRECTIONAL);
 802                 if (dma_mapping_error(dev->hw->dev, page_phys)) {
 803                         __free_page(page);
 804                         return -ENOMEM;
 805                 }
 806
 807                 page_list[page_idx].p = page;
 808                 page_list[page_idx++].phy_addr = page_phys;
 809                 dma_sync_single_for_cpu(dev->hw->dev, page_phys, PAGE_SIZE,
 810                                         DMA_BIDIRECTIONAL);
 811
 812                 buf_phys = page_phys;
 813                 for (s = 0; s < MTK_WED_RX_BUF_PER_PAGE; s++) {
 814                         desc->buf0 = cpu_to_le32(buf_phys);
 815                         desc->token = cpu_to_le32(RX_DMA_PREP_ADDR64(buf_phys));
 816                         buf_phys += MTK_WED_PAGE_BUF_SIZE;
 817                         desc++;
 818                 }
 819
 820                 dma_sync_single_for_device(dev->hw->dev, page_phys, PAGE_SIZE,
 821                                            DMA_BIDIRECTIONAL);
 822         }
 823
 824         return 0;
 825 }
 826
 827 static int
 828 mtk_wed_rx_buffer_alloc(struct mtk_wed_device *dev)
 829 {
 830         struct mtk_wed_bm_desc *desc;
 831         dma_addr_t desc_phys;
 832
 833         dev->rx_buf_ring.size = dev->wlan.rx_nbuf;
 834         desc = dma_alloc_coherent(dev->hw->dev,
 835                                   dev->wlan.rx_nbuf * sizeof(*desc),
 836                                   &desc_phys, GFP_KERNEL);
 837         if (!desc)
 838                 return -ENOMEM;
 839
 840         dev->rx_buf_ring.desc = desc;
 841         dev->rx_buf_ring.desc_phys = desc_phys;
 842         dev->wlan.init_rx_buf(dev, dev->wlan.rx_npkt);
 843
 844         return mtk_wed_hwrro_buffer_alloc(dev);
 845 }
 846
 847 static void
 848 mtk_wed_hwrro_free_buffer(struct mtk_wed_device *dev)
 849 {
 850         struct mtk_wed_buf *page_list = dev->hw_rro.pages;
 851         struct mtk_wed_bm_desc *desc = dev->hw_rro.desc;
 852         int i, page_idx = 0;
 853
 854         if (!dev->wlan.hw_rro)
 855                 return;
 856
 857         if (!page_list)
 858                 return;
 859
 860         if (!desc)
 861                 goto free_pagelist;
 862
 863         for (i = 0; i < MTK_WED_RX_PG_BM_CNT; i += MTK_WED_RX_BUF_PER_PAGE) {
 864                 dma_addr_t buf_addr = page_list[page_idx].phy_addr;
 865                 void *page = page_list[page_idx++].p;
 866
 867                 if (!page)
 868                         break;
 869
 870                 dma_unmap_page(dev->hw->dev, buf_addr, PAGE_SIZE,
 871                                DMA_BIDIRECTIONAL);
 872                 __free_page(page);
 873         }
 874
 875         dma_free_coherent(dev->hw->dev, dev->hw_rro.size * sizeof(*desc),
 876                           desc, dev->hw_rro.desc_phys);
 877
 878 free_pagelist:
 879         kfree(page_list);
 880 }
 881
 882 static void
 883 mtk_wed_free_rx_buffer(struct mtk_wed_device *dev)
 884 {
 885         struct mtk_wed_bm_desc *desc = dev->rx_buf_ring.desc;
 886
 887         if (!desc)
 888                 return;
 889
 890         dev->wlan.release_rx_buf(dev);
 891         dma_free_coherent(dev->hw->dev, dev->rx_buf_ring.size * sizeof(*desc),
 892                           desc, dev->rx_buf_ring.desc_phys);
 893
 894         mtk_wed_hwrro_free_buffer(dev);
 895 }
 896
 897 static void
 898 mtk_wed_hwrro_init(struct mtk_wed_device *dev)
 899 {
 900         if (!mtk_wed_get_rx_capa(dev) || !dev->wlan.hw_rro)
 901                 return;
 902
 903         wed_set(dev, MTK_WED_RRO_PG_BM_RX_DMAM,
 904                 FIELD_PREP(MTK_WED_RRO_PG_BM_RX_SDL0, 128));
 905
 906         wed_w32(dev, MTK_WED_RRO_PG_BM_BASE, dev->hw_rro.desc_phys);
 907
 908         wed_w32(dev, MTK_WED_RRO_PG_BM_INIT_PTR,
 909                 MTK_WED_RRO_PG_BM_INIT_SW_TAIL_IDX |
 910                 FIELD_PREP(MTK_WED_RRO_PG_BM_SW_TAIL_IDX,
 911                            MTK_WED_RX_PG_BM_CNT));
 912
 913         /* enable rx_page_bm to fetch dmad */
 914         wed_set(dev, MTK_WED_CTRL, MTK_WED_CTRL_WED_RX_PG_BM_EN);
 915 }
 916
 917 static void
 918 mtk_wed_rx_buffer_hw_init(struct mtk_wed_device *dev)
 919 {
 920         wed_w32(dev, MTK_WED_RX_BM_RX_DMAD,
 921                 FIELD_PREP(MTK_WED_RX_BM_RX_DMAD_SDL0, dev->wlan.rx_size));
 922         wed_w32(dev, MTK_WED_RX_BM_BASE, dev->rx_buf_ring.desc_phys);
 923         wed_w32(dev, MTK_WED_RX_BM_INIT_PTR, MTK_WED_RX_BM_INIT_SW_TAIL |
 924                 FIELD_PREP(MTK_WED_RX_BM_SW_TAIL, dev->wlan.rx_npkt));
 925         wed_w32(dev, MTK_WED_RX_BM_DYN_ALLOC_TH,
 926                 FIELD_PREP(MTK_WED_RX_BM_DYN_ALLOC_TH_H, 0xffff));
 927         wed_set(dev, MTK_WED_CTRL, MTK_WED_CTRL_WED_RX_BM_EN);
 928
 929         mtk_wed_hwrro_init(dev);
 930 }
 931
 932 static void
 933 mtk_wed_free_ring(struct mtk_wed_device *dev, struct mtk_wed_ring *ring)
 934 {
 935         if (!ring->desc)
 936                 return;
 937
 938         dma_free_coherent(dev->hw->dev, ring->size * ring->desc_size,
 939                           ring->desc, ring->desc_phys);
 940 }
 941
 942 static void
 943 mtk_wed_free_rx_rings(struct mtk_wed_device *dev)
 944 {
 945         mtk_wed_free_rx_buffer(dev);
 946         mtk_wed_free_ring(dev, &dev->rro.ring);
 947 }
 948
 949 static void
 950 mtk_wed_free_tx_rings(struct mtk_wed_device *dev)
 951 {
 952         int i;
 953
 954         for (i = 0; i < ARRAY_SIZE(dev->tx_ring); i++)
 955                 mtk_wed_free_ring(dev, &dev->tx_ring[i]);
 956         for (i = 0; i < ARRAY_SIZE(dev->rx_wdma); i++)
 957                 mtk_wed_free_ring(dev, &dev->rx_wdma[i]);
 958 }
 959
 960 static void
 961 mtk_wed_set_ext_int(struct mtk_wed_device *dev, bool en)
 962 {
 963         u32 mask = MTK_WED_EXT_INT_STATUS_ERROR_MASK;
 964
 965         switch (dev->hw->version) {
 966         case 1:
 967                 mask |= MTK_WED_EXT_INT_STATUS_TX_DRV_R_RESP_ERR;
 968                 break;
 969         case 2:
 970                 mask |= MTK_WED_EXT_INT_STATUS_RX_FBUF_LO_TH |
 971                         MTK_WED_EXT_INT_STATUS_RX_FBUF_HI_TH |
 972                         MTK_WED_EXT_INT_STATUS_RX_DRV_COHERENT |
 973                         MTK_WED_EXT_INT_STATUS_TX_DMA_W_RESP_ERR;
 974                 break;
 975         case 3:
 976                 mask = MTK_WED_EXT_INT_STATUS_RX_DRV_COHERENT |
 977                        MTK_WED_EXT_INT_STATUS_TKID_WO_PYLD;
 978                 break;
 979         default:
 980                 break;
 981         }
 982
 983         if (!dev->hw->num_flows)
 984                 mask &= ~MTK_WED_EXT_INT_STATUS_TKID_WO_PYLD;
 985
 986         wed_w32(dev, MTK_WED_EXT_INT_MASK, en ? mask : 0);
 987         wed_r32(dev, MTK_WED_EXT_INT_MASK);
 988 }
 989
 990 static void
 991 mtk_wed_set_512_support(struct mtk_wed_device *dev, bool enable)
 992 {
 993         if (!mtk_wed_is_v2(dev->hw))
 994                 return;
 995
 996         if (enable) {
 997                 wed_w32(dev, MTK_WED_TXDP_CTRL, MTK_WED_TXDP_DW9_OVERWR);
 998                 wed_w32(dev, MTK_WED_TXP_DW1,
 999                         FIELD_PREP(MTK_WED_WPDMA_WRITE_TXP, 0x0103));
1000         } else {
1001                 wed_w32(dev, MTK_WED_TXP_DW1,
1002                         FIELD_PREP(MTK_WED_WPDMA_WRITE_TXP, 0x0100));
1003                 wed_clr(dev, MTK_WED_TXDP_CTRL, MTK_WED_TXDP_DW9_OVERWR);
1004         }
1005 }
1006
1007 static int
1008 mtk_wed_check_wfdma_rx_fill(struct mtk_wed_device *dev,
1009                             struct mtk_wed_ring *ring)
1010 {
1011         int i;
1012
1013         for (i = 0; i < 3; i++) {
1014                 u32 cur_idx = readl(ring->wpdma + MTK_WED_RING_OFS_CPU_IDX);
1015
1016                 if (cur_idx == MTK_WED_RX_RING_SIZE - 1)
1017                         break;
1018
1019                 usleep_range(100000, 200000);
1020         }
1021
1022         if (i == 3) {
1023                 dev_err(dev->hw->dev, "rx dma enable failed\n");
1024                 return -ETIMEDOUT;
1025         }
1026
1027         return 0;
1028 }
1029
1030 static void
1031 mtk_wed_dma_disable(struct mtk_wed_device *dev)
1032 {
1033         wed_clr(dev, MTK_WED_WPDMA_GLO_CFG,
1034                 MTK_WED_WPDMA_GLO_CFG_TX_DRV_EN |
1035                 MTK_WED_WPDMA_GLO_CFG_RX_DRV_EN);
1036
1037         wed_clr(dev, MTK_WED_WDMA_GLO_CFG, MTK_WED_WDMA_GLO_CFG_RX_DRV_EN);
1038
1039         wed_clr(dev, MTK_WED_GLO_CFG,
1040                 MTK_WED_GLO_CFG_TX_DMA_EN |
1041                 MTK_WED_GLO_CFG_RX_DMA_EN);
1042
1043         wdma_clr(dev, MTK_WDMA_GLO_CFG,
1044                  MTK_WDMA_GLO_CFG_TX_DMA_EN |
1045                  MTK_WDMA_GLO_CFG_RX_INFO1_PRERES |
1046                  MTK_WDMA_GLO_CFG_RX_INFO2_PRERES);
1047
1048         if (mtk_wed_is_v1(dev->hw)) {
1049                 regmap_write(dev->hw->mirror, dev->hw->index * 4, 0);
1050                 wdma_clr(dev, MTK_WDMA_GLO_CFG,
1051                          MTK_WDMA_GLO_CFG_RX_INFO3_PRERES);
1052         } else {
1053                 wed_clr(dev, MTK_WED_WPDMA_GLO_CFG,
1054                         MTK_WED_WPDMA_GLO_CFG_RX_DRV_R0_PKT_PROC |
1055                         MTK_WED_WPDMA_GLO_CFG_RX_DRV_R0_CRX_SYNC);
1056
1057                 wed_clr(dev, MTK_WED_WPDMA_RX_D_GLO_CFG,
1058                         MTK_WED_WPDMA_RX_D_RX_DRV_EN);
1059                 wed_clr(dev, MTK_WED_WDMA_GLO_CFG,
1060                         MTK_WED_WDMA_GLO_CFG_TX_DDONE_CHK);
1061
1062                 if (mtk_wed_is_v3_or_greater(dev->hw) &&
1063                     mtk_wed_get_rx_capa(dev)) {
1064                         wdma_clr(dev, MTK_WDMA_PREF_TX_CFG,
1065                                  MTK_WDMA_PREF_TX_CFG_PREF_EN);
1066                         wdma_clr(dev, MTK_WDMA_PREF_RX_CFG,
1067                                  MTK_WDMA_PREF_RX_CFG_PREF_EN);
1068                 }
1069         }
1070
1071         mtk_wed_set_512_support(dev, false);
1072 }
1073
1074 static void
1075 mtk_wed_stop(struct mtk_wed_device *dev)
1076 {
1077         mtk_wed_dma_disable(dev);
1078         mtk_wed_set_ext_int(dev, false);
1079
1080         wed_w32(dev, MTK_WED_WPDMA_INT_TRIGGER, 0);
1081         wed_w32(dev, MTK_WED_WDMA_INT_TRIGGER, 0);
1082         wdma_w32(dev, MTK_WDMA_INT_MASK, 0);
1083         wdma_w32(dev, MTK_WDMA_INT_GRP2, 0);
1084
1085         if (!mtk_wed_get_rx_capa(dev))
1086                 return;
1087
1088         wed_w32(dev, MTK_WED_EXT_INT_MASK1, 0);
1089         wed_w32(dev, MTK_WED_EXT_INT_MASK2, 0);
1090 }
1091
1092 static void
1093 mtk_wed_deinit(struct mtk_wed_device *dev)
1094 {
1095         mtk_wed_stop(dev);
1096
1097         wed_clr(dev, MTK_WED_CTRL,
1098                 MTK_WED_CTRL_WDMA_INT_AGENT_EN |
1099                 MTK_WED_CTRL_WPDMA_INT_AGENT_EN |
1100                 MTK_WED_CTRL_WED_TX_BM_EN |
1101                 MTK_WED_CTRL_WED_TX_FREE_AGENT_EN);
1102
1103         if (mtk_wed_is_v1(dev->hw))
1104                 return;
1105
1106         wed_clr(dev, MTK_WED_CTRL,
1107                 MTK_WED_CTRL_RX_ROUTE_QM_EN |
1108                 MTK_WED_CTRL_WED_RX_BM_EN |
1109                 MTK_WED_CTRL_RX_RRO_QM_EN);
1110
1111         if (mtk_wed_is_v3_or_greater(dev->hw)) {
1112                 wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_TX_AMSDU_EN);
1113                 wed_clr(dev, MTK_WED_RESET, MTK_WED_RESET_TX_AMSDU);
1114                 wed_clr(dev, MTK_WED_PCIE_INT_CTRL,
1115                         MTK_WED_PCIE_INT_CTRL_MSK_EN_POLA |
1116                         MTK_WED_PCIE_INT_CTRL_MSK_IRQ_FILTER);
1117         }
1118 }
1119
1120 static void
1121 __mtk_wed_detach(struct mtk_wed_device *dev)
1122 {
1123         struct mtk_wed_hw *hw = dev->hw;
1124
1125         mtk_wed_deinit(dev);
1126
1127         mtk_wdma_rx_reset(dev);
1128         mtk_wed_reset(dev, MTK_WED_RESET_WED);
1129         mtk_wed_amsdu_free_buffer(dev);
1130         mtk_wed_free_tx_buffer(dev);
1131         mtk_wed_free_tx_rings(dev);
1132
1133         if (mtk_wed_get_rx_capa(dev)) {
1134                 if (hw->wed_wo)
1135                         mtk_wed_wo_reset(dev);
1136                 mtk_wed_free_rx_rings(dev);
1137                 if (hw->wed_wo)
1138                         mtk_wed_wo_deinit(hw);
1139         }
1140
1141         if (dev->wlan.bus_type == MTK_WED_BUS_PCIE) {
1142                 struct device_node *wlan_node;
1143
1144                 wlan_node = dev->wlan.pci_dev->dev.of_node;
1145                 if (of_dma_is_coherent(wlan_node) && hw->hifsys)
1146                         regmap_update_bits(hw->hifsys, HIFSYS_DMA_AG_MAP,
1147                                            BIT(hw->index), BIT(hw->index));
1148         }
1149
1150         if ((!hw_list[!hw->index] || !hw_list[!hw->index]->wed_dev) &&
1151             hw->eth->dma_dev != hw->eth->dev)
1152                 mtk_eth_set_dma_device(hw->eth, hw->eth->dev);
1153
1154         memset(dev, 0, sizeof(*dev));
1155         module_put(THIS_MODULE);
1156
1157         hw->wed_dev = NULL;
1158 }
1159
1160 static void
1161 mtk_wed_detach(struct mtk_wed_device *dev)
1162 {
1163         mutex_lock(&hw_lock);
1164         __mtk_wed_detach(dev);
1165         mutex_unlock(&hw_lock);
1166 }
1167
1168 static void
1169 mtk_wed_bus_init(struct mtk_wed_device *dev)
1170 {
1171         switch (dev->wlan.bus_type) {
1172         case MTK_WED_BUS_PCIE: {
1173                 struct device_node *np = dev->hw->eth->dev->of_node;
1174
1175                 if (mtk_wed_is_v2(dev->hw)) {
1176                         struct regmap *regs;
1177
1178                         regs = syscon_regmap_lookup_by_phandle(np,
1179                                                                "mediatek,wed-pcie");
1180                         if (IS_ERR(regs))
1181                                 break;
1182
1183                         regmap_update_bits(regs, 0, BIT(0), BIT(0));
1184                 }
1185
1186                 if (dev->wlan.msi) {
1187                         wed_w32(dev, MTK_WED_PCIE_CFG_INTM,
1188                                 dev->hw->pcie_base | 0xc08);
1189                         wed_w32(dev, MTK_WED_PCIE_CFG_BASE,
1190                                 dev->hw->pcie_base | 0xc04);
1191                         wed_w32(dev, MTK_WED_PCIE_INT_TRIGGER, BIT(8));
1192                 } else {
1193                         wed_w32(dev, MTK_WED_PCIE_CFG_INTM,
1194                                 dev->hw->pcie_base | 0x180);
1195                         wed_w32(dev, MTK_WED_PCIE_CFG_BASE,
1196                                 dev->hw->pcie_base | 0x184);
1197                         wed_w32(dev, MTK_WED_PCIE_INT_TRIGGER, BIT(24));
1198                 }
1199
1200                 wed_w32(dev, MTK_WED_PCIE_INT_CTRL,
1201                         FIELD_PREP(MTK_WED_PCIE_INT_CTRL_POLL_EN, 2));
1202
1203                 /* pcie interrupt control: pola/source selection */
1204                 wed_set(dev, MTK_WED_PCIE_INT_CTRL,
1205                         MTK_WED_PCIE_INT_CTRL_MSK_EN_POLA |
1206                         MTK_WED_PCIE_INT_CTRL_MSK_IRQ_FILTER  |
1207                         FIELD_PREP(MTK_WED_PCIE_INT_CTRL_SRC_SEL,
1208                                    dev->hw->index));
1209                 break;
1210         }
1211         case MTK_WED_BUS_AXI:
1212                 wed_set(dev, MTK_WED_WPDMA_INT_CTRL,
1213                         MTK_WED_WPDMA_INT_CTRL_SIG_SRC |
1214                         FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_SRC_SEL, 0));
1215                 break;
1216         default:
1217                 break;
1218         }
1219 }
1220
1221 static void
1222 mtk_wed_set_wpdma(struct mtk_wed_device *dev)
1223 {
1224         int i;
1225
1226         if (mtk_wed_is_v1(dev->hw)) {
1227                 wed_w32(dev, MTK_WED_WPDMA_CFG_BASE,  dev->wlan.wpdma_phys);
1228                 return;
1229         }
1230
1231         mtk_wed_bus_init(dev);
1232
1233         wed_w32(dev, MTK_WED_WPDMA_CFG_BASE, dev->wlan.wpdma_int);
1234         wed_w32(dev, MTK_WED_WPDMA_CFG_INT_MASK, dev->wlan.wpdma_mask);
1235         wed_w32(dev, MTK_WED_WPDMA_CFG_TX, dev->wlan.wpdma_tx);
1236         wed_w32(dev, MTK_WED_WPDMA_CFG_TX_FREE, dev->wlan.wpdma_txfree);
1237
1238         if (!mtk_wed_get_rx_capa(dev))
1239                 return;
1240
1241         wed_w32(dev, MTK_WED_WPDMA_RX_GLO_CFG, dev->wlan.wpdma_rx_glo);
1242         wed_w32(dev, dev->hw->soc->regmap.wpdma_rx_ring0, dev->wlan.wpdma_rx);
1243
1244         if (!dev->wlan.hw_rro)
1245                 return;
1246
1247         wed_w32(dev, MTK_WED_RRO_RX_D_CFG(0), dev->wlan.wpdma_rx_rro[0]);
1248         wed_w32(dev, MTK_WED_RRO_RX_D_CFG(1), dev->wlan.wpdma_rx_rro[1]);
1249         for (i = 0; i < MTK_WED_RX_PAGE_QUEUES; i++)
1250                 wed_w32(dev, MTK_WED_RRO_MSDU_PG_RING_CFG(i),
1251                         dev->wlan.wpdma_rx_pg + i * 0x10);
1252 }
1253
1254 static void
1255 mtk_wed_hw_init_early(struct mtk_wed_device *dev)
1256 {
1257         u32 set = FIELD_PREP(MTK_WED_WDMA_GLO_CFG_BT_SIZE, 2);
1258         u32 mask = MTK_WED_WDMA_GLO_CFG_BT_SIZE;
1259
1260         mtk_wed_deinit(dev);
1261         mtk_wed_reset(dev, MTK_WED_RESET_WED);
1262         mtk_wed_set_wpdma(dev);
1263
1264         if (!mtk_wed_is_v3_or_greater(dev->hw)) {
1265                 mask |= MTK_WED_WDMA_GLO_CFG_DYNAMIC_DMAD_RECYCLE |
1266                         MTK_WED_WDMA_GLO_CFG_RX_DIS_FSM_AUTO_IDLE;
1267                 set |= MTK_WED_WDMA_GLO_CFG_DYNAMIC_SKIP_DMAD_PREP |
1268                        MTK_WED_WDMA_GLO_CFG_IDLE_DMAD_SUPPLY;
1269         }
1270         wed_m32(dev, MTK_WED_WDMA_GLO_CFG, mask, set);
1271
1272         if (mtk_wed_is_v1(dev->hw)) {
1273                 u32 offset = dev->hw->index ? 0x04000400 : 0;
1274
1275                 wdma_set(dev, MTK_WDMA_GLO_CFG,
1276                          MTK_WDMA_GLO_CFG_RX_INFO1_PRERES |
1277                          MTK_WDMA_GLO_CFG_RX_INFO2_PRERES |
1278                          MTK_WDMA_GLO_CFG_RX_INFO3_PRERES);
1279
1280                 wed_w32(dev, MTK_WED_WDMA_OFFSET0, 0x2a042a20 + offset);
1281                 wed_w32(dev, MTK_WED_WDMA_OFFSET1, 0x29002800 + offset);
1282                 wed_w32(dev, MTK_WED_PCIE_CFG_BASE,
1283                         MTK_PCIE_BASE(dev->hw->index));
1284         } else {
1285                 wed_w32(dev, MTK_WED_WDMA_CFG_BASE, dev->hw->wdma_phy);
1286                 wed_set(dev, MTK_WED_CTRL, MTK_WED_CTRL_ETH_DMAD_FMT);
1287                 wed_w32(dev, MTK_WED_WDMA_OFFSET0,
1288                         FIELD_PREP(MTK_WED_WDMA_OFST0_GLO_INTS,
1289                                    MTK_WDMA_INT_STATUS) |
1290                         FIELD_PREP(MTK_WED_WDMA_OFST0_GLO_CFG,
1291                                    MTK_WDMA_GLO_CFG));
1292
1293                 wed_w32(dev, MTK_WED_WDMA_OFFSET1,
1294                         FIELD_PREP(MTK_WED_WDMA_OFST1_TX_CTRL,
1295                                    MTK_WDMA_RING_TX(0)) |
1296                         FIELD_PREP(MTK_WED_WDMA_OFST1_RX_CTRL,
1297                                    MTK_WDMA_RING_RX(0)));
1298         }
1299 }
1300
1301 static int
1302 mtk_wed_rro_ring_alloc(struct mtk_wed_device *dev, struct mtk_wed_ring *ring,
1303                        int size)
1304 {
1305         ring->desc = dma_alloc_coherent(dev->hw->dev,
1306                                         size * sizeof(*ring->desc),
1307                                         &ring->desc_phys, GFP_KERNEL);
1308         if (!ring->desc)
1309                 return -ENOMEM;
1310
1311         ring->desc_size = sizeof(*ring->desc);
1312         ring->size = size;
1313
1314         return 0;
1315 }
1316
1317 #define MTK_WED_MIOD_COUNT      (MTK_WED_MIOD_ENTRY_CNT * MTK_WED_MIOD_CNT)
1318 static int
1319 mtk_wed_rro_alloc(struct mtk_wed_device *dev)
1320 {
1321         struct reserved_mem *rmem;
1322         struct device_node *np;
1323         int index;
1324
1325         index = of_property_match_string(dev->hw->node, "memory-region-names",
1326                                          "wo-dlm");
1327         if (index < 0)
1328                 return index;
1329
1330         np = of_parse_phandle(dev->hw->node, "memory-region", index);
1331         if (!np)
1332                 return -ENODEV;
1333
1334         rmem = of_reserved_mem_lookup(np);
1335         of_node_put(np);
1336
1337         if (!rmem)
1338                 return -ENODEV;
1339
1340         dev->rro.miod_phys = rmem->base;
1341         dev->rro.fdbk_phys = MTK_WED_MIOD_COUNT + dev->rro.miod_phys;
1342
1343         return mtk_wed_rro_ring_alloc(dev, &dev->rro.ring,
1344                                       MTK_WED_RRO_QUE_CNT);
1345 }
1346
1347 static int
1348 mtk_wed_rro_cfg(struct mtk_wed_device *dev)
1349 {
1350         struct mtk_wed_wo *wo = dev->hw->wed_wo;
1351         struct {
1352                 struct {
1353                         __le32 base;
1354                         __le32 cnt;
1355                         __le32 unit;
1356                 } ring[2];
1357                 __le32 wed;
1358                 u8 version;
1359         } req = {
1360                 .ring[0] = {
1361                         .base = cpu_to_le32(MTK_WED_WOCPU_VIEW_MIOD_BASE),
1362                         .cnt = cpu_to_le32(MTK_WED_MIOD_CNT),
1363                         .unit = cpu_to_le32(MTK_WED_MIOD_ENTRY_CNT),
1364                 },
1365                 .ring[1] = {
1366                         .base = cpu_to_le32(MTK_WED_WOCPU_VIEW_MIOD_BASE +
1367                                             MTK_WED_MIOD_COUNT),
1368                         .cnt = cpu_to_le32(MTK_WED_FB_CMD_CNT),
1369                         .unit = cpu_to_le32(4),
1370                 },
1371         };
1372
1373         return mtk_wed_mcu_send_msg(wo, MTK_WED_MODULE_ID_WO,
1374                                     MTK_WED_WO_CMD_WED_CFG,
1375                                     &req, sizeof(req), true);
1376 }
1377
1378 static void
1379 mtk_wed_rro_hw_init(struct mtk_wed_device *dev)
1380 {
1381         wed_w32(dev, MTK_WED_RROQM_MIOD_CFG,
1382                 FIELD_PREP(MTK_WED_RROQM_MIOD_MID_DW, 0x70 >> 2) |
1383                 FIELD_PREP(MTK_WED_RROQM_MIOD_MOD_DW, 0x10 >> 2) |
1384                 FIELD_PREP(MTK_WED_RROQM_MIOD_ENTRY_DW,
1385                            MTK_WED_MIOD_ENTRY_CNT >> 2));
1386
1387         wed_w32(dev, MTK_WED_RROQM_MIOD_CTRL0, dev->rro.miod_phys);
1388         wed_w32(dev, MTK_WED_RROQM_MIOD_CTRL1,
1389                 FIELD_PREP(MTK_WED_RROQM_MIOD_CNT, MTK_WED_MIOD_CNT));
1390         wed_w32(dev, MTK_WED_RROQM_FDBK_CTRL0, dev->rro.fdbk_phys);
1391         wed_w32(dev, MTK_WED_RROQM_FDBK_CTRL1,
1392                 FIELD_PREP(MTK_WED_RROQM_FDBK_CNT, MTK_WED_FB_CMD_CNT));
1393         wed_w32(dev, MTK_WED_RROQM_FDBK_CTRL2, 0);
1394         wed_w32(dev, MTK_WED_RROQ_BASE_L, dev->rro.ring.desc_phys);
1395
1396         wed_set(dev, MTK_WED_RROQM_RST_IDX,
1397                 MTK_WED_RROQM_RST_IDX_MIOD |
1398                 MTK_WED_RROQM_RST_IDX_FDBK);
1399
1400         wed_w32(dev, MTK_WED_RROQM_RST_IDX, 0);
1401         wed_w32(dev, MTK_WED_RROQM_MIOD_CTRL2, MTK_WED_MIOD_CNT - 1);
1402         wed_set(dev, MTK_WED_CTRL, MTK_WED_CTRL_RX_RRO_QM_EN);
1403 }
1404
1405 static void
1406 mtk_wed_route_qm_hw_init(struct mtk_wed_device *dev)
1407 {
1408         wed_w32(dev, MTK_WED_RESET, MTK_WED_RESET_RX_ROUTE_QM);
1409
1410         for (;;) {
1411                 usleep_range(100, 200);
1412                 if (!(wed_r32(dev, MTK_WED_RESET) & MTK_WED_RESET_RX_ROUTE_QM))
1413                         break;
1414         }
1415
1416         /* configure RX_ROUTE_QM */
1417         if (mtk_wed_is_v2(dev->hw)) {
1418                 wed_clr(dev, MTK_WED_RTQM_GLO_CFG, MTK_WED_RTQM_Q_RST);
1419                 wed_clr(dev, MTK_WED_RTQM_GLO_CFG, MTK_WED_RTQM_TXDMAD_FPORT);
1420                 wed_set(dev, MTK_WED_RTQM_GLO_CFG,
1421                         FIELD_PREP(MTK_WED_RTQM_TXDMAD_FPORT,
1422                                    0x3 + dev->hw->index));
1423                 wed_clr(dev, MTK_WED_RTQM_GLO_CFG, MTK_WED_RTQM_Q_RST);
1424         } else {
1425                 wed_set(dev, MTK_WED_RTQM_ENQ_CFG0,
1426                         FIELD_PREP(MTK_WED_RTQM_ENQ_CFG_TXDMAD_FPORT,
1427                                    0x3 + dev->hw->index));
1428         }
1429         /* enable RX_ROUTE_QM */
1430         wed_set(dev, MTK_WED_CTRL, MTK_WED_CTRL_RX_ROUTE_QM_EN);
1431 }
1432
1433 static void
1434 mtk_wed_hw_init(struct mtk_wed_device *dev)
1435 {
1436         if (dev->init_done)
1437                 return;
1438
1439         dev->init_done = true;
1440         mtk_wed_set_ext_int(dev, false);
1441
1442         wed_w32(dev, MTK_WED_TX_BM_BASE, dev->tx_buf_ring.desc_phys);
1443         wed_w32(dev, MTK_WED_TX_BM_BUF_LEN, MTK_WED_PKT_SIZE);
1444
1445         if (mtk_wed_is_v1(dev->hw)) {
1446                 wed_w32(dev, MTK_WED_TX_BM_CTRL,
1447                         MTK_WED_TX_BM_CTRL_PAUSE |
1448                         FIELD_PREP(MTK_WED_TX_BM_CTRL_VLD_GRP_NUM,
1449                                    dev->tx_buf_ring.size / 128) |
1450                         FIELD_PREP(MTK_WED_TX_BM_CTRL_RSV_GRP_NUM,
1451                                    MTK_WED_TX_RING_SIZE / 256));
1452                 wed_w32(dev, MTK_WED_TX_BM_DYN_THR,
1453                         FIELD_PREP(MTK_WED_TX_BM_DYN_THR_LO, 1) |
1454                         MTK_WED_TX_BM_DYN_THR_HI);
1455         } else if (mtk_wed_is_v2(dev->hw)) {
1456                 wed_w32(dev, MTK_WED_TX_BM_CTRL,
1457                         MTK_WED_TX_BM_CTRL_PAUSE |
1458                         FIELD_PREP(MTK_WED_TX_BM_CTRL_VLD_GRP_NUM,
1459                                    dev->tx_buf_ring.size / 128) |
1460                         FIELD_PREP(MTK_WED_TX_BM_CTRL_RSV_GRP_NUM,
1461                                    MTK_WED_TX_RING_SIZE / 256));
1462                 wed_w32(dev, MTK_WED_TX_TKID_DYN_THR,
1463                         FIELD_PREP(MTK_WED_TX_TKID_DYN_THR_LO, 0) |
1464                         MTK_WED_TX_TKID_DYN_THR_HI);
1465                 wed_w32(dev, MTK_WED_TX_BM_DYN_THR,
1466                         FIELD_PREP(MTK_WED_TX_BM_DYN_THR_LO_V2, 0) |
1467                         MTK_WED_TX_BM_DYN_THR_HI_V2);
1468                 wed_w32(dev, MTK_WED_TX_TKID_CTRL,
1469                         MTK_WED_TX_TKID_CTRL_PAUSE |
1470                         FIELD_PREP(MTK_WED_TX_TKID_CTRL_VLD_GRP_NUM,
1471                                    dev->tx_buf_ring.size / 128) |
1472                         FIELD_PREP(MTK_WED_TX_TKID_CTRL_RSV_GRP_NUM,
1473                                    dev->tx_buf_ring.size / 128));
1474         }
1475
1476         wed_w32(dev, dev->hw->soc->regmap.tx_bm_tkid,
1477                 FIELD_PREP(MTK_WED_TX_BM_TKID_START, dev->wlan.token_start) |
1478                 FIELD_PREP(MTK_WED_TX_BM_TKID_END,
1479                            dev->wlan.token_start + dev->wlan.nbuf - 1));
1480
1481         mtk_wed_reset(dev, MTK_WED_RESET_TX_BM);
1482
1483         if (mtk_wed_is_v3_or_greater(dev->hw)) {
1484                 /* switch to new bm architecture */
1485                 wed_clr(dev, MTK_WED_TX_BM_CTRL,
1486                         MTK_WED_TX_BM_CTRL_LEGACY_EN);
1487
1488                 wed_w32(dev, MTK_WED_TX_TKID_CTRL,
1489                         MTK_WED_TX_TKID_CTRL_PAUSE |
1490                         FIELD_PREP(MTK_WED_TX_TKID_CTRL_VLD_GRP_NUM_V3,
1491                                    dev->wlan.nbuf / 128) |
1492                         FIELD_PREP(MTK_WED_TX_TKID_CTRL_RSV_GRP_NUM_V3,
1493                                    dev->wlan.nbuf / 128));
1494                 /* return SKBID + SDP back to bm */
1495                 wed_set(dev, MTK_WED_TX_TKID_CTRL,
1496                         MTK_WED_TX_TKID_CTRL_FREE_FORMAT);
1497
1498                 wed_w32(dev, MTK_WED_TX_BM_INIT_PTR,
1499                         MTK_WED_TX_BM_PKT_CNT |
1500                         MTK_WED_TX_BM_INIT_SW_TAIL_IDX);
1501         }
1502
1503         if (mtk_wed_is_v1(dev->hw)) {
1504                 wed_set(dev, MTK_WED_CTRL,
1505                         MTK_WED_CTRL_WED_TX_BM_EN |
1506                         MTK_WED_CTRL_WED_TX_FREE_AGENT_EN);
1507         } else if (mtk_wed_get_rx_capa(dev)) {
1508                 /* rx hw init */
1509                 wed_w32(dev, MTK_WED_WPDMA_RX_D_RST_IDX,
1510                         MTK_WED_WPDMA_RX_D_RST_CRX_IDX |
1511                         MTK_WED_WPDMA_RX_D_RST_DRV_IDX);
1512                 wed_w32(dev, MTK_WED_WPDMA_RX_D_RST_IDX, 0);
1513
1514                 /* reset prefetch index of ring */
1515                 wed_set(dev, MTK_WED_WPDMA_RX_D_PREF_RX0_SIDX,
1516                         MTK_WED_WPDMA_RX_D_PREF_SIDX_IDX_CLR);
1517                 wed_clr(dev, MTK_WED_WPDMA_RX_D_PREF_RX0_SIDX,
1518                         MTK_WED_WPDMA_RX_D_PREF_SIDX_IDX_CLR);
1519
1520                 wed_set(dev, MTK_WED_WPDMA_RX_D_PREF_RX1_SIDX,
1521                         MTK_WED_WPDMA_RX_D_PREF_SIDX_IDX_CLR);
1522                 wed_clr(dev, MTK_WED_WPDMA_RX_D_PREF_RX1_SIDX,
1523                         MTK_WED_WPDMA_RX_D_PREF_SIDX_IDX_CLR);
1524
1525                 /* reset prefetch FIFO of ring */
1526                 wed_set(dev, MTK_WED_WPDMA_RX_D_PREF_FIFO_CFG,
1527                         MTK_WED_WPDMA_RX_D_PREF_FIFO_CFG_R0_CLR |
1528                         MTK_WED_WPDMA_RX_D_PREF_FIFO_CFG_R1_CLR);
1529                 wed_w32(dev, MTK_WED_WPDMA_RX_D_PREF_FIFO_CFG, 0);
1530
1531                 mtk_wed_rx_buffer_hw_init(dev);
1532                 mtk_wed_rro_hw_init(dev);
1533                 mtk_wed_route_qm_hw_init(dev);
1534         }
1535
1536         wed_clr(dev, MTK_WED_TX_BM_CTRL, MTK_WED_TX_BM_CTRL_PAUSE);
1537         if (!mtk_wed_is_v1(dev->hw))
1538                 wed_clr(dev, MTK_WED_TX_TKID_CTRL, MTK_WED_TX_TKID_CTRL_PAUSE);
1539 }
1540
1541 static void
1542 mtk_wed_ring_reset(struct mtk_wed_ring *ring, int size, bool tx)
1543 {
1544         void *head = (void *)ring->desc;
1545         int i;
1546
1547         for (i = 0; i < size; i++) {
1548                 struct mtk_wdma_desc *desc;
1549
1550                 desc = (struct mtk_wdma_desc *)(head + i * ring->desc_size);
1551                 desc->buf0 = 0;
1552                 if (tx)
1553                         desc->ctrl = cpu_to_le32(MTK_WDMA_DESC_CTRL_DMA_DONE);
1554                 else
1555                         desc->ctrl = cpu_to_le32(MTK_WFDMA_DESC_CTRL_TO_HOST);
1556                 desc->buf1 = 0;
1557                 desc->info = 0;
1558         }
1559 }
1560
1561 static int
1562 mtk_wed_rx_reset(struct mtk_wed_device *dev)
1563 {
1564         struct mtk_wed_wo *wo = dev->hw->wed_wo;
1565         u8 val = MTK_WED_WO_STATE_SER_RESET;
1566         int i, ret;
1567
1568         ret = mtk_wed_mcu_send_msg(wo, MTK_WED_MODULE_ID_WO,
1569                                    MTK_WED_WO_CMD_CHANGE_STATE, &val,
1570                                    sizeof(val), true);
1571         if (ret)
1572                 return ret;
1573
1574         if (dev->wlan.hw_rro) {
1575                 wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_WED_RX_IND_CMD_EN);
1576                 mtk_wed_poll_busy(dev, MTK_WED_RRO_RX_HW_STS,
1577                                   MTK_WED_RX_IND_CMD_BUSY);
1578                 mtk_wed_reset(dev, MTK_WED_RESET_RRO_RX_TO_PG);
1579         }
1580
1581         wed_clr(dev, MTK_WED_WPDMA_RX_D_GLO_CFG, MTK_WED_WPDMA_RX_D_RX_DRV_EN);
1582         ret = mtk_wed_poll_busy(dev, MTK_WED_WPDMA_RX_D_GLO_CFG,
1583                                 MTK_WED_WPDMA_RX_D_RX_DRV_BUSY);
1584         if (!ret && mtk_wed_is_v3_or_greater(dev->hw))
1585                 ret = mtk_wed_poll_busy(dev, MTK_WED_WPDMA_RX_D_PREF_CFG,
1586                                         MTK_WED_WPDMA_RX_D_PREF_BUSY);
1587         if (ret) {
1588                 mtk_wed_reset(dev, MTK_WED_RESET_WPDMA_INT_AGENT);
1589                 mtk_wed_reset(dev, MTK_WED_RESET_WPDMA_RX_D_DRV);
1590         } else {
1591                 if (mtk_wed_is_v3_or_greater(dev->hw)) {
1592                         /* 1.a. disable prefetch HW */
1593                         wed_clr(dev, MTK_WED_WPDMA_RX_D_PREF_CFG,
1594                                 MTK_WED_WPDMA_RX_D_PREF_EN);
1595                         mtk_wed_poll_busy(dev, MTK_WED_WPDMA_RX_D_PREF_CFG,
1596                                           MTK_WED_WPDMA_RX_D_PREF_BUSY);
1597                         wed_w32(dev, MTK_WED_WPDMA_RX_D_RST_IDX,
1598                                 MTK_WED_WPDMA_RX_D_RST_DRV_IDX_ALL);
1599                 }
1600
1601                 wed_w32(dev, MTK_WED_WPDMA_RX_D_RST_IDX,
1602                         MTK_WED_WPDMA_RX_D_RST_CRX_IDX |
1603                         MTK_WED_WPDMA_RX_D_RST_DRV_IDX);
1604
1605                 wed_set(dev, MTK_WED_WPDMA_RX_D_GLO_CFG,
1606                         MTK_WED_WPDMA_RX_D_RST_INIT_COMPLETE |
1607                         MTK_WED_WPDMA_RX_D_FSM_RETURN_IDLE);
1608                 wed_clr(dev, MTK_WED_WPDMA_RX_D_GLO_CFG,
1609                         MTK_WED_WPDMA_RX_D_RST_INIT_COMPLETE |
1610                         MTK_WED_WPDMA_RX_D_FSM_RETURN_IDLE);
1611
1612                 wed_w32(dev, MTK_WED_WPDMA_RX_D_RST_IDX, 0);
1613         }
1614
1615         /* reset rro qm */
1616         wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_RX_RRO_QM_EN);
1617         ret = mtk_wed_poll_busy(dev, MTK_WED_CTRL,
1618                                 MTK_WED_CTRL_RX_RRO_QM_BUSY);
1619         if (ret) {
1620                 mtk_wed_reset(dev, MTK_WED_RESET_RX_RRO_QM);
1621         } else {
1622                 wed_set(dev, MTK_WED_RROQM_RST_IDX,
1623                         MTK_WED_RROQM_RST_IDX_MIOD |
1624                         MTK_WED_RROQM_RST_IDX_FDBK);
1625                 wed_w32(dev, MTK_WED_RROQM_RST_IDX, 0);
1626         }
1627
1628         if (dev->wlan.hw_rro) {
1629                 /* disable rro msdu page drv */
1630                 wed_clr(dev, MTK_WED_RRO_MSDU_PG_RING2_CFG,
1631                         MTK_WED_RRO_MSDU_PG_DRV_EN);
1632
1633                 /* disable rro data drv */
1634                 wed_clr(dev, MTK_WED_RRO_RX_D_CFG(2), MTK_WED_RRO_RX_D_DRV_EN);
1635
1636                 /* rro msdu page drv reset */
1637                 wed_w32(dev, MTK_WED_RRO_MSDU_PG_RING2_CFG,
1638                         MTK_WED_RRO_MSDU_PG_DRV_CLR);
1639                 mtk_wed_poll_busy(dev, MTK_WED_RRO_MSDU_PG_RING2_CFG,
1640                                   MTK_WED_RRO_MSDU_PG_DRV_CLR);
1641
1642                 /* rro data drv reset */
1643                 wed_w32(dev, MTK_WED_RRO_RX_D_CFG(2),
1644                         MTK_WED_RRO_RX_D_DRV_CLR);
1645                 mtk_wed_poll_busy(dev, MTK_WED_RRO_RX_D_CFG(2),
1646                                   MTK_WED_RRO_RX_D_DRV_CLR);
1647         }
1648
1649         /* reset route qm */
1650         wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_RX_ROUTE_QM_EN);
1651         ret = mtk_wed_poll_busy(dev, MTK_WED_CTRL,
1652                                 MTK_WED_CTRL_RX_ROUTE_QM_BUSY);
1653         if (ret) {
1654                 mtk_wed_reset(dev, MTK_WED_RESET_RX_ROUTE_QM);
1655         } else if (mtk_wed_is_v3_or_greater(dev->hw)) {
1656                 wed_set(dev, MTK_WED_RTQM_RST, BIT(0));
1657                 wed_clr(dev, MTK_WED_RTQM_RST, BIT(0));
1658                 mtk_wed_reset(dev, MTK_WED_RESET_RX_ROUTE_QM);
1659         } else {
1660                 wed_set(dev, MTK_WED_RTQM_GLO_CFG, MTK_WED_RTQM_Q_RST);
1661         }
1662
1663         /* reset tx wdma */
1664         mtk_wdma_tx_reset(dev);
1665
1666         /* reset tx wdma drv */
1667         wed_clr(dev, MTK_WED_WDMA_GLO_CFG, MTK_WED_WDMA_GLO_CFG_TX_DRV_EN);
1668         if (mtk_wed_is_v3_or_greater(dev->hw))
1669                 mtk_wed_poll_busy(dev, MTK_WED_WPDMA_STATUS,
1670                                   MTK_WED_WPDMA_STATUS_TX_DRV);
1671         else
1672                 mtk_wed_poll_busy(dev, MTK_WED_CTRL,
1673                                   MTK_WED_CTRL_WDMA_INT_AGENT_BUSY);
1674         mtk_wed_reset(dev, MTK_WED_RESET_WDMA_TX_DRV);
1675
1676         /* reset wed rx dma */
1677         ret = mtk_wed_poll_busy(dev, MTK_WED_GLO_CFG,
1678                                 MTK_WED_GLO_CFG_RX_DMA_BUSY);
1679         wed_clr(dev, MTK_WED_GLO_CFG, MTK_WED_GLO_CFG_RX_DMA_EN);
1680         if (ret) {
1681                 mtk_wed_reset(dev, MTK_WED_RESET_WED_RX_DMA);
1682         } else {
1683                 wed_set(dev, MTK_WED_RESET_IDX,
1684                         dev->hw->soc->regmap.reset_idx_rx_mask);
1685                 wed_w32(dev, MTK_WED_RESET_IDX, 0);
1686         }
1687
1688         /* reset rx bm */
1689         wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_WED_RX_BM_EN);
1690         mtk_wed_poll_busy(dev, MTK_WED_CTRL,
1691                           MTK_WED_CTRL_WED_RX_BM_BUSY);
1692         mtk_wed_reset(dev, MTK_WED_RESET_RX_BM);
1693
1694         if (dev->wlan.hw_rro) {
1695                 wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_WED_RX_PG_BM_EN);
1696                 mtk_wed_poll_busy(dev, MTK_WED_CTRL,
1697                                   MTK_WED_CTRL_WED_RX_PG_BM_BUSY);
1698                 wed_set(dev, MTK_WED_RESET, MTK_WED_RESET_RX_PG_BM);
1699                 wed_clr(dev, MTK_WED_RESET, MTK_WED_RESET_RX_PG_BM);
1700         }
1701
1702         /* wo change to enable state */
1703         val = MTK_WED_WO_STATE_ENABLE;
1704         ret = mtk_wed_mcu_send_msg(wo, MTK_WED_MODULE_ID_WO,
1705                                    MTK_WED_WO_CMD_CHANGE_STATE, &val,
1706                                    sizeof(val), true);
1707         if (ret)
1708                 return ret;
1709
1710         /* wed_rx_ring_reset */
1711         for (i = 0; i < ARRAY_SIZE(dev->rx_ring); i++) {
1712                 if (!dev->rx_ring[i].desc)
1713                         continue;
1714
1715                 mtk_wed_ring_reset(&dev->rx_ring[i], MTK_WED_RX_RING_SIZE,
1716                                    false);
1717         }
1718         mtk_wed_free_rx_buffer(dev);
1719         mtk_wed_hwrro_free_buffer(dev);
1720
1721         return 0;
1722 }
1723
1724 static void
1725 mtk_wed_reset_dma(struct mtk_wed_device *dev)
1726 {
1727         bool busy = false;
1728         u32 val;
1729         int i;
1730
1731         for (i = 0; i < ARRAY_SIZE(dev->tx_ring); i++) {
1732                 if (!dev->tx_ring[i].desc)
1733                         continue;
1734
1735                 mtk_wed_ring_reset(&dev->tx_ring[i], MTK_WED_TX_RING_SIZE,
1736                                    true);
1737         }
1738
1739         /* 1. reset WED tx DMA */
1740         wed_clr(dev, MTK_WED_GLO_CFG, MTK_WED_GLO_CFG_TX_DMA_EN);
1741         busy = mtk_wed_poll_busy(dev, MTK_WED_GLO_CFG,
1742                                  MTK_WED_GLO_CFG_TX_DMA_BUSY);
1743         if (busy) {
1744                 mtk_wed_reset(dev, MTK_WED_RESET_WED_TX_DMA);
1745         } else {
1746                 wed_w32(dev, MTK_WED_RESET_IDX,
1747                         dev->hw->soc->regmap.reset_idx_tx_mask);
1748                 wed_w32(dev, MTK_WED_RESET_IDX, 0);
1749         }
1750
1751         /* 2. reset WDMA rx DMA */
1752         busy = !!mtk_wdma_rx_reset(dev);
1753         if (mtk_wed_is_v3_or_greater(dev->hw)) {
1754                 val = MTK_WED_WDMA_GLO_CFG_RX_DIS_FSM_AUTO_IDLE |
1755                       wed_r32(dev, MTK_WED_WDMA_GLO_CFG);
1756                 val &= ~MTK_WED_WDMA_GLO_CFG_RX_DRV_EN;
1757                 wed_w32(dev, MTK_WED_WDMA_GLO_CFG, val);
1758         } else {
1759                 wed_clr(dev, MTK_WED_WDMA_GLO_CFG,
1760                         MTK_WED_WDMA_GLO_CFG_RX_DRV_EN);
1761         }
1762
1763         if (!busy)
1764                 busy = mtk_wed_poll_busy(dev, MTK_WED_WDMA_GLO_CFG,
1765                                          MTK_WED_WDMA_GLO_CFG_RX_DRV_BUSY);
1766         if (!busy && mtk_wed_is_v3_or_greater(dev->hw))
1767                 busy = mtk_wed_poll_busy(dev, MTK_WED_WDMA_RX_PREF_CFG,
1768                                          MTK_WED_WDMA_RX_PREF_BUSY);
1769
1770         if (busy) {
1771                 mtk_wed_reset(dev, MTK_WED_RESET_WDMA_INT_AGENT);
1772                 mtk_wed_reset(dev, MTK_WED_RESET_WDMA_RX_DRV);
1773         } else {
1774                 if (mtk_wed_is_v3_or_greater(dev->hw)) {
1775                         /* 1.a. disable prefetch HW */
1776                         wed_clr(dev, MTK_WED_WDMA_RX_PREF_CFG,
1777                                 MTK_WED_WDMA_RX_PREF_EN);
1778                         mtk_wed_poll_busy(dev, MTK_WED_WDMA_RX_PREF_CFG,
1779                                           MTK_WED_WDMA_RX_PREF_BUSY);
1780                         wed_clr(dev, MTK_WED_WDMA_RX_PREF_CFG,
1781                                 MTK_WED_WDMA_RX_PREF_DDONE2_EN);
1782
1783                         /* 2. Reset dma index */
1784                         wed_w32(dev, MTK_WED_WDMA_RESET_IDX,
1785                                 MTK_WED_WDMA_RESET_IDX_RX_ALL);
1786                 }
1787
1788                 wed_w32(dev, MTK_WED_WDMA_RESET_IDX,
1789                         MTK_WED_WDMA_RESET_IDX_RX | MTK_WED_WDMA_RESET_IDX_DRV);
1790                 wed_w32(dev, MTK_WED_WDMA_RESET_IDX, 0);
1791
1792                 wed_set(dev, MTK_WED_WDMA_GLO_CFG,
1793                         MTK_WED_WDMA_GLO_CFG_RST_INIT_COMPLETE);
1794
1795                 wed_clr(dev, MTK_WED_WDMA_GLO_CFG,
1796                         MTK_WED_WDMA_GLO_CFG_RST_INIT_COMPLETE);
1797         }
1798
1799         /* 3. reset WED WPDMA tx */
1800         wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_WED_TX_FREE_AGENT_EN);
1801
1802         for (i = 0; i < 100; i++) {
1803                 if (mtk_wed_is_v1(dev->hw))
1804                         val = FIELD_GET(MTK_WED_TX_BM_INTF_TKFIFO_FDEP,
1805                                         wed_r32(dev, MTK_WED_TX_BM_INTF));
1806                 else
1807                         val = FIELD_GET(MTK_WED_TX_TKID_INTF_TKFIFO_FDEP,
1808                                         wed_r32(dev, MTK_WED_TX_TKID_INTF));
1809                 if (val == 0x40)
1810                         break;
1811         }
1812
1813         mtk_wed_reset(dev, MTK_WED_RESET_TX_FREE_AGENT);
1814         wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_WED_TX_BM_EN);
1815         mtk_wed_reset(dev, MTK_WED_RESET_TX_BM);
1816
1817         /* 4. reset WED WPDMA tx */
1818         busy = mtk_wed_poll_busy(dev, MTK_WED_WPDMA_GLO_CFG,
1819                                  MTK_WED_WPDMA_GLO_CFG_TX_DRV_BUSY);
1820         wed_clr(dev, MTK_WED_WPDMA_GLO_CFG,
1821                 MTK_WED_WPDMA_GLO_CFG_TX_DRV_EN |
1822                 MTK_WED_WPDMA_GLO_CFG_RX_DRV_EN);
1823         if (!busy)
1824                 busy = mtk_wed_poll_busy(dev, MTK_WED_WPDMA_GLO_CFG,
1825                                          MTK_WED_WPDMA_GLO_CFG_RX_DRV_BUSY);
1826
1827         if (busy) {
1828                 mtk_wed_reset(dev, MTK_WED_RESET_WPDMA_INT_AGENT);
1829                 mtk_wed_reset(dev, MTK_WED_RESET_WPDMA_TX_DRV);
1830                 mtk_wed_reset(dev, MTK_WED_RESET_WPDMA_RX_DRV);
1831                 if (mtk_wed_is_v3_or_greater(dev->hw))
1832                         wed_w32(dev, MTK_WED_RX1_CTRL2, 0);
1833         } else {
1834                 wed_w32(dev, MTK_WED_WPDMA_RESET_IDX,
1835                         MTK_WED_WPDMA_RESET_IDX_TX |
1836                         MTK_WED_WPDMA_RESET_IDX_RX);
1837                 wed_w32(dev, MTK_WED_WPDMA_RESET_IDX, 0);
1838         }
1839
1840         dev->init_done = false;
1841         if (mtk_wed_is_v1(dev->hw))
1842                 return;
1843
1844         if (!busy) {
1845                 wed_w32(dev, MTK_WED_RESET_IDX, MTK_WED_RESET_WPDMA_IDX_RX);
1846                 wed_w32(dev, MTK_WED_RESET_IDX, 0);
1847         }
1848
1849         if (mtk_wed_is_v3_or_greater(dev->hw)) {
1850                 /* reset amsdu engine */
1851                 wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_TX_AMSDU_EN);
1852                 mtk_wed_reset(dev, MTK_WED_RESET_TX_AMSDU);
1853         }
1854
1855         if (mtk_wed_get_rx_capa(dev))
1856                 mtk_wed_rx_reset(dev);
1857 }
1858
1859 static int
1860 mtk_wed_ring_alloc(struct mtk_wed_device *dev, struct mtk_wed_ring *ring,
1861                    int size, u32 desc_size, bool tx)
1862 {
1863         ring->desc = dma_alloc_coherent(dev->hw->dev, size * desc_size,
1864                                         &ring->desc_phys, GFP_KERNEL);
1865         if (!ring->desc)
1866                 return -ENOMEM;
1867
1868         ring->desc_size = desc_size;
1869         ring->size = size;
1870         mtk_wed_ring_reset(ring, size, tx);
1871
1872         return 0;
1873 }
1874
1875 static int
1876 mtk_wed_wdma_rx_ring_setup(struct mtk_wed_device *dev, int idx, int size,
1877                            bool reset)
1878 {
1879         struct mtk_wed_ring *wdma;
1880
1881         if (idx >= ARRAY_SIZE(dev->rx_wdma))
1882                 return -EINVAL;
1883
1884         wdma = &dev->rx_wdma[idx];
1885         if (!reset && mtk_wed_ring_alloc(dev, wdma, MTK_WED_WDMA_RING_SIZE,
1886                                          dev->hw->soc->wdma_desc_size, true))
1887                 return -ENOMEM;
1888
1889         wdma_w32(dev, MTK_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_BASE,
1890                  wdma->desc_phys);
1891         wdma_w32(dev, MTK_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_COUNT,
1892                  size);
1893         wdma_w32(dev, MTK_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_CPU_IDX, 0);
1894
1895         wed_w32(dev, MTK_WED_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_BASE,
1896                 wdma->desc_phys);
1897         wed_w32(dev, MTK_WED_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_COUNT,
1898                 size);
1899
1900         return 0;
1901 }
1902
1903 static int
1904 mtk_wed_wdma_tx_ring_setup(struct mtk_wed_device *dev, int idx, int size,
1905                            bool reset)
1906 {
1907         struct mtk_wed_ring *wdma;
1908
1909         if (idx >= ARRAY_SIZE(dev->tx_wdma))
1910                 return -EINVAL;
1911
1912         wdma = &dev->tx_wdma[idx];
1913         if (!reset && mtk_wed_ring_alloc(dev, wdma, MTK_WED_WDMA_RING_SIZE,
1914                                          dev->hw->soc->wdma_desc_size, true))
1915                 return -ENOMEM;
1916
1917         if (mtk_wed_is_v3_or_greater(dev->hw)) {
1918                 struct mtk_wdma_desc *desc = wdma->desc;
1919                 int i;
1920
1921                 for (i = 0; i < MTK_WED_WDMA_RING_SIZE; i++) {
1922                         desc->buf0 = 0;
1923                         desc->ctrl = cpu_to_le32(MTK_WDMA_DESC_CTRL_DMA_DONE);
1924                         desc->buf1 = 0;
1925                         desc->info = cpu_to_le32(MTK_WDMA_TXD0_DESC_INFO_DMA_DONE);
1926                         desc++;
1927                         desc->buf0 = 0;
1928                         desc->ctrl = cpu_to_le32(MTK_WDMA_DESC_CTRL_DMA_DONE);
1929                         desc->buf1 = 0;
1930                         desc->info = cpu_to_le32(MTK_WDMA_TXD1_DESC_INFO_DMA_DONE);
1931                         desc++;
1932                 }
1933         }
1934
1935         wdma_w32(dev, MTK_WDMA_RING_TX(idx) + MTK_WED_RING_OFS_BASE,
1936                  wdma->desc_phys);
1937         wdma_w32(dev, MTK_WDMA_RING_TX(idx) + MTK_WED_RING_OFS_COUNT,
1938                  size);
1939         wdma_w32(dev, MTK_WDMA_RING_TX(idx) + MTK_WED_RING_OFS_CPU_IDX, 0);
1940         wdma_w32(dev, MTK_WDMA_RING_TX(idx) + MTK_WED_RING_OFS_DMA_IDX, 0);
1941
1942         if (reset)
1943                 mtk_wed_ring_reset(wdma, MTK_WED_WDMA_RING_SIZE, true);
1944
1945         if (!idx)  {
1946                 wed_w32(dev, MTK_WED_WDMA_RING_TX + MTK_WED_RING_OFS_BASE,
1947                         wdma->desc_phys);
1948                 wed_w32(dev, MTK_WED_WDMA_RING_TX + MTK_WED_RING_OFS_COUNT,
1949                         size);
1950                 wed_w32(dev, MTK_WED_WDMA_RING_TX + MTK_WED_RING_OFS_CPU_IDX,
1951                         0);
1952                 wed_w32(dev, MTK_WED_WDMA_RING_TX + MTK_WED_RING_OFS_DMA_IDX,
1953                         0);
1954         }
1955
1956         return 0;
1957 }
1958
1959 static void
1960 mtk_wed_ppe_check(struct mtk_wed_device *dev, struct sk_buff *skb,
1961                   u32 reason, u32 hash)
1962 {
1963         struct mtk_eth *eth = dev->hw->eth;
1964         struct ethhdr *eh;
1965
1966         if (!skb)
1967                 return;
1968
1969         if (reason != MTK_PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED)
1970                 return;
1971
1972         skb_set_mac_header(skb, 0);
1973         eh = eth_hdr(skb);
1974         skb->protocol = eh->h_proto;
1975         mtk_ppe_check_skb(eth->ppe[dev->hw->index], skb, hash);
1976 }
1977
1978 static void
1979 mtk_wed_configure_irq(struct mtk_wed_device *dev, u32 irq_mask)
1980 {
1981         u32 wdma_mask = FIELD_PREP(MTK_WDMA_INT_MASK_RX_DONE, GENMASK(1, 0));
1982
1983         /* wed control cr set */
1984         wed_set(dev, MTK_WED_CTRL,
1985                 MTK_WED_CTRL_WDMA_INT_AGENT_EN |
1986                 MTK_WED_CTRL_WPDMA_INT_AGENT_EN |
1987                 MTK_WED_CTRL_WED_TX_BM_EN |
1988                 MTK_WED_CTRL_WED_TX_FREE_AGENT_EN);
1989
1990         if (mtk_wed_is_v1(dev->hw)) {
1991                 wed_w32(dev, MTK_WED_PCIE_INT_TRIGGER,
1992                         MTK_WED_PCIE_INT_TRIGGER_STATUS);
1993
1994                 wed_w32(dev, MTK_WED_WPDMA_INT_TRIGGER,
1995                         MTK_WED_WPDMA_INT_TRIGGER_RX_DONE |
1996                         MTK_WED_WPDMA_INT_TRIGGER_TX_DONE);
1997
1998                 wed_clr(dev, MTK_WED_WDMA_INT_CTRL, wdma_mask);
1999         } else {
2000                 if (mtk_wed_is_v3_or_greater(dev->hw))
2001                         wed_set(dev, MTK_WED_CTRL, MTK_WED_CTRL_TX_TKID_ALI_EN);
2002
2003                 /* initail tx interrupt trigger */
2004                 wed_w32(dev, MTK_WED_WPDMA_INT_CTRL_TX,
2005                         MTK_WED_WPDMA_INT_CTRL_TX0_DONE_EN |
2006                         MTK_WED_WPDMA_INT_CTRL_TX0_DONE_CLR |
2007                         MTK_WED_WPDMA_INT_CTRL_TX1_DONE_EN |
2008                         MTK_WED_WPDMA_INT_CTRL_TX1_DONE_CLR |
2009                         FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_TX0_DONE_TRIG,
2010                                    dev->wlan.tx_tbit[0]) |
2011                         FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_TX1_DONE_TRIG,
2012                                    dev->wlan.tx_tbit[1]));
2013
2014                 /* initail txfree interrupt trigger */
2015                 wed_w32(dev, MTK_WED_WPDMA_INT_CTRL_TX_FREE,
2016                         MTK_WED_WPDMA_INT_CTRL_TX_FREE_DONE_EN |
2017                         MTK_WED_WPDMA_INT_CTRL_TX_FREE_DONE_CLR |
2018                         FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_TX_FREE_DONE_TRIG,
2019                                    dev->wlan.txfree_tbit));
2020
2021                 if (mtk_wed_get_rx_capa(dev)) {
2022                         wed_w32(dev, MTK_WED_WPDMA_INT_CTRL_RX,
2023                                 MTK_WED_WPDMA_INT_CTRL_RX0_EN |
2024                                 MTK_WED_WPDMA_INT_CTRL_RX0_CLR |
2025                                 MTK_WED_WPDMA_INT_CTRL_RX1_EN |
2026                                 MTK_WED_WPDMA_INT_CTRL_RX1_CLR |
2027                                 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_RX0_DONE_TRIG,
2028                                            dev->wlan.rx_tbit[0]) |
2029                                 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_RX1_DONE_TRIG,
2030                                            dev->wlan.rx_tbit[1]));
2031
2032                         wdma_mask |= FIELD_PREP(MTK_WDMA_INT_MASK_TX_DONE,
2033                                                 GENMASK(1, 0));
2034                 }
2035
2036                 wed_w32(dev, MTK_WED_WDMA_INT_CLR, wdma_mask);
2037                 wed_set(dev, MTK_WED_WDMA_INT_CTRL,
2038                         FIELD_PREP(MTK_WED_WDMA_INT_CTRL_POLL_SRC_SEL,
2039                                    dev->wdma_idx));
2040         }
2041
2042         wed_w32(dev, MTK_WED_WDMA_INT_TRIGGER, wdma_mask);
2043
2044         wdma_w32(dev, MTK_WDMA_INT_MASK, wdma_mask);
2045         wdma_w32(dev, MTK_WDMA_INT_GRP2, wdma_mask);
2046         wed_w32(dev, MTK_WED_WPDMA_INT_MASK, irq_mask);
2047         wed_w32(dev, MTK_WED_INT_MASK, irq_mask);
2048 }
2049
2050 #define MTK_WFMDA_RX_DMA_EN     BIT(2)
2051 static void
2052 mtk_wed_dma_enable(struct mtk_wed_device *dev)
2053 {
2054         int i;
2055
2056         if (!mtk_wed_is_v3_or_greater(dev->hw)) {
2057                 wed_set(dev, MTK_WED_WPDMA_INT_CTRL,
2058                         MTK_WED_WPDMA_INT_CTRL_SUBRT_ADV);
2059                 wed_set(dev, MTK_WED_WPDMA_GLO_CFG,
2060                         MTK_WED_WPDMA_GLO_CFG_TX_DRV_EN |
2061                         MTK_WED_WPDMA_GLO_CFG_RX_DRV_EN);
2062                 wdma_set(dev, MTK_WDMA_GLO_CFG,
2063                          MTK_WDMA_GLO_CFG_TX_DMA_EN |
2064                          MTK_WDMA_GLO_CFG_RX_INFO1_PRERES |
2065                          MTK_WDMA_GLO_CFG_RX_INFO2_PRERES);
2066                 wed_set(dev, MTK_WED_WPDMA_CTRL, MTK_WED_WPDMA_CTRL_SDL1_FIXED);
2067         } else {
2068                 wed_set(dev, MTK_WED_WPDMA_GLO_CFG,
2069                         MTK_WED_WPDMA_GLO_CFG_TX_DRV_EN |
2070                         MTK_WED_WPDMA_GLO_CFG_RX_DRV_EN |
2071                         MTK_WED_WPDMA_GLO_CFG_RX_DDONE2_WR);
2072                 wdma_set(dev, MTK_WDMA_GLO_CFG, MTK_WDMA_GLO_CFG_TX_DMA_EN);
2073         }
2074
2075         wed_set(dev, MTK_WED_GLO_CFG,
2076                 MTK_WED_GLO_CFG_TX_DMA_EN |
2077                 MTK_WED_GLO_CFG_RX_DMA_EN);
2078
2079         wed_set(dev, MTK_WED_WDMA_GLO_CFG,
2080                 MTK_WED_WDMA_GLO_CFG_RX_DRV_EN);
2081
2082         if (mtk_wed_is_v1(dev->hw)) {
2083                 wdma_set(dev, MTK_WDMA_GLO_CFG,
2084                          MTK_WDMA_GLO_CFG_RX_INFO3_PRERES);
2085                 return;
2086         }
2087
2088         wed_set(dev, MTK_WED_WPDMA_GLO_CFG,
2089                 MTK_WED_WPDMA_GLO_CFG_RX_DRV_R0_PKT_PROC |
2090                 MTK_WED_WPDMA_GLO_CFG_RX_DRV_R0_CRX_SYNC);
2091
2092         if (mtk_wed_is_v3_or_greater(dev->hw)) {
2093                 wed_set(dev, MTK_WED_WDMA_RX_PREF_CFG,
2094                         FIELD_PREP(MTK_WED_WDMA_RX_PREF_BURST_SIZE, 0x10) |
2095                         FIELD_PREP(MTK_WED_WDMA_RX_PREF_LOW_THRES, 0x8));
2096                 wed_clr(dev, MTK_WED_WDMA_RX_PREF_CFG,
2097                         MTK_WED_WDMA_RX_PREF_DDONE2_EN);
2098                 wed_set(dev, MTK_WED_WDMA_RX_PREF_CFG, MTK_WED_WDMA_RX_PREF_EN);
2099
2100                 wed_clr(dev, MTK_WED_WPDMA_GLO_CFG,
2101                         MTK_WED_WPDMA_GLO_CFG_TX_DDONE_CHK_LAST);
2102                 wed_set(dev, MTK_WED_WPDMA_GLO_CFG,
2103                         MTK_WED_WPDMA_GLO_CFG_TX_DDONE_CHK |
2104                         MTK_WED_WPDMA_GLO_CFG_RX_DRV_EVENT_PKT_FMT_CHK |
2105                         MTK_WED_WPDMA_GLO_CFG_RX_DRV_UNS_VER_FORCE_4);
2106
2107                 wdma_set(dev, MTK_WDMA_PREF_RX_CFG, MTK_WDMA_PREF_RX_CFG_PREF_EN);
2108                 wdma_set(dev, MTK_WDMA_WRBK_RX_CFG, MTK_WDMA_WRBK_RX_CFG_WRBK_EN);
2109         }
2110
2111         wed_clr(dev, MTK_WED_WPDMA_GLO_CFG,
2112                 MTK_WED_WPDMA_GLO_CFG_TX_TKID_KEEP |
2113                 MTK_WED_WPDMA_GLO_CFG_TX_DMAD_DW3_PREV);
2114
2115         if (!mtk_wed_get_rx_capa(dev))
2116                 return;
2117
2118         wed_set(dev, MTK_WED_WDMA_GLO_CFG,
2119                 MTK_WED_WDMA_GLO_CFG_TX_DRV_EN |
2120                 MTK_WED_WDMA_GLO_CFG_TX_DDONE_CHK);
2121
2122         wed_clr(dev, MTK_WED_WPDMA_RX_D_GLO_CFG, MTK_WED_WPDMA_RX_D_RXD_READ_LEN);
2123         wed_set(dev, MTK_WED_WPDMA_RX_D_GLO_CFG,
2124                 MTK_WED_WPDMA_RX_D_RX_DRV_EN |
2125                 FIELD_PREP(MTK_WED_WPDMA_RX_D_RXD_READ_LEN, 0x18) |
2126                 FIELD_PREP(MTK_WED_WPDMA_RX_D_INIT_PHASE_RXEN_SEL, 0x2));
2127
2128         if (mtk_wed_is_v3_or_greater(dev->hw)) {
2129                 wed_set(dev, MTK_WED_WPDMA_RX_D_PREF_CFG,
2130                         MTK_WED_WPDMA_RX_D_PREF_EN |
2131                         FIELD_PREP(MTK_WED_WPDMA_RX_D_PREF_BURST_SIZE, 0x10) |
2132                         FIELD_PREP(MTK_WED_WPDMA_RX_D_PREF_LOW_THRES, 0x8));
2133
2134                 wed_set(dev, MTK_WED_RRO_RX_D_CFG(2), MTK_WED_RRO_RX_D_DRV_EN);
2135                 wdma_set(dev, MTK_WDMA_PREF_TX_CFG, MTK_WDMA_PREF_TX_CFG_PREF_EN);
2136                 wdma_set(dev, MTK_WDMA_WRBK_TX_CFG, MTK_WDMA_WRBK_TX_CFG_WRBK_EN);
2137         }
2138
2139         for (i = 0; i < MTK_WED_RX_QUEUES; i++) {
2140                 struct mtk_wed_ring *ring = &dev->rx_ring[i];
2141                 u32 val;
2142
2143                 if (!(ring->flags & MTK_WED_RING_CONFIGURED))
2144                         continue; /* queue is not configured by mt76 */
2145
2146                 if (mtk_wed_check_wfdma_rx_fill(dev, ring)) {
2147                         dev_err(dev->hw->dev,
2148                                 "rx_ring(%d) dma enable failed\n", i);
2149                         continue;
2150                 }
2151
2152                 val = wifi_r32(dev,
2153                                dev->wlan.wpdma_rx_glo -
2154                                dev->wlan.phy_base) | MTK_WFMDA_RX_DMA_EN;
2155                 wifi_w32(dev,
2156                          dev->wlan.wpdma_rx_glo - dev->wlan.phy_base,
2157                          val);
2158         }
2159 }
2160
2161 static void
2162 mtk_wed_start_hw_rro(struct mtk_wed_device *dev, u32 irq_mask, bool reset)
2163 {
2164         int i;
2165
2166         wed_w32(dev, MTK_WED_WPDMA_INT_MASK, irq_mask);
2167         wed_w32(dev, MTK_WED_INT_MASK, irq_mask);
2168
2169         if (!mtk_wed_get_rx_capa(dev) || !dev->wlan.hw_rro)
2170                 return;
2171
2172         if (reset) {
2173                 wed_set(dev, MTK_WED_RRO_MSDU_PG_RING2_CFG,
2174                         MTK_WED_RRO_MSDU_PG_DRV_EN);
2175                 return;
2176         }
2177
2178         wed_set(dev, MTK_WED_RRO_RX_D_CFG(2), MTK_WED_RRO_MSDU_PG_DRV_CLR);
2179         wed_w32(dev, MTK_WED_RRO_MSDU_PG_RING2_CFG,
2180                 MTK_WED_RRO_MSDU_PG_DRV_CLR);
2181
2182         wed_w32(dev, MTK_WED_WPDMA_INT_CTRL_RRO_RX,
2183                 MTK_WED_WPDMA_INT_CTRL_RRO_RX0_EN |
2184                 MTK_WED_WPDMA_INT_CTRL_RRO_RX0_CLR |
2185                 MTK_WED_WPDMA_INT_CTRL_RRO_RX1_EN |
2186                 MTK_WED_WPDMA_INT_CTRL_RRO_RX1_CLR |
2187                 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_RRO_RX0_DONE_TRIG,
2188                            dev->wlan.rro_rx_tbit[0]) |
2189                 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_RRO_RX1_DONE_TRIG,
2190                            dev->wlan.rro_rx_tbit[1]));
2191
2192         wed_w32(dev, MTK_WED_WPDMA_INT_CTRL_RRO_MSDU_PG,
2193                 MTK_WED_WPDMA_INT_CTRL_RRO_PG0_EN |
2194                 MTK_WED_WPDMA_INT_CTRL_RRO_PG0_CLR |
2195                 MTK_WED_WPDMA_INT_CTRL_RRO_PG1_EN |
2196                 MTK_WED_WPDMA_INT_CTRL_RRO_PG1_CLR |
2197                 MTK_WED_WPDMA_INT_CTRL_RRO_PG2_EN |
2198                 MTK_WED_WPDMA_INT_CTRL_RRO_PG2_CLR |
2199                 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_RRO_PG0_DONE_TRIG,
2200                            dev->wlan.rx_pg_tbit[0]) |
2201                 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_RRO_PG1_DONE_TRIG,
2202                            dev->wlan.rx_pg_tbit[1]) |
2203                 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_RRO_PG2_DONE_TRIG,
2204                            dev->wlan.rx_pg_tbit[2]));
2205
2206         /* RRO_MSDU_PG_RING2_CFG1_FLD_DRV_EN should be enabled after
2207          * WM FWDL completed, otherwise RRO_MSDU_PG ring may broken
2208          */
2209         wed_set(dev, MTK_WED_RRO_MSDU_PG_RING2_CFG,
2210                 MTK_WED_RRO_MSDU_PG_DRV_EN);
2211
2212         for (i = 0; i < MTK_WED_RX_QUEUES; i++) {
2213                 struct mtk_wed_ring *ring = &dev->rx_rro_ring[i];
2214
2215                 if (!(ring->flags & MTK_WED_RING_CONFIGURED))
2216                         continue;
2217
2218                 if (mtk_wed_check_wfdma_rx_fill(dev, ring))
2219                         dev_err(dev->hw->dev,
2220                                 "rx_rro_ring(%d) initialization failed\n", i);
2221         }
2222
2223         for (i = 0; i < MTK_WED_RX_PAGE_QUEUES; i++) {
2224                 struct mtk_wed_ring *ring = &dev->rx_page_ring[i];
2225
2226                 if (!(ring->flags & MTK_WED_RING_CONFIGURED))
2227                         continue;
2228
2229                 if (mtk_wed_check_wfdma_rx_fill(dev, ring))
2230                         dev_err(dev->hw->dev,
2231                                 "rx_page_ring(%d) initialization failed\n", i);
2232         }
2233 }
2234
2235 static void
2236 mtk_wed_rro_rx_ring_setup(struct mtk_wed_device *dev, int idx,
2237                           void __iomem *regs)
2238 {
2239         struct mtk_wed_ring *ring = &dev->rx_rro_ring[idx];
2240
2241         ring->wpdma = regs;
2242         wed_w32(dev, MTK_WED_RRO_RX_D_RX(idx) + MTK_WED_RING_OFS_BASE,
2243                 readl(regs));
2244         wed_w32(dev, MTK_WED_RRO_RX_D_RX(idx) + MTK_WED_RING_OFS_COUNT,
2245                 readl(regs + MTK_WED_RING_OFS_COUNT));
2246         ring->flags |= MTK_WED_RING_CONFIGURED;
2247 }
2248
2249 static void
2250 mtk_wed_msdu_pg_rx_ring_setup(struct mtk_wed_device *dev, int idx, void __iomem *regs)
2251 {
2252         struct mtk_wed_ring *ring = &dev->rx_page_ring[idx];
2253
2254         ring->wpdma = regs;
2255         wed_w32(dev, MTK_WED_RRO_MSDU_PG_CTRL0(idx) + MTK_WED_RING_OFS_BASE,
2256                 readl(regs));
2257         wed_w32(dev, MTK_WED_RRO_MSDU_PG_CTRL0(idx) + MTK_WED_RING_OFS_COUNT,
2258                 readl(regs + MTK_WED_RING_OFS_COUNT));
2259         ring->flags |= MTK_WED_RING_CONFIGURED;
2260 }
2261
2262 static int
2263 mtk_wed_ind_rx_ring_setup(struct mtk_wed_device *dev, void __iomem *regs)
2264 {
2265         struct mtk_wed_ring *ring = &dev->ind_cmd_ring;
2266         u32 val = readl(regs + MTK_WED_RING_OFS_COUNT);
2267         int i, count = 0;
2268
2269         ring->wpdma = regs;
2270         wed_w32(dev, MTK_WED_IND_CMD_RX_CTRL1 + MTK_WED_RING_OFS_BASE,
2271                 readl(regs) & 0xfffffff0);
2272
2273         wed_w32(dev, MTK_WED_IND_CMD_RX_CTRL1 + MTK_WED_RING_OFS_COUNT,
2274                 readl(regs + MTK_WED_RING_OFS_COUNT));
2275
2276         /* ack sn cr */
2277         wed_w32(dev, MTK_WED_RRO_CFG0, dev->wlan.phy_base +
2278                 dev->wlan.ind_cmd.ack_sn_addr);
2279         wed_w32(dev, MTK_WED_RRO_CFG1,
2280                 FIELD_PREP(MTK_WED_RRO_CFG1_MAX_WIN_SZ,
2281                            dev->wlan.ind_cmd.win_size) |
2282                 FIELD_PREP(MTK_WED_RRO_CFG1_PARTICL_SE_ID,
2283                            dev->wlan.ind_cmd.particular_sid));
2284
2285         /* particular session addr element */
2286         wed_w32(dev, MTK_WED_ADDR_ELEM_CFG0,
2287                 dev->wlan.ind_cmd.particular_se_phys);
2288
2289         for (i = 0; i < dev->wlan.ind_cmd.se_group_nums; i++) {
2290                 wed_w32(dev, MTK_WED_RADDR_ELEM_TBL_WDATA,
2291                         dev->wlan.ind_cmd.addr_elem_phys[i] >> 4);
2292                 wed_w32(dev, MTK_WED_ADDR_ELEM_TBL_CFG,
2293                         MTK_WED_ADDR_ELEM_TBL_WR | (i & 0x7f));
2294
2295                 val = wed_r32(dev, MTK_WED_ADDR_ELEM_TBL_CFG);
2296                 while (!(val & MTK_WED_ADDR_ELEM_TBL_WR_RDY) && count++ < 100)
2297                         val = wed_r32(dev, MTK_WED_ADDR_ELEM_TBL_CFG);
2298                 if (count >= 100)
2299                         dev_err(dev->hw->dev,
2300                                 "write ba session base failed\n");
2301         }
2302
2303         /* pn check init */
2304         for (i = 0; i < dev->wlan.ind_cmd.particular_sid; i++) {
2305                 wed_w32(dev, MTK_WED_PN_CHECK_WDATA_M,
2306                         MTK_WED_PN_CHECK_IS_FIRST);
2307
2308                 wed_w32(dev, MTK_WED_PN_CHECK_CFG, MTK_WED_PN_CHECK_WR |
2309                         FIELD_PREP(MTK_WED_PN_CHECK_SE_ID, i));
2310
2311                 count = 0;
2312                 val = wed_r32(dev, MTK_WED_PN_CHECK_CFG);
2313                 while (!(val & MTK_WED_PN_CHECK_WR_RDY) && count++ < 100)
2314                         val = wed_r32(dev, MTK_WED_PN_CHECK_CFG);
2315                 if (count >= 100)
2316                         dev_err(dev->hw->dev,
2317                                 "session(%d) initialization failed\n", i);
2318         }
2319
2320         wed_w32(dev, MTK_WED_RX_IND_CMD_CNT0, MTK_WED_RX_IND_CMD_DBG_CNT_EN);
2321         wed_set(dev, MTK_WED_CTRL, MTK_WED_CTRL_WED_RX_IND_CMD_EN);
2322
2323         return 0;
2324 }
2325
2326 static void
2327 mtk_wed_start(struct mtk_wed_device *dev, u32 irq_mask)
2328 {
2329         int i;
2330
2331         if (mtk_wed_get_rx_capa(dev) && mtk_wed_rx_buffer_alloc(dev))
2332                 return;
2333
2334         for (i = 0; i < ARRAY_SIZE(dev->rx_wdma); i++)
2335                 if (!dev->rx_wdma[i].desc)
2336                         mtk_wed_wdma_rx_ring_setup(dev, i, 16, false);
2337
2338         mtk_wed_hw_init(dev);
2339         mtk_wed_configure_irq(dev, irq_mask);
2340
2341         mtk_wed_set_ext_int(dev, true);
2342
2343         if (mtk_wed_is_v1(dev->hw)) {
2344                 u32 val = dev->wlan.wpdma_phys | MTK_PCIE_MIRROR_MAP_EN |
2345                           FIELD_PREP(MTK_PCIE_MIRROR_MAP_WED_ID,
2346                                      dev->hw->index);
2347
2348                 val |= BIT(0) | (BIT(1) * !!dev->hw->index);
2349                 regmap_write(dev->hw->mirror, dev->hw->index * 4, val);
2350         } else if (mtk_wed_get_rx_capa(dev)) {
2351                 /* driver set mid ready and only once */
2352                 wed_w32(dev, MTK_WED_EXT_INT_MASK1,
2353                         MTK_WED_EXT_INT_STATUS_WPDMA_MID_RDY);
2354                 wed_w32(dev, MTK_WED_EXT_INT_MASK2,
2355                         MTK_WED_EXT_INT_STATUS_WPDMA_MID_RDY);
2356
2357                 wed_r32(dev, MTK_WED_EXT_INT_MASK1);
2358                 wed_r32(dev, MTK_WED_EXT_INT_MASK2);
2359
2360                 if (mtk_wed_is_v3_or_greater(dev->hw)) {
2361                         wed_w32(dev, MTK_WED_EXT_INT_MASK3,
2362                                 MTK_WED_EXT_INT_STATUS_WPDMA_MID_RDY);
2363                         wed_r32(dev, MTK_WED_EXT_INT_MASK3);
2364                 }
2365
2366                 if (mtk_wed_rro_cfg(dev))
2367                         return;
2368         }
2369
2370         mtk_wed_set_512_support(dev, dev->wlan.wcid_512);
2371         mtk_wed_amsdu_init(dev);
2372
2373         mtk_wed_dma_enable(dev);
2374         dev->running = true;
2375 }
2376
2377 static int
2378 mtk_wed_attach(struct mtk_wed_device *dev)
2379         __releases(RCU)
2380 {
2381         struct mtk_wed_hw *hw;
2382         struct device *device;
2383         int ret = 0;
2384
2385         RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
2386                          "mtk_wed_attach without holding the RCU read lock");
2387
2388         if ((dev->wlan.bus_type == MTK_WED_BUS_PCIE &&
2389              pci_domain_nr(dev->wlan.pci_dev->bus) > 1) ||
2390             !try_module_get(THIS_MODULE))
2391                 ret = -ENODEV;
2392
2393         rcu_read_unlock();
2394
2395         if (ret)
2396                 return ret;
2397
2398         mutex_lock(&hw_lock);
2399
2400         hw = mtk_wed_assign(dev);
2401         if (!hw) {
2402                 module_put(THIS_MODULE);
2403                 ret = -ENODEV;
2404                 goto unlock;
2405         }
2406
2407         device = dev->wlan.bus_type == MTK_WED_BUS_PCIE
2408                 ? &dev->wlan.pci_dev->dev
2409                 : &dev->wlan.platform_dev->dev;
2410         dev_info(device, "attaching wed device %d version %d\n",
2411                  hw->index, hw->version);
2412
2413         dev->hw = hw;
2414         dev->dev = hw->dev;
2415         dev->irq = hw->irq;
2416         dev->wdma_idx = hw->index;
2417         dev->version = hw->version;
2418         dev->hw->pcie_base = mtk_wed_get_pcie_base(dev);
2419
2420         if (hw->eth->dma_dev == hw->eth->dev &&
2421             of_dma_is_coherent(hw->eth->dev->of_node))
2422                 mtk_eth_set_dma_device(hw->eth, hw->dev);
2423
2424         ret = mtk_wed_tx_buffer_alloc(dev);
2425         if (ret)
2426                 goto out;
2427
2428         ret = mtk_wed_amsdu_buffer_alloc(dev);
2429         if (ret)
2430                 goto out;
2431
2432         if (mtk_wed_get_rx_capa(dev)) {
2433                 ret = mtk_wed_rro_alloc(dev);
2434                 if (ret)
2435                         goto out;
2436         }
2437
2438         mtk_wed_hw_init_early(dev);
2439         if (mtk_wed_is_v1(hw))
2440                 regmap_update_bits(hw->hifsys, HIFSYS_DMA_AG_MAP,
2441                                    BIT(hw->index), 0);
2442         else
2443                 dev->rev_id = wed_r32(dev, MTK_WED_REV_ID);
2444
2445         if (mtk_wed_get_rx_capa(dev))
2446                 ret = mtk_wed_wo_init(hw);
2447 out:
2448         if (ret) {
2449                 dev_err(dev->hw->dev, "failed to attach wed device\n");
2450                 __mtk_wed_detach(dev);
2451         }
2452 unlock:
2453         mutex_unlock(&hw_lock);
2454
2455         return ret;
2456 }
2457
2458 static int
2459 mtk_wed_tx_ring_setup(struct mtk_wed_device *dev, int idx, void __iomem *regs,
2460                       bool reset)
2461 {
2462         struct mtk_wed_ring *ring = &dev->tx_ring[idx];
2463
2464         /*
2465          * Tx ring redirection:
2466          * Instead of configuring the WLAN PDMA TX ring directly, the WLAN
2467          * driver allocated DMA ring gets configured into WED MTK_WED_RING_TX(n)
2468          * registers.
2469          *
2470          * WED driver posts its own DMA ring as WLAN PDMA TX and configures it
2471          * into MTK_WED_WPDMA_RING_TX(n) registers.
2472          * It gets filled with packets picked up from WED TX ring and from
2473          * WDMA RX.
2474          */
2475
2476         if (WARN_ON(idx >= ARRAY_SIZE(dev->tx_ring)))
2477                 return -EINVAL;
2478
2479         if (!reset && mtk_wed_ring_alloc(dev, ring, MTK_WED_TX_RING_SIZE,
2480                                          sizeof(*ring->desc), true))
2481                 return -ENOMEM;
2482
2483         if (mtk_wed_wdma_rx_ring_setup(dev, idx, MTK_WED_WDMA_RING_SIZE,
2484                                        reset))
2485                 return -ENOMEM;
2486
2487         ring->reg_base = MTK_WED_RING_TX(idx);
2488         ring->wpdma = regs;
2489
2490         if (mtk_wed_is_v3_or_greater(dev->hw) && idx == 1) {
2491                 /* reset prefetch index */
2492                 wed_set(dev, MTK_WED_WDMA_RX_PREF_CFG,
2493                         MTK_WED_WDMA_RX_PREF_RX0_SIDX_CLR |
2494                         MTK_WED_WDMA_RX_PREF_RX1_SIDX_CLR);
2495
2496                 wed_clr(dev, MTK_WED_WDMA_RX_PREF_CFG,
2497                         MTK_WED_WDMA_RX_PREF_RX0_SIDX_CLR |
2498                         MTK_WED_WDMA_RX_PREF_RX1_SIDX_CLR);
2499
2500                 /* reset prefetch FIFO */
2501                 wed_w32(dev, MTK_WED_WDMA_RX_PREF_FIFO_CFG,
2502                         MTK_WED_WDMA_RX_PREF_FIFO_RX0_CLR |
2503                         MTK_WED_WDMA_RX_PREF_FIFO_RX1_CLR);
2504                 wed_w32(dev, MTK_WED_WDMA_RX_PREF_FIFO_CFG, 0);
2505         }
2506
2507         /* WED -> WPDMA */
2508         wpdma_tx_w32(dev, idx, MTK_WED_RING_OFS_BASE, ring->desc_phys);
2509         wpdma_tx_w32(dev, idx, MTK_WED_RING_OFS_COUNT, MTK_WED_TX_RING_SIZE);
2510         wpdma_tx_w32(dev, idx, MTK_WED_RING_OFS_CPU_IDX, 0);
2511
2512         wed_w32(dev, MTK_WED_WPDMA_RING_TX(idx) + MTK_WED_RING_OFS_BASE,
2513                 ring->desc_phys);
2514         wed_w32(dev, MTK_WED_WPDMA_RING_TX(idx) + MTK_WED_RING_OFS_COUNT,
2515                 MTK_WED_TX_RING_SIZE);
2516         wed_w32(dev, MTK_WED_WPDMA_RING_TX(idx) + MTK_WED_RING_OFS_CPU_IDX, 0);
2517
2518         return 0;
2519 }
2520
2521 static int
2522 mtk_wed_txfree_ring_setup(struct mtk_wed_device *dev, void __iomem *regs)
2523 {
2524         struct mtk_wed_ring *ring = &dev->txfree_ring;
2525         int i, index = mtk_wed_is_v1(dev->hw);
2526
2527         /*
2528          * For txfree event handling, the same DMA ring is shared between WED
2529          * and WLAN. The WLAN driver accesses the ring index registers through
2530          * WED
2531          */
2532         ring->reg_base = MTK_WED_RING_RX(index);
2533         ring->wpdma = regs;
2534
2535         for (i = 0; i < 12; i += 4) {
2536                 u32 val = readl(regs + i);
2537
2538                 wed_w32(dev, MTK_WED_RING_RX(index) + i, val);
2539                 wed_w32(dev, MTK_WED_WPDMA_RING_RX(index) + i, val);
2540         }
2541
2542         return 0;
2543 }
2544
2545 static int
2546 mtk_wed_rx_ring_setup(struct mtk_wed_device *dev, int idx, void __iomem *regs,
2547                       bool reset)
2548 {
2549         struct mtk_wed_ring *ring = &dev->rx_ring[idx];
2550
2551         if (WARN_ON(idx >= ARRAY_SIZE(dev->rx_ring)))
2552                 return -EINVAL;
2553
2554         if (!reset && mtk_wed_ring_alloc(dev, ring, MTK_WED_RX_RING_SIZE,
2555                                          sizeof(*ring->desc), false))
2556                 return -ENOMEM;
2557
2558         if (mtk_wed_wdma_tx_ring_setup(dev, idx, MTK_WED_WDMA_RING_SIZE,
2559                                        reset))
2560                 return -ENOMEM;
2561
2562         ring->reg_base = MTK_WED_RING_RX_DATA(idx);
2563         ring->wpdma = regs;
2564         ring->flags |= MTK_WED_RING_CONFIGURED;
2565
2566         /* WPDMA ->  WED */
2567         wpdma_rx_w32(dev, idx, MTK_WED_RING_OFS_BASE, ring->desc_phys);
2568         wpdma_rx_w32(dev, idx, MTK_WED_RING_OFS_COUNT, MTK_WED_RX_RING_SIZE);
2569
2570         wed_w32(dev, MTK_WED_WPDMA_RING_RX_DATA(idx) + MTK_WED_RING_OFS_BASE,
2571                 ring->desc_phys);
2572         wed_w32(dev, MTK_WED_WPDMA_RING_RX_DATA(idx) + MTK_WED_RING_OFS_COUNT,
2573                 MTK_WED_RX_RING_SIZE);
2574
2575         return 0;
2576 }
2577
2578 static u32
2579 mtk_wed_irq_get(struct mtk_wed_device *dev, u32 mask)
2580 {
2581         u32 val, ext_mask;
2582
2583         if (mtk_wed_is_v3_or_greater(dev->hw))
2584                 ext_mask = MTK_WED_EXT_INT_STATUS_RX_DRV_COHERENT |
2585                            MTK_WED_EXT_INT_STATUS_TKID_WO_PYLD;
2586         else
2587                 ext_mask = MTK_WED_EXT_INT_STATUS_ERROR_MASK;
2588
2589         val = wed_r32(dev, MTK_WED_EXT_INT_STATUS);
2590         wed_w32(dev, MTK_WED_EXT_INT_STATUS, val);
2591         val &= ext_mask;
2592         if (!dev->hw->num_flows)
2593                 val &= ~MTK_WED_EXT_INT_STATUS_TKID_WO_PYLD;
2594         if (val && net_ratelimit())
2595                 pr_err("mtk_wed%d: error status=%08x\n", dev->hw->index, val);
2596
2597         val = wed_r32(dev, MTK_WED_INT_STATUS);
2598         val &= mask;
2599         wed_w32(dev, MTK_WED_INT_STATUS, val); /* ACK */
2600
2601         return val;
2602 }
2603
2604 static void
2605 mtk_wed_irq_set_mask(struct mtk_wed_device *dev, u32 mask)
2606 {
2607         mtk_wed_set_ext_int(dev, !!mask);
2608         wed_w32(dev, MTK_WED_INT_MASK, mask);
2609 }
2610
2611 int mtk_wed_flow_add(int index)
2612 {
2613         struct mtk_wed_hw *hw = hw_list[index];
2614         int ret = 0;
2615
2616         mutex_lock(&hw_lock);
2617
2618         if (!hw || !hw->wed_dev) {
2619                 ret = -ENODEV;
2620                 goto out;
2621         }
2622
2623         if (!hw->wed_dev->wlan.offload_enable)
2624                 goto out;
2625
2626         if (hw->num_flows) {
2627                 hw->num_flows++;
2628                 goto out;
2629         }
2630
2631         ret = hw->wed_dev->wlan.offload_enable(hw->wed_dev);
2632         if (!ret)
2633                 hw->num_flows++;
2634         mtk_wed_set_ext_int(hw->wed_dev, true);
2635
2636 out:
2637         mutex_unlock(&hw_lock);
2638
2639         return ret;
2640 }
2641
2642 void mtk_wed_flow_remove(int index)
2643 {
2644         struct mtk_wed_hw *hw = hw_list[index];
2645
2646         mutex_lock(&hw_lock);
2647
2648         if (!hw || !hw->wed_dev)
2649                 goto out;
2650
2651         if (!hw->wed_dev->wlan.offload_disable)
2652                 goto out;
2653
2654         if (--hw->num_flows)
2655                 goto out;
2656
2657         hw->wed_dev->wlan.offload_disable(hw->wed_dev);
2658         mtk_wed_set_ext_int(hw->wed_dev, true);
2659
2660 out:
2661         mutex_unlock(&hw_lock);
2662 }
2663
2664 static int
2665 mtk_wed_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv)
2666 {
2667         struct mtk_wed_flow_block_priv *priv = cb_priv;
2668         struct flow_cls_offload *cls = type_data;
2669         struct mtk_wed_hw *hw = NULL;
2670
2671         if (!priv || !tc_can_offload(priv->dev))
2672                 return -EOPNOTSUPP;
2673
2674         if (type != TC_SETUP_CLSFLOWER)
2675                 return -EOPNOTSUPP;
2676
2677         hw = priv->hw;
2678         return mtk_flow_offload_cmd(hw->eth, cls, hw->index);
2679 }
2680
2681 static int
2682 mtk_wed_setup_tc_block(struct mtk_wed_hw *hw, struct net_device *dev,
2683                        struct flow_block_offload *f)
2684 {
2685         struct mtk_wed_flow_block_priv *priv;
2686         static LIST_HEAD(block_cb_list);
2687         struct flow_block_cb *block_cb;
2688         struct mtk_eth *eth = hw->eth;
2689         flow_setup_cb_t *cb;
2690
2691         if (!eth->soc->offload_version)
2692                 return -EOPNOTSUPP;
2693
2694         if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
2695                 return -EOPNOTSUPP;
2696
2697         cb = mtk_wed_setup_tc_block_cb;
2698         f->driver_block_list = &block_cb_list;
2699
2700         switch (f->command) {
2701         case FLOW_BLOCK_BIND:
2702                 block_cb = flow_block_cb_lookup(f->block, cb, dev);
2703                 if (block_cb) {
2704                         flow_block_cb_incref(block_cb);
2705                         return 0;
2706                 }
2707
2708                 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
2709                 if (!priv)
2710                         return -ENOMEM;
2711
2712                 priv->hw = hw;
2713                 priv->dev = dev;
2714                 block_cb = flow_block_cb_alloc(cb, dev, priv, NULL);
2715                 if (IS_ERR(block_cb)) {
2716                         kfree(priv);
2717                         return PTR_ERR(block_cb);
2718                 }
2719
2720                 flow_block_cb_incref(block_cb);
2721                 flow_block_cb_add(block_cb, f);
2722                 list_add_tail(&block_cb->driver_list, &block_cb_list);
2723                 return 0;
2724         case FLOW_BLOCK_UNBIND:
2725                 block_cb = flow_block_cb_lookup(f->block, cb, dev);
2726                 if (!block_cb)
2727                         return -ENOENT;
2728
2729                 if (!flow_block_cb_decref(block_cb)) {
2730                         flow_block_cb_remove(block_cb, f);
2731                         list_del(&block_cb->driver_list);
2732                         kfree(block_cb->cb_priv);
2733                         block_cb->cb_priv = NULL;
2734                 }
2735                 return 0;
2736         default:
2737                 return -EOPNOTSUPP;
2738         }
2739 }
2740
2741 static int
2742 mtk_wed_setup_tc(struct mtk_wed_device *wed, struct net_device *dev,
2743                  enum tc_setup_type type, void *type_data)
2744 {
2745         struct mtk_wed_hw *hw = wed->hw;
2746
2747         if (mtk_wed_is_v1(hw))
2748                 return -EOPNOTSUPP;
2749
2750         switch (type) {
2751         case TC_SETUP_BLOCK:
2752         case TC_SETUP_FT:
2753                 return mtk_wed_setup_tc_block(hw, dev, type_data);
2754         default:
2755                 return -EOPNOTSUPP;
2756         }
2757 }
2758
2759 void mtk_wed_add_hw(struct device_node *np, struct mtk_eth *eth,
2760                     void __iomem *wdma, phys_addr_t wdma_phy,
2761                     int index)
2762 {
2763         static const struct mtk_wed_ops wed_ops = {
2764                 .attach = mtk_wed_attach,
2765                 .tx_ring_setup = mtk_wed_tx_ring_setup,
2766                 .rx_ring_setup = mtk_wed_rx_ring_setup,
2767                 .txfree_ring_setup = mtk_wed_txfree_ring_setup,
2768                 .msg_update = mtk_wed_mcu_msg_update,
2769                 .start = mtk_wed_start,
2770                 .stop = mtk_wed_stop,
2771                 .reset_dma = mtk_wed_reset_dma,
2772                 .reg_read = wed_r32,
2773                 .reg_write = wed_w32,
2774                 .irq_get = mtk_wed_irq_get,
2775                 .irq_set_mask = mtk_wed_irq_set_mask,
2776                 .detach = mtk_wed_detach,
2777                 .ppe_check = mtk_wed_ppe_check,
2778                 .setup_tc = mtk_wed_setup_tc,
2779                 .start_hw_rro = mtk_wed_start_hw_rro,
2780                 .rro_rx_ring_setup = mtk_wed_rro_rx_ring_setup,
2781                 .msdu_pg_rx_ring_setup = mtk_wed_msdu_pg_rx_ring_setup,
2782                 .ind_rx_ring_setup = mtk_wed_ind_rx_ring_setup,
2783         };
2784         struct device_node *eth_np = eth->dev->of_node;
2785         struct platform_device *pdev;
2786         struct mtk_wed_hw *hw;
2787         struct regmap *regs;
2788         int irq;
2789
2790         if (!np)
2791                 return;
2792
2793         pdev = of_find_device_by_node(np);
2794         if (!pdev)
2795                 goto err_of_node_put;
2796
2797         get_device(&pdev->dev);
2798         irq = platform_get_irq(pdev, 0);
2799         if (irq < 0)
2800                 goto err_put_device;
2801
2802         regs = syscon_regmap_lookup_by_phandle(np, NULL);
2803         if (IS_ERR(regs))
2804                 goto err_put_device;
2805
2806         rcu_assign_pointer(mtk_soc_wed_ops, &wed_ops);
2807
2808         mutex_lock(&hw_lock);
2809
2810         if (WARN_ON(hw_list[index]))
2811                 goto unlock;
2812
2813         hw = kzalloc(sizeof(*hw), GFP_KERNEL);
2814         if (!hw)
2815                 goto unlock;
2816
2817         hw->node = np;
2818         hw->regs = regs;
2819         hw->eth = eth;
2820         hw->dev = &pdev->dev;
2821         hw->wdma_phy = wdma_phy;
2822         hw->wdma = wdma;
2823         hw->index = index;
2824         hw->irq = irq;
2825         hw->version = eth->soc->version;
2826
2827         switch (hw->version) {
2828         case 2:
2829                 hw->soc = &mt7986_data;
2830                 break;
2831         case 3:
2832                 hw->soc = &mt7988_data;
2833                 break;
2834         default:
2835         case 1:
2836                 hw->mirror = syscon_regmap_lookup_by_phandle(eth_np,
2837                                 "mediatek,pcie-mirror");
2838                 hw->hifsys = syscon_regmap_lookup_by_phandle(eth_np,
2839                                 "mediatek,hifsys");
2840                 if (IS_ERR(hw->mirror) || IS_ERR(hw->hifsys)) {
2841                         kfree(hw);
2842                         goto unlock;
2843                 }
2844
2845                 if (!index) {
2846                         regmap_write(hw->mirror, 0, 0);
2847                         regmap_write(hw->mirror, 4, 0);
2848                 }
2849                 hw->soc = &mt7622_data;
2850                 break;
2851         }
2852
2853         mtk_wed_hw_add_debugfs(hw);
2854
2855         hw_list[index] = hw;
2856
2857         mutex_unlock(&hw_lock);
2858
2859         return;
2860
2861 unlock:
2862         mutex_unlock(&hw_lock);
2863 err_put_device:
2864         put_device(&pdev->dev);
2865 err_of_node_put:
2866         of_node_put(np);
2867 }
2868
2869 void mtk_wed_exit(void)
2870 {
2871         int i;
2872
2873         rcu_assign_pointer(mtk_soc_wed_ops, NULL);
2874
2875         synchronize_rcu();
2876
2877         for (i = 0; i < ARRAY_SIZE(hw_list); i++) {
2878                 struct mtk_wed_hw *hw;
2879
2880                 hw = hw_list[i];
2881                 if (!hw)
2882                         continue;
2883
2884                 hw_list[i] = NULL;
2885                 debugfs_remove(hw->debugfs_dir);
2886                 put_device(hw->dev);
2887                 of_node_put(hw->node);
2888                 kfree(hw);
2889         }
2890 }