drivers/net/wwan/t7xx/t7xx_hif_cldma.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2021, MediaTek Inc.
   4  * Copyright (c) 2021-2022, Intel Corporation.
   5  *
   6  * Authors:
   7  *  Amir Hanania <[email protected]>
   8  *  Haijun Liu <[email protected]>
   9  *  Moises Veleta <[email protected]>
  10  *  Ricardo Martinez <[email protected]>
  11  *  Sreehari Kancharla <[email protected]>
  12  *
  13  * Contributors:
  14  *  Andy Shevchenko <[email protected]>
  15  *  Chiranjeevi Rapolu <[email protected]>
  16  *  Eliot Lee <[email protected]>
  17  */
  18
  19 #include <linux/bits.h>
  20 #include <linux/bitops.h>
  21 #include <linux/delay.h>
  22 #include <linux/device.h>
  23 #include <linux/dmapool.h>
  24 #include <linux/dma-mapping.h>
  25 #include <linux/dma-direction.h>
  26 #include <linux/gfp.h>
  27 #include <linux/io.h>
  28 #include <linux/io-64-nonatomic-lo-hi.h>
  29 #include <linux/iopoll.h>
  30 #include <linux/irqreturn.h>
  31 #include <linux/kernel.h>
  32 #include <linux/kthread.h>
  33 #include <linux/list.h>
  34 #include <linux/netdevice.h>
  35 #include <linux/pci.h>
  36 #include <linux/pm_runtime.h>
  37 #include <linux/sched.h>
  38 #include <linux/skbuff.h>
  39 #include <linux/slab.h>
  40 #include <linux/spinlock.h>
  41 #include <linux/types.h>
  42 #include <linux/wait.h>
  43 #include <linux/workqueue.h>
  44
  45 #include "t7xx_cldma.h"
  46 #include "t7xx_hif_cldma.h"
  47 #include "t7xx_mhccif.h"
  48 #include "t7xx_pci.h"
  49 #include "t7xx_pcie_mac.h"
  50 #include "t7xx_port_proxy.h"
  51 #include "t7xx_reg.h"
  52 #include "t7xx_state_monitor.h"
  53
  54 #define MAX_TX_BUDGET                   16
  55 #define MAX_RX_BUDGET                   16
  56
  57 #define CHECK_Q_STOP_TIMEOUT_US         1000000
  58 #define CHECK_Q_STOP_STEP_US            10000
  59
  60 #define CLDMA_JUMBO_BUFF_SZ             (63 * 1024 + sizeof(struct ccci_header))
  61
  62 static void md_cd_queue_struct_reset(struct cldma_queue *queue, struct cldma_ctrl *md_ctrl,
  63                                      enum mtk_txrx tx_rx, unsigned int index)
  64 {
  65         queue->dir = tx_rx;
  66         queue->index = index;
  67         queue->md_ctrl = md_ctrl;
  68         queue->tr_ring = NULL;
  69         queue->tr_done = NULL;
  70         queue->tx_next = NULL;
  71 }
  72
  73 static void md_cd_queue_struct_init(struct cldma_queue *queue, struct cldma_ctrl *md_ctrl,
  74                                     enum mtk_txrx tx_rx, unsigned int index)
  75 {
  76         md_cd_queue_struct_reset(queue, md_ctrl, tx_rx, index);
  77         init_waitqueue_head(&queue->req_wq);
  78         spin_lock_init(&queue->ring_lock);
  79 }
  80
  81 static void t7xx_cldma_gpd_set_data_ptr(struct cldma_gpd *gpd, dma_addr_t data_ptr)
  82 {
  83         gpd->data_buff_bd_ptr_h = cpu_to_le32(upper_32_bits(data_ptr));
  84         gpd->data_buff_bd_ptr_l = cpu_to_le32(lower_32_bits(data_ptr));
  85 }
  86
  87 static void t7xx_cldma_gpd_set_next_ptr(struct cldma_gpd *gpd, dma_addr_t next_ptr)
  88 {
  89         gpd->next_gpd_ptr_h = cpu_to_le32(upper_32_bits(next_ptr));
  90         gpd->next_gpd_ptr_l = cpu_to_le32(lower_32_bits(next_ptr));
  91 }
  92
  93 static int t7xx_cldma_alloc_and_map_skb(struct cldma_ctrl *md_ctrl, struct cldma_request *req,
  94                                         size_t size, gfp_t gfp_mask)
  95 {
  96         req->skb = __dev_alloc_skb(size, gfp_mask);
  97         if (!req->skb)
  98                 return -ENOMEM;
  99
 100         req->mapped_buff = dma_map_single(md_ctrl->dev, req->skb->data, size, DMA_FROM_DEVICE);
 101         if (dma_mapping_error(md_ctrl->dev, req->mapped_buff)) {
 102                 dev_kfree_skb_any(req->skb);
 103                 req->skb = NULL;
 104                 req->mapped_buff = 0;
 105                 dev_err(md_ctrl->dev, "DMA mapping failed\n");
 106                 return -ENOMEM;
 107         }
 108
 109         return 0;
 110 }
 111
 112 static int t7xx_cldma_gpd_rx_from_q(struct cldma_queue *queue, int budget, bool *over_budget)
 113 {
 114         struct cldma_ctrl *md_ctrl = queue->md_ctrl;
 115         unsigned int hwo_polling_count = 0;
 116         struct t7xx_cldma_hw *hw_info;
 117         bool rx_not_done = true;
 118         unsigned long flags;
 119         int count = 0;
 120
 121         hw_info = &md_ctrl->hw_info;
 122
 123         do {
 124                 struct cldma_request *req;
 125                 struct cldma_gpd *gpd;
 126                 struct sk_buff *skb;
 127                 int ret;
 128
 129                 req = queue->tr_done;
 130                 if (!req)
 131                         return -ENODATA;
 132
 133                 gpd = req->gpd;
 134                 if ((gpd->flags & GPD_FLAGS_HWO) || !req->skb) {
 135                         dma_addr_t gpd_addr;
 136
 137                         if (!pci_device_is_present(to_pci_dev(md_ctrl->dev))) {
 138                                 dev_err(md_ctrl->dev, "PCIe Link disconnected\n");
 139                                 return -ENODEV;
 140                         }
 141
 142                         gpd_addr = ioread64(hw_info->ap_pdn_base + REG_CLDMA_DL_CURRENT_ADDRL_0 +
 143                                             queue->index * sizeof(u64));
 144                         if (req->gpd_addr == gpd_addr || hwo_polling_count++ >= 100)
 145                                 return 0;
 146
 147                         udelay(1);
 148                         continue;
 149                 }
 150
 151                 hwo_polling_count = 0;
 152                 skb = req->skb;
 153
 154                 if (req->mapped_buff) {
 155                         dma_unmap_single(md_ctrl->dev, req->mapped_buff,
 156                                          queue->tr_ring->pkt_size, DMA_FROM_DEVICE);
 157                         req->mapped_buff = 0;
 158                 }
 159
 160                 skb->len = 0;
 161                 skb_reset_tail_pointer(skb);
 162                 skb_put(skb, le16_to_cpu(gpd->data_buff_len));
 163
 164                 ret = md_ctrl->recv_skb(queue, skb);
 165                 /* Break processing, will try again later */
 166                 if (ret < 0)
 167                         return ret;
 168
 169                 req->skb = NULL;
 170                 t7xx_cldma_gpd_set_data_ptr(gpd, 0);
 171
 172                 spin_lock_irqsave(&queue->ring_lock, flags);
 173                 queue->tr_done = list_next_entry_circular(req, &queue->tr_ring->gpd_ring, entry);
 174                 spin_unlock_irqrestore(&queue->ring_lock, flags);
 175                 req = queue->rx_refill;
 176
 177                 ret = t7xx_cldma_alloc_and_map_skb(md_ctrl, req, queue->tr_ring->pkt_size, GFP_KERNEL);
 178                 if (ret)
 179                         return ret;
 180
 181                 gpd = req->gpd;
 182                 t7xx_cldma_gpd_set_data_ptr(gpd, req->mapped_buff);
 183                 gpd->data_buff_len = 0;
 184                 gpd->flags = GPD_FLAGS_IOC | GPD_FLAGS_HWO;
 185
 186                 spin_lock_irqsave(&queue->ring_lock, flags);
 187                 queue->rx_refill = list_next_entry_circular(req, &queue->tr_ring->gpd_ring, entry);
 188                 spin_unlock_irqrestore(&queue->ring_lock, flags);
 189
 190                 rx_not_done = ++count < budget || !need_resched();
 191         } while (rx_not_done);
 192
 193         *over_budget = true;
 194         return 0;
 195 }
 196
 197 static int t7xx_cldma_gpd_rx_collect(struct cldma_queue *queue, int budget)
 198 {
 199         struct cldma_ctrl *md_ctrl = queue->md_ctrl;
 200         struct t7xx_cldma_hw *hw_info;
 201         unsigned int pending_rx_int;
 202         bool over_budget = false;
 203         unsigned long flags;
 204         int ret;
 205
 206         hw_info = &md_ctrl->hw_info;
 207
 208         do {
 209                 ret = t7xx_cldma_gpd_rx_from_q(queue, budget, &over_budget);
 210                 if (ret == -ENODATA)
 211                         return 0;
 212                 else if (ret)
 213                         return ret;
 214
 215                 pending_rx_int = 0;
 216
 217                 spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 218                 if (md_ctrl->rxq_active & BIT(queue->index)) {
 219                         if (!t7xx_cldma_hw_queue_status(hw_info, queue->index, MTK_RX))
 220                                 t7xx_cldma_hw_resume_queue(hw_info, queue->index, MTK_RX);
 221
 222                         pending_rx_int = t7xx_cldma_hw_int_status(hw_info, BIT(queue->index),
 223                                                                   MTK_RX);
 224                         if (pending_rx_int) {
 225                                 t7xx_cldma_hw_rx_done(hw_info, pending_rx_int);
 226
 227                                 if (over_budget) {
 228                                         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 229                                         return -EAGAIN;
 230                                 }
 231                         }
 232                 }
 233                 spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 234         } while (pending_rx_int);
 235
 236         return 0;
 237 }
 238
 239 static void t7xx_cldma_rx_done(struct work_struct *work)
 240 {
 241         struct cldma_queue *queue = container_of(work, struct cldma_queue, cldma_work);
 242         struct cldma_ctrl *md_ctrl = queue->md_ctrl;
 243         int value;
 244
 245         value = t7xx_cldma_gpd_rx_collect(queue, queue->budget);
 246         if (value && md_ctrl->rxq_active & BIT(queue->index)) {
 247                 queue_work(queue->worker, &queue->cldma_work);
 248                 return;
 249         }
 250
 251         t7xx_cldma_clear_ip_busy(&md_ctrl->hw_info);
 252         t7xx_cldma_hw_irq_en_txrx(&md_ctrl->hw_info, queue->index, MTK_RX);
 253         t7xx_cldma_hw_irq_en_eq(&md_ctrl->hw_info, queue->index, MTK_RX);
 254         pm_runtime_mark_last_busy(md_ctrl->dev);
 255         pm_runtime_put_autosuspend(md_ctrl->dev);
 256 }
 257
 258 static int t7xx_cldma_gpd_tx_collect(struct cldma_queue *queue)
 259 {
 260         struct cldma_ctrl *md_ctrl = queue->md_ctrl;
 261         unsigned int dma_len, count = 0;
 262         struct cldma_request *req;
 263         struct cldma_gpd *gpd;
 264         unsigned long flags;
 265         dma_addr_t dma_free;
 266         struct sk_buff *skb;
 267
 268         while (!kthread_should_stop()) {
 269                 spin_lock_irqsave(&queue->ring_lock, flags);
 270                 req = queue->tr_done;
 271                 if (!req) {
 272                         spin_unlock_irqrestore(&queue->ring_lock, flags);
 273                         break;
 274                 }
 275                 gpd = req->gpd;
 276                 if ((gpd->flags & GPD_FLAGS_HWO) || !req->skb) {
 277                         spin_unlock_irqrestore(&queue->ring_lock, flags);
 278                         break;
 279                 }
 280                 queue->budget++;
 281                 dma_free = req->mapped_buff;
 282                 dma_len = le16_to_cpu(gpd->data_buff_len);
 283                 skb = req->skb;
 284                 req->skb = NULL;
 285                 queue->tr_done = list_next_entry_circular(req, &queue->tr_ring->gpd_ring, entry);
 286                 spin_unlock_irqrestore(&queue->ring_lock, flags);
 287
 288                 count++;
 289                 dma_unmap_single(md_ctrl->dev, dma_free, dma_len, DMA_TO_DEVICE);
 290                 dev_kfree_skb_any(skb);
 291         }
 292
 293         if (count)
 294                 wake_up_nr(&queue->req_wq, count);
 295
 296         return count;
 297 }
 298
 299 static void t7xx_cldma_txq_empty_hndl(struct cldma_queue *queue)
 300 {
 301         struct cldma_ctrl *md_ctrl = queue->md_ctrl;
 302         struct cldma_request *req;
 303         dma_addr_t ul_curr_addr;
 304         unsigned long flags;
 305         bool pending_gpd;
 306
 307         if (!(md_ctrl->txq_active & BIT(queue->index)))
 308                 return;
 309
 310         spin_lock_irqsave(&queue->ring_lock, flags);
 311         req = list_prev_entry_circular(queue->tx_next, &queue->tr_ring->gpd_ring, entry);
 312         spin_unlock_irqrestore(&queue->ring_lock, flags);
 313
 314         pending_gpd = (req->gpd->flags & GPD_FLAGS_HWO) && req->skb;
 315
 316         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 317         if (pending_gpd) {
 318                 struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
 319
 320                 /* Check current processing TGPD, 64-bit address is in a table by Q index */
 321                 ul_curr_addr = ioread64(hw_info->ap_pdn_base + REG_CLDMA_UL_CURRENT_ADDRL_0 +
 322                                         queue->index * sizeof(u64));
 323                 if (req->gpd_addr != ul_curr_addr) {
 324                         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 325                         dev_err(md_ctrl->dev, "CLDMA%d queue %d is not empty\n",
 326                                 md_ctrl->hif_id, queue->index);
 327                         return;
 328                 }
 329
 330                 t7xx_cldma_hw_resume_queue(hw_info, queue->index, MTK_TX);
 331         }
 332         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 333 }
 334
 335 static void t7xx_cldma_tx_done(struct work_struct *work)
 336 {
 337         struct cldma_queue *queue = container_of(work, struct cldma_queue, cldma_work);
 338         struct cldma_ctrl *md_ctrl = queue->md_ctrl;
 339         struct t7xx_cldma_hw *hw_info;
 340         unsigned int l2_tx_int;
 341         unsigned long flags;
 342
 343         hw_info = &md_ctrl->hw_info;
 344         t7xx_cldma_gpd_tx_collect(queue);
 345         l2_tx_int = t7xx_cldma_hw_int_status(hw_info, BIT(queue->index) | EQ_STA_BIT(queue->index),
 346                                              MTK_TX);
 347         if (l2_tx_int & EQ_STA_BIT(queue->index)) {
 348                 t7xx_cldma_hw_tx_done(hw_info, EQ_STA_BIT(queue->index));
 349                 t7xx_cldma_txq_empty_hndl(queue);
 350         }
 351
 352         if (l2_tx_int & BIT(queue->index)) {
 353                 t7xx_cldma_hw_tx_done(hw_info, BIT(queue->index));
 354                 queue_work(queue->worker, &queue->cldma_work);
 355                 return;
 356         }
 357
 358         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 359         if (md_ctrl->txq_active & BIT(queue->index)) {
 360                 t7xx_cldma_clear_ip_busy(hw_info);
 361                 t7xx_cldma_hw_irq_en_eq(hw_info, queue->index, MTK_TX);
 362                 t7xx_cldma_hw_irq_en_txrx(hw_info, queue->index, MTK_TX);
 363         }
 364         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 365
 366         pm_runtime_mark_last_busy(md_ctrl->dev);
 367         pm_runtime_put_autosuspend(md_ctrl->dev);
 368 }
 369
 370 static void t7xx_cldma_ring_free(struct cldma_ctrl *md_ctrl,
 371                                  struct cldma_ring *ring, enum dma_data_direction tx_rx)
 372 {
 373         struct cldma_request *req_cur, *req_next;
 374
 375         list_for_each_entry_safe(req_cur, req_next, &ring->gpd_ring, entry) {
 376                 if (req_cur->mapped_buff && req_cur->skb) {
 377                         dma_unmap_single(md_ctrl->dev, req_cur->mapped_buff,
 378                                          ring->pkt_size, tx_rx);
 379                         req_cur->mapped_buff = 0;
 380                 }
 381
 382                 dev_kfree_skb_any(req_cur->skb);
 383
 384                 if (req_cur->gpd)
 385                         dma_pool_free(md_ctrl->gpd_dmapool, req_cur->gpd, req_cur->gpd_addr);
 386
 387                 list_del(&req_cur->entry);
 388                 kfree(req_cur);
 389         }
 390 }
 391
 392 static struct cldma_request *t7xx_alloc_rx_request(struct cldma_ctrl *md_ctrl, size_t pkt_size)
 393 {
 394         struct cldma_request *req;
 395         int val;
 396
 397         req = kzalloc(sizeof(*req), GFP_KERNEL);
 398         if (!req)
 399                 return NULL;
 400
 401         req->gpd = dma_pool_zalloc(md_ctrl->gpd_dmapool, GFP_KERNEL, &req->gpd_addr);
 402         if (!req->gpd)
 403                 goto err_free_req;
 404
 405         val = t7xx_cldma_alloc_and_map_skb(md_ctrl, req, pkt_size, GFP_KERNEL);
 406         if (val)
 407                 goto err_free_pool;
 408
 409         return req;
 410
 411 err_free_pool:
 412         dma_pool_free(md_ctrl->gpd_dmapool, req->gpd, req->gpd_addr);
 413
 414 err_free_req:
 415         kfree(req);
 416
 417         return NULL;
 418 }
 419
 420 static int t7xx_cldma_rx_ring_init(struct cldma_ctrl *md_ctrl, struct cldma_ring *ring)
 421 {
 422         struct cldma_request *req;
 423         struct cldma_gpd *gpd;
 424         int i;
 425
 426         INIT_LIST_HEAD(&ring->gpd_ring);
 427         ring->length = MAX_RX_BUDGET;
 428
 429         for (i = 0; i < ring->length; i++) {
 430                 req = t7xx_alloc_rx_request(md_ctrl, ring->pkt_size);
 431                 if (!req) {
 432                         t7xx_cldma_ring_free(md_ctrl, ring, DMA_FROM_DEVICE);
 433                         return -ENOMEM;
 434                 }
 435
 436                 gpd = req->gpd;
 437                 t7xx_cldma_gpd_set_data_ptr(gpd, req->mapped_buff);
 438                 gpd->rx_data_allow_len = cpu_to_le16(ring->pkt_size);
 439                 gpd->flags = GPD_FLAGS_IOC | GPD_FLAGS_HWO;
 440                 INIT_LIST_HEAD(&req->entry);
 441                 list_add_tail(&req->entry, &ring->gpd_ring);
 442         }
 443
 444         /* Link previous GPD to next GPD, circular */
 445         list_for_each_entry(req, &ring->gpd_ring, entry) {
 446                 t7xx_cldma_gpd_set_next_ptr(gpd, req->gpd_addr);
 447                 gpd = req->gpd;
 448         }
 449
 450         return 0;
 451 }
 452
 453 static struct cldma_request *t7xx_alloc_tx_request(struct cldma_ctrl *md_ctrl)
 454 {
 455         struct cldma_request *req;
 456
 457         req = kzalloc(sizeof(*req), GFP_KERNEL);
 458         if (!req)
 459                 return NULL;
 460
 461         req->gpd = dma_pool_zalloc(md_ctrl->gpd_dmapool, GFP_KERNEL, &req->gpd_addr);
 462         if (!req->gpd) {
 463                 kfree(req);
 464                 return NULL;
 465         }
 466
 467         return req;
 468 }
 469
 470 static int t7xx_cldma_tx_ring_init(struct cldma_ctrl *md_ctrl, struct cldma_ring *ring)
 471 {
 472         struct cldma_request *req;
 473         struct cldma_gpd *gpd;
 474         int i;
 475
 476         INIT_LIST_HEAD(&ring->gpd_ring);
 477         ring->length = MAX_TX_BUDGET;
 478
 479         for (i = 0; i < ring->length; i++) {
 480                 req = t7xx_alloc_tx_request(md_ctrl);
 481                 if (!req) {
 482                         t7xx_cldma_ring_free(md_ctrl, ring, DMA_TO_DEVICE);
 483                         return -ENOMEM;
 484                 }
 485
 486                 gpd = req->gpd;
 487                 gpd->flags = GPD_FLAGS_IOC;
 488                 INIT_LIST_HEAD(&req->entry);
 489                 list_add_tail(&req->entry, &ring->gpd_ring);
 490         }
 491
 492         /* Link previous GPD to next GPD, circular */
 493         list_for_each_entry(req, &ring->gpd_ring, entry) {
 494                 t7xx_cldma_gpd_set_next_ptr(gpd, req->gpd_addr);
 495                 gpd = req->gpd;
 496         }
 497
 498         return 0;
 499 }
 500
 501 /**
 502  * t7xx_cldma_q_reset() - Reset CLDMA request pointers to their initial values.
 503  * @queue: Pointer to the queue structure.
 504  *
 505  * Called with ring_lock (unless called during initialization phase)
 506  */
 507 static void t7xx_cldma_q_reset(struct cldma_queue *queue)
 508 {
 509         struct cldma_request *req;
 510
 511         req = list_first_entry(&queue->tr_ring->gpd_ring, struct cldma_request, entry);
 512         queue->tr_done = req;
 513         queue->budget = queue->tr_ring->length;
 514
 515         if (queue->dir == MTK_TX)
 516                 queue->tx_next = req;
 517         else
 518                 queue->rx_refill = req;
 519 }
 520
 521 static void t7xx_cldma_rxq_init(struct cldma_queue *queue)
 522 {
 523         struct cldma_ctrl *md_ctrl = queue->md_ctrl;
 524
 525         queue->dir = MTK_RX;
 526         queue->tr_ring = &md_ctrl->rx_ring[queue->index];
 527         t7xx_cldma_q_reset(queue);
 528 }
 529
 530 static void t7xx_cldma_txq_init(struct cldma_queue *queue)
 531 {
 532         struct cldma_ctrl *md_ctrl = queue->md_ctrl;
 533
 534         queue->dir = MTK_TX;
 535         queue->tr_ring = &md_ctrl->tx_ring[queue->index];
 536         t7xx_cldma_q_reset(queue);
 537 }
 538
 539 static void t7xx_cldma_enable_irq(struct cldma_ctrl *md_ctrl)
 540 {
 541         t7xx_pcie_mac_set_int(md_ctrl->t7xx_dev, md_ctrl->hw_info.phy_interrupt_id);
 542 }
 543
 544 static void t7xx_cldma_disable_irq(struct cldma_ctrl *md_ctrl)
 545 {
 546         t7xx_pcie_mac_clear_int(md_ctrl->t7xx_dev, md_ctrl->hw_info.phy_interrupt_id);
 547 }
 548
 549 static void t7xx_cldma_irq_work_cb(struct cldma_ctrl *md_ctrl)
 550 {
 551         unsigned long l2_tx_int_msk, l2_rx_int_msk, l2_tx_int, l2_rx_int, val;
 552         struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
 553         int i;
 554
 555         /* L2 raw interrupt status */
 556         l2_tx_int = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L2TISAR0);
 557         l2_rx_int = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L2RISAR0);
 558         l2_tx_int_msk = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L2TIMR0);
 559         l2_rx_int_msk = ioread32(hw_info->ap_ao_base + REG_CLDMA_L2RIMR0);
 560         l2_tx_int &= ~l2_tx_int_msk;
 561         l2_rx_int &= ~l2_rx_int_msk;
 562
 563         if (l2_tx_int) {
 564                 if (l2_tx_int & (TQ_ERR_INT_BITMASK | TQ_ACTIVE_START_ERR_INT_BITMASK)) {
 565                         /* Read and clear L3 TX interrupt status */
 566                         val = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L3TISAR0);
 567                         iowrite32(val, hw_info->ap_pdn_base + REG_CLDMA_L3TISAR0);
 568                         val = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L3TISAR1);
 569                         iowrite32(val, hw_info->ap_pdn_base + REG_CLDMA_L3TISAR1);
 570                 }
 571
 572                 t7xx_cldma_hw_tx_done(hw_info, l2_tx_int);
 573                 if (l2_tx_int & (TXRX_STATUS_BITMASK | EMPTY_STATUS_BITMASK)) {
 574                         for_each_set_bit(i, &l2_tx_int, L2_INT_BIT_COUNT) {
 575                                 if (i < CLDMA_TXQ_NUM) {
 576                                         pm_runtime_get(md_ctrl->dev);
 577                                         t7xx_cldma_hw_irq_dis_eq(hw_info, i, MTK_TX);
 578                                         t7xx_cldma_hw_irq_dis_txrx(hw_info, i, MTK_TX);
 579                                         queue_work(md_ctrl->txq[i].worker,
 580                                                    &md_ctrl->txq[i].cldma_work);
 581                                 } else {
 582                                         t7xx_cldma_txq_empty_hndl(&md_ctrl->txq[i - CLDMA_TXQ_NUM]);
 583                                 }
 584                         }
 585                 }
 586         }
 587
 588         if (l2_rx_int) {
 589                 if (l2_rx_int & (RQ_ERR_INT_BITMASK | RQ_ACTIVE_START_ERR_INT_BITMASK)) {
 590                         /* Read and clear L3 RX interrupt status */
 591                         val = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L3RISAR0);
 592                         iowrite32(val, hw_info->ap_pdn_base + REG_CLDMA_L3RISAR0);
 593                         val = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L3RISAR1);
 594                         iowrite32(val, hw_info->ap_pdn_base + REG_CLDMA_L3RISAR1);
 595                 }
 596
 597                 t7xx_cldma_hw_rx_done(hw_info, l2_rx_int);
 598                 if (l2_rx_int & (TXRX_STATUS_BITMASK | EMPTY_STATUS_BITMASK)) {
 599                         l2_rx_int |= l2_rx_int >> CLDMA_RXQ_NUM;
 600                         for_each_set_bit(i, &l2_rx_int, CLDMA_RXQ_NUM) {
 601                                 pm_runtime_get(md_ctrl->dev);
 602                                 t7xx_cldma_hw_irq_dis_eq(hw_info, i, MTK_RX);
 603                                 t7xx_cldma_hw_irq_dis_txrx(hw_info, i, MTK_RX);
 604                                 queue_work(md_ctrl->rxq[i].worker, &md_ctrl->rxq[i].cldma_work);
 605                         }
 606                 }
 607         }
 608 }
 609
 610 static bool t7xx_cldma_qs_are_active(struct cldma_ctrl *md_ctrl)
 611 {
 612         struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
 613         unsigned int tx_active;
 614         unsigned int rx_active;
 615
 616         if (!pci_device_is_present(to_pci_dev(md_ctrl->dev)))
 617                 return false;
 618
 619         tx_active = t7xx_cldma_hw_queue_status(hw_info, CLDMA_ALL_Q, MTK_TX);
 620         rx_active = t7xx_cldma_hw_queue_status(hw_info, CLDMA_ALL_Q, MTK_RX);
 621
 622         return tx_active || rx_active;
 623 }
 624
 625 /**
 626  * t7xx_cldma_stop() - Stop CLDMA.
 627  * @md_ctrl: CLDMA context structure.
 628  *
 629  * Stop TX and RX queues. Disable L1 and L2 interrupts.
 630  * Clear status registers.
 631  *
 632  * Return:
 633  * * 0          - Success.
 634  * * -ERROR     - Error code from polling cldma_queues_active.
 635  */
 636 int t7xx_cldma_stop(struct cldma_ctrl *md_ctrl)
 637 {
 638         struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
 639         bool active;
 640         int i, ret;
 641
 642         md_ctrl->rxq_active = 0;
 643         t7xx_cldma_hw_stop_all_qs(hw_info, MTK_RX);
 644         md_ctrl->txq_active = 0;
 645         t7xx_cldma_hw_stop_all_qs(hw_info, MTK_TX);
 646         md_ctrl->txq_started = 0;
 647         t7xx_cldma_disable_irq(md_ctrl);
 648         t7xx_cldma_hw_stop(hw_info, MTK_RX);
 649         t7xx_cldma_hw_stop(hw_info, MTK_TX);
 650         t7xx_cldma_hw_tx_done(hw_info, CLDMA_L2TISAR0_ALL_INT_MASK);
 651         t7xx_cldma_hw_rx_done(hw_info, CLDMA_L2RISAR0_ALL_INT_MASK);
 652
 653         if (md_ctrl->is_late_init) {
 654                 for (i = 0; i < CLDMA_TXQ_NUM; i++)
 655                         flush_work(&md_ctrl->txq[i].cldma_work);
 656
 657                 for (i = 0; i < CLDMA_RXQ_NUM; i++)
 658                         flush_work(&md_ctrl->rxq[i].cldma_work);
 659         }
 660
 661         ret = read_poll_timeout(t7xx_cldma_qs_are_active, active, !active, CHECK_Q_STOP_STEP_US,
 662                                 CHECK_Q_STOP_TIMEOUT_US, true, md_ctrl);
 663         if (ret)
 664                 dev_err(md_ctrl->dev, "Could not stop CLDMA%d queues", md_ctrl->hif_id);
 665
 666         return ret;
 667 }
 668
 669 static void t7xx_cldma_late_release(struct cldma_ctrl *md_ctrl)
 670 {
 671         int i;
 672
 673         if (!md_ctrl->is_late_init)
 674                 return;
 675
 676         for (i = 0; i < CLDMA_TXQ_NUM; i++)
 677                 t7xx_cldma_ring_free(md_ctrl, &md_ctrl->tx_ring[i], DMA_TO_DEVICE);
 678
 679         for (i = 0; i < CLDMA_RXQ_NUM; i++)
 680                 t7xx_cldma_ring_free(md_ctrl, &md_ctrl->rx_ring[i], DMA_FROM_DEVICE);
 681
 682         dma_pool_destroy(md_ctrl->gpd_dmapool);
 683         md_ctrl->gpd_dmapool = NULL;
 684         md_ctrl->is_late_init = false;
 685 }
 686
 687 void t7xx_cldma_reset(struct cldma_ctrl *md_ctrl)
 688 {
 689         unsigned long flags;
 690         int i;
 691
 692         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 693         md_ctrl->txq_active = 0;
 694         md_ctrl->rxq_active = 0;
 695         t7xx_cldma_disable_irq(md_ctrl);
 696         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 697
 698         for (i = 0; i < CLDMA_TXQ_NUM; i++) {
 699                 cancel_work_sync(&md_ctrl->txq[i].cldma_work);
 700
 701                 spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 702                 md_cd_queue_struct_reset(&md_ctrl->txq[i], md_ctrl, MTK_TX, i);
 703                 spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 704         }
 705
 706         for (i = 0; i < CLDMA_RXQ_NUM; i++) {
 707                 cancel_work_sync(&md_ctrl->rxq[i].cldma_work);
 708
 709                 spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 710                 md_cd_queue_struct_reset(&md_ctrl->rxq[i], md_ctrl, MTK_RX, i);
 711                 spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 712         }
 713
 714         t7xx_cldma_late_release(md_ctrl);
 715 }
 716
 717 /**
 718  * t7xx_cldma_start() - Start CLDMA.
 719  * @md_ctrl: CLDMA context structure.
 720  *
 721  * Set TX/RX start address.
 722  * Start all RX queues and enable L2 interrupt.
 723  */
 724 void t7xx_cldma_start(struct cldma_ctrl *md_ctrl)
 725 {
 726         unsigned long flags;
 727
 728         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 729         if (md_ctrl->is_late_init) {
 730                 struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
 731                 int i;
 732
 733                 t7xx_cldma_enable_irq(md_ctrl);
 734
 735                 for (i = 0; i < CLDMA_TXQ_NUM; i++) {
 736                         if (md_ctrl->txq[i].tr_done)
 737                                 t7xx_cldma_hw_set_start_addr(hw_info, i,
 738                                                              md_ctrl->txq[i].tr_done->gpd_addr,
 739                                                              MTK_TX);
 740                 }
 741
 742                 for (i = 0; i < CLDMA_RXQ_NUM; i++) {
 743                         if (md_ctrl->rxq[i].tr_done)
 744                                 t7xx_cldma_hw_set_start_addr(hw_info, i,
 745                                                              md_ctrl->rxq[i].tr_done->gpd_addr,
 746                                                              MTK_RX);
 747                 }
 748
 749                 /* Enable L2 interrupt */
 750                 t7xx_cldma_hw_start_queue(hw_info, CLDMA_ALL_Q, MTK_RX);
 751                 t7xx_cldma_hw_start(hw_info);
 752                 md_ctrl->txq_started = 0;
 753                 md_ctrl->txq_active |= TXRX_STATUS_BITMASK;
 754                 md_ctrl->rxq_active |= TXRX_STATUS_BITMASK;
 755         }
 756         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 757 }
 758
 759 static void t7xx_cldma_clear_txq(struct cldma_ctrl *md_ctrl, int qnum)
 760 {
 761         struct cldma_queue *txq = &md_ctrl->txq[qnum];
 762         struct cldma_request *req;
 763         struct cldma_gpd *gpd;
 764         unsigned long flags;
 765
 766         spin_lock_irqsave(&txq->ring_lock, flags);
 767         t7xx_cldma_q_reset(txq);
 768         list_for_each_entry(req, &txq->tr_ring->gpd_ring, entry) {
 769                 gpd = req->gpd;
 770                 gpd->flags &= ~GPD_FLAGS_HWO;
 771                 t7xx_cldma_gpd_set_data_ptr(gpd, 0);
 772                 gpd->data_buff_len = 0;
 773                 dev_kfree_skb_any(req->skb);
 774                 req->skb = NULL;
 775         }
 776         spin_unlock_irqrestore(&txq->ring_lock, flags);
 777 }
 778
 779 static int t7xx_cldma_clear_rxq(struct cldma_ctrl *md_ctrl, int qnum)
 780 {
 781         struct cldma_queue *rxq = &md_ctrl->rxq[qnum];
 782         struct cldma_request *req;
 783         struct cldma_gpd *gpd;
 784         unsigned long flags;
 785         int ret = 0;
 786
 787         spin_lock_irqsave(&rxq->ring_lock, flags);
 788         t7xx_cldma_q_reset(rxq);
 789         list_for_each_entry(req, &rxq->tr_ring->gpd_ring, entry) {
 790                 gpd = req->gpd;
 791                 gpd->flags = GPD_FLAGS_IOC | GPD_FLAGS_HWO;
 792                 gpd->data_buff_len = 0;
 793
 794                 if (req->skb) {
 795                         req->skb->len = 0;
 796                         skb_reset_tail_pointer(req->skb);
 797                 }
 798         }
 799
 800         list_for_each_entry(req, &rxq->tr_ring->gpd_ring, entry) {
 801                 if (req->skb)
 802                         continue;
 803
 804                 ret = t7xx_cldma_alloc_and_map_skb(md_ctrl, req, rxq->tr_ring->pkt_size, GFP_ATOMIC);
 805                 if (ret)
 806                         break;
 807
 808                 t7xx_cldma_gpd_set_data_ptr(req->gpd, req->mapped_buff);
 809         }
 810         spin_unlock_irqrestore(&rxq->ring_lock, flags);
 811
 812         return ret;
 813 }
 814
 815 void t7xx_cldma_clear_all_qs(struct cldma_ctrl *md_ctrl, enum mtk_txrx tx_rx)
 816 {
 817         int i;
 818
 819         if (tx_rx == MTK_TX) {
 820                 for (i = 0; i < CLDMA_TXQ_NUM; i++)
 821                         t7xx_cldma_clear_txq(md_ctrl, i);
 822         } else {
 823                 for (i = 0; i < CLDMA_RXQ_NUM; i++)
 824                         t7xx_cldma_clear_rxq(md_ctrl, i);
 825         }
 826 }
 827
 828 void t7xx_cldma_stop_all_qs(struct cldma_ctrl *md_ctrl, enum mtk_txrx tx_rx)
 829 {
 830         struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
 831         unsigned long flags;
 832
 833         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 834         t7xx_cldma_hw_irq_dis_eq(hw_info, CLDMA_ALL_Q, tx_rx);
 835         t7xx_cldma_hw_irq_dis_txrx(hw_info, CLDMA_ALL_Q, tx_rx);
 836         if (tx_rx == MTK_RX)
 837                 md_ctrl->rxq_active &= ~TXRX_STATUS_BITMASK;
 838         else
 839                 md_ctrl->txq_active &= ~TXRX_STATUS_BITMASK;
 840         t7xx_cldma_hw_stop_all_qs(hw_info, tx_rx);
 841         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 842 }
 843
 844 static int t7xx_cldma_gpd_handle_tx_request(struct cldma_queue *queue, struct cldma_request *tx_req,
 845                                             struct sk_buff *skb)
 846 {
 847         struct cldma_ctrl *md_ctrl = queue->md_ctrl;
 848         struct cldma_gpd *gpd = tx_req->gpd;
 849         unsigned long flags;
 850
 851         /* Update GPD */
 852         tx_req->mapped_buff = dma_map_single(md_ctrl->dev, skb->data, skb->len, DMA_TO_DEVICE);
 853
 854         if (dma_mapping_error(md_ctrl->dev, tx_req->mapped_buff)) {
 855                 dev_err(md_ctrl->dev, "DMA mapping failed\n");
 856                 return -ENOMEM;
 857         }
 858
 859         t7xx_cldma_gpd_set_data_ptr(gpd, tx_req->mapped_buff);
 860         gpd->data_buff_len = cpu_to_le16(skb->len);
 861
 862         /* This lock must cover TGPD setting, as even without a resume operation,
 863          * CLDMA can send next HWO=1 if last TGPD just finished.
 864          */
 865         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 866         if (md_ctrl->txq_active & BIT(queue->index))
 867                 gpd->flags |= GPD_FLAGS_HWO;
 868
 869         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 870
 871         tx_req->skb = skb;
 872         return 0;
 873 }
 874
 875 /* Called with cldma_lock */
 876 static void t7xx_cldma_hw_start_send(struct cldma_ctrl *md_ctrl, int qno,
 877                                      struct cldma_request *prev_req)
 878 {
 879         struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
 880
 881         /* Check whether the device was powered off (CLDMA start address is not set) */
 882         if (!t7xx_cldma_tx_addr_is_set(hw_info, qno)) {
 883                 t7xx_cldma_hw_init(hw_info);
 884                 t7xx_cldma_hw_set_start_addr(hw_info, qno, prev_req->gpd_addr, MTK_TX);
 885                 md_ctrl->txq_started &= ~BIT(qno);
 886         }
 887
 888         if (!t7xx_cldma_hw_queue_status(hw_info, qno, MTK_TX)) {
 889                 if (md_ctrl->txq_started & BIT(qno))
 890                         t7xx_cldma_hw_resume_queue(hw_info, qno, MTK_TX);
 891                 else
 892                         t7xx_cldma_hw_start_queue(hw_info, qno, MTK_TX);
 893
 894                 md_ctrl->txq_started |= BIT(qno);
 895         }
 896 }
 897
 898 /**
 899  * t7xx_cldma_set_recv_skb() - Set the callback to handle RX packets.
 900  * @md_ctrl: CLDMA context structure.
 901  * @recv_skb: Receiving skb callback.
 902  */
 903 void t7xx_cldma_set_recv_skb(struct cldma_ctrl *md_ctrl,
 904                              int (*recv_skb)(struct cldma_queue *queue, struct sk_buff *skb))
 905 {
 906         md_ctrl->recv_skb = recv_skb;
 907 }
 908
 909 /**
 910  * t7xx_cldma_send_skb() - Send control data to modem.
 911  * @md_ctrl: CLDMA context structure.
 912  * @qno: Queue number.
 913  * @skb: Socket buffer.
 914  *
 915  * Return:
 916  * * 0          - Success.
 917  * * -ENOMEM    - Allocation failure.
 918  * * -EINVAL    - Invalid queue request.
 919  * * -EIO       - Queue is not active.
 920  * * -ETIMEDOUT - Timeout waiting for the device to wake up.
 921  */
 922 int t7xx_cldma_send_skb(struct cldma_ctrl *md_ctrl, int qno, struct sk_buff *skb)
 923 {
 924         struct cldma_request *tx_req;
 925         struct cldma_queue *queue;
 926         unsigned long flags;
 927         int ret;
 928
 929         if (qno >= CLDMA_TXQ_NUM)
 930                 return -EINVAL;
 931
 932         ret = pm_runtime_resume_and_get(md_ctrl->dev);
 933         if (ret < 0 && ret != -EACCES)
 934                 return ret;
 935
 936         t7xx_pci_disable_sleep(md_ctrl->t7xx_dev);
 937         queue = &md_ctrl->txq[qno];
 938
 939         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 940         if (!(md_ctrl->txq_active & BIT(qno))) {
 941                 ret = -EIO;
 942                 spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 943                 goto allow_sleep;
 944         }
 945         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 946
 947         do {
 948                 spin_lock_irqsave(&queue->ring_lock, flags);
 949                 tx_req = queue->tx_next;
 950                 if (queue->budget > 0 && !tx_req->skb) {
 951                         struct list_head *gpd_ring = &queue->tr_ring->gpd_ring;
 952
 953                         queue->budget--;
 954                         t7xx_cldma_gpd_handle_tx_request(queue, tx_req, skb);
 955                         queue->tx_next = list_next_entry_circular(tx_req, gpd_ring, entry);
 956                         spin_unlock_irqrestore(&queue->ring_lock, flags);
 957
 958                         if (!t7xx_pci_sleep_disable_complete(md_ctrl->t7xx_dev)) {
 959                                 ret = -ETIMEDOUT;
 960                                 break;
 961                         }
 962
 963                         /* Protect the access to the modem for queues operations (resume/start)
 964                          * which access shared locations by all the queues.
 965                          * cldma_lock is independent of ring_lock which is per queue.
 966                          */
 967                         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 968                         t7xx_cldma_hw_start_send(md_ctrl, qno, tx_req);
 969                         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 970
 971                         break;
 972                 }
 973                 spin_unlock_irqrestore(&queue->ring_lock, flags);
 974
 975                 if (!t7xx_pci_sleep_disable_complete(md_ctrl->t7xx_dev)) {
 976                         ret = -ETIMEDOUT;
 977                         break;
 978                 }
 979
 980                 if (!t7xx_cldma_hw_queue_status(&md_ctrl->hw_info, qno, MTK_TX)) {
 981                         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 982                         t7xx_cldma_hw_resume_queue(&md_ctrl->hw_info, qno, MTK_TX);
 983                         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 984                 }
 985
 986                 ret = wait_event_interruptible_exclusive(queue->req_wq, queue->budget > 0);
 987         } while (!ret);
 988
 989 allow_sleep:
 990         t7xx_pci_enable_sleep(md_ctrl->t7xx_dev);
 991         pm_runtime_mark_last_busy(md_ctrl->dev);
 992         pm_runtime_put_autosuspend(md_ctrl->dev);
 993         return ret;
 994 }
 995
 996 static int t7xx_cldma_late_init(struct cldma_ctrl *md_ctrl)
 997 {
 998         char dma_pool_name[32];
 999         int i, j, ret;
1000
1001         if (md_ctrl->is_late_init) {
1002                 dev_err(md_ctrl->dev, "CLDMA late init was already done\n");
1003                 return -EALREADY;
1004         }
1005
1006         snprintf(dma_pool_name, sizeof(dma_pool_name), "cldma_req_hif%d", md_ctrl->hif_id);
1007
1008         md_ctrl->gpd_dmapool = dma_pool_create(dma_pool_name, md_ctrl->dev,
1009                                                sizeof(struct cldma_gpd), GPD_DMAPOOL_ALIGN, 0);
1010         if (!md_ctrl->gpd_dmapool) {
1011                 dev_err(md_ctrl->dev, "DMA pool alloc fail\n");
1012                 return -ENOMEM;
1013         }
1014
1015         for (i = 0; i < CLDMA_TXQ_NUM; i++) {
1016                 ret = t7xx_cldma_tx_ring_init(md_ctrl, &md_ctrl->tx_ring[i]);
1017                 if (ret) {
1018                         dev_err(md_ctrl->dev, "control TX ring init fail\n");
1019                         goto err_free_tx_ring;
1020                 }
1021
1022                 md_ctrl->tx_ring[i].pkt_size = CLDMA_MTU;
1023         }
1024
1025         for (j = 0; j < CLDMA_RXQ_NUM; j++) {
1026                 md_ctrl->rx_ring[j].pkt_size = CLDMA_MTU;
1027
1028                 if (j == CLDMA_RXQ_NUM - 1)
1029                         md_ctrl->rx_ring[j].pkt_size = CLDMA_JUMBO_BUFF_SZ;
1030
1031                 ret = t7xx_cldma_rx_ring_init(md_ctrl, &md_ctrl->rx_ring[j]);
1032                 if (ret) {
1033                         dev_err(md_ctrl->dev, "Control RX ring init fail\n");
1034                         goto err_free_rx_ring;
1035                 }
1036         }
1037
1038         for (i = 0; i < CLDMA_TXQ_NUM; i++)
1039                 t7xx_cldma_txq_init(&md_ctrl->txq[i]);
1040
1041         for (j = 0; j < CLDMA_RXQ_NUM; j++)
1042                 t7xx_cldma_rxq_init(&md_ctrl->rxq[j]);
1043
1044         md_ctrl->is_late_init = true;
1045         return 0;
1046
1047 err_free_rx_ring:
1048         while (j--)
1049                 t7xx_cldma_ring_free(md_ctrl, &md_ctrl->rx_ring[j], DMA_FROM_DEVICE);
1050
1051 err_free_tx_ring:
1052         while (i--)
1053                 t7xx_cldma_ring_free(md_ctrl, &md_ctrl->tx_ring[i], DMA_TO_DEVICE);
1054
1055         return ret;
1056 }
1057
1058 static void __iomem *t7xx_pcie_addr_transfer(void __iomem *addr, u32 addr_trs1, u32 phy_addr)
1059 {
1060         return addr + phy_addr - addr_trs1;
1061 }
1062
1063 static void t7xx_hw_info_init(struct cldma_ctrl *md_ctrl)
1064 {
1065         struct t7xx_addr_base *pbase = &md_ctrl->t7xx_dev->base_addr;
1066         struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
1067         u32 phy_ao_base, phy_pd_base;
1068
1069         if (md_ctrl->hif_id != CLDMA_ID_MD)
1070                 return;
1071
1072         phy_ao_base = CLDMA1_AO_BASE;
1073         phy_pd_base = CLDMA1_PD_BASE;
1074         hw_info->phy_interrupt_id = CLDMA1_INT;
1075         hw_info->hw_mode = MODE_BIT_64;
1076         hw_info->ap_ao_base = t7xx_pcie_addr_transfer(pbase->pcie_ext_reg_base,
1077                                                       pbase->pcie_dev_reg_trsl_addr, phy_ao_base);
1078         hw_info->ap_pdn_base = t7xx_pcie_addr_transfer(pbase->pcie_ext_reg_base,
1079                                                        pbase->pcie_dev_reg_trsl_addr, phy_pd_base);
1080 }
1081
1082 static int t7xx_cldma_default_recv_skb(struct cldma_queue *queue, struct sk_buff *skb)
1083 {
1084         dev_kfree_skb_any(skb);
1085         return 0;
1086 }
1087
1088 int t7xx_cldma_alloc(enum cldma_id hif_id, struct t7xx_pci_dev *t7xx_dev)
1089 {
1090         struct device *dev = &t7xx_dev->pdev->dev;
1091         struct cldma_ctrl *md_ctrl;
1092
1093         md_ctrl = devm_kzalloc(dev, sizeof(*md_ctrl), GFP_KERNEL);
1094         if (!md_ctrl)
1095                 return -ENOMEM;
1096
1097         md_ctrl->t7xx_dev = t7xx_dev;
1098         md_ctrl->dev = dev;
1099         md_ctrl->hif_id = hif_id;
1100         md_ctrl->recv_skb = t7xx_cldma_default_recv_skb;
1101         t7xx_hw_info_init(md_ctrl);
1102         t7xx_dev->md->md_ctrl[hif_id] = md_ctrl;
1103         return 0;
1104 }
1105
1106 static void t7xx_cldma_resume_early(struct t7xx_pci_dev *t7xx_dev, void *entity_param)
1107 {
1108         struct cldma_ctrl *md_ctrl = entity_param;
1109         struct t7xx_cldma_hw *hw_info;
1110         unsigned long flags;
1111         int qno_t;
1112
1113         hw_info = &md_ctrl->hw_info;
1114
1115         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
1116         t7xx_cldma_hw_restore(hw_info);
1117         for (qno_t = 0; qno_t < CLDMA_TXQ_NUM; qno_t++) {
1118                 t7xx_cldma_hw_set_start_addr(hw_info, qno_t, md_ctrl->txq[qno_t].tx_next->gpd_addr,
1119                                              MTK_TX);
1120                 t7xx_cldma_hw_set_start_addr(hw_info, qno_t, md_ctrl->rxq[qno_t].tr_done->gpd_addr,
1121                                              MTK_RX);
1122         }
1123         t7xx_cldma_enable_irq(md_ctrl);
1124         t7xx_cldma_hw_start_queue(hw_info, CLDMA_ALL_Q, MTK_RX);
1125         md_ctrl->rxq_active |= TXRX_STATUS_BITMASK;
1126         t7xx_cldma_hw_irq_en_eq(hw_info, CLDMA_ALL_Q, MTK_RX);
1127         t7xx_cldma_hw_irq_en_txrx(hw_info, CLDMA_ALL_Q, MTK_RX);
1128         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
1129 }
1130
1131 static int t7xx_cldma_resume(struct t7xx_pci_dev *t7xx_dev, void *entity_param)
1132 {
1133         struct cldma_ctrl *md_ctrl = entity_param;
1134         unsigned long flags;
1135
1136         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
1137         md_ctrl->txq_active |= TXRX_STATUS_BITMASK;
1138         t7xx_cldma_hw_irq_en_txrx(&md_ctrl->hw_info, CLDMA_ALL_Q, MTK_TX);
1139         t7xx_cldma_hw_irq_en_eq(&md_ctrl->hw_info, CLDMA_ALL_Q, MTK_TX);
1140         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
1141
1142         if (md_ctrl->hif_id == CLDMA_ID_MD)
1143                 t7xx_mhccif_mask_clr(t7xx_dev, D2H_SW_INT_MASK);
1144
1145         return 0;
1146 }
1147
1148 static void t7xx_cldma_suspend_late(struct t7xx_pci_dev *t7xx_dev, void *entity_param)
1149 {
1150         struct cldma_ctrl *md_ctrl = entity_param;
1151         struct t7xx_cldma_hw *hw_info;
1152         unsigned long flags;
1153
1154         hw_info = &md_ctrl->hw_info;
1155
1156         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
1157         t7xx_cldma_hw_irq_dis_eq(hw_info, CLDMA_ALL_Q, MTK_RX);
1158         t7xx_cldma_hw_irq_dis_txrx(hw_info, CLDMA_ALL_Q, MTK_RX);
1159         md_ctrl->rxq_active &= ~TXRX_STATUS_BITMASK;
1160         t7xx_cldma_hw_stop_all_qs(hw_info, MTK_RX);
1161         t7xx_cldma_clear_ip_busy(hw_info);
1162         t7xx_cldma_disable_irq(md_ctrl);
1163         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
1164 }
1165
1166 static int t7xx_cldma_suspend(struct t7xx_pci_dev *t7xx_dev, void *entity_param)
1167 {
1168         struct cldma_ctrl *md_ctrl = entity_param;
1169         struct t7xx_cldma_hw *hw_info;
1170         unsigned long flags;
1171
1172         if (md_ctrl->hif_id == CLDMA_ID_MD)
1173                 t7xx_mhccif_mask_set(t7xx_dev, D2H_SW_INT_MASK);
1174
1175         hw_info = &md_ctrl->hw_info;
1176
1177         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
1178         t7xx_cldma_hw_irq_dis_eq(hw_info, CLDMA_ALL_Q, MTK_TX);
1179         t7xx_cldma_hw_irq_dis_txrx(hw_info, CLDMA_ALL_Q, MTK_TX);
1180         md_ctrl->txq_active &= ~TXRX_STATUS_BITMASK;
1181         t7xx_cldma_hw_stop_all_qs(hw_info, MTK_TX);
1182         md_ctrl->txq_started = 0;
1183         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
1184
1185         return 0;
1186 }
1187
1188 static int t7xx_cldma_pm_init(struct cldma_ctrl *md_ctrl)
1189 {
1190         md_ctrl->pm_entity = kzalloc(sizeof(*md_ctrl->pm_entity), GFP_KERNEL);
1191         if (!md_ctrl->pm_entity)
1192                 return -ENOMEM;
1193
1194         md_ctrl->pm_entity->entity_param = md_ctrl;
1195
1196         if (md_ctrl->hif_id == CLDMA_ID_MD)
1197                 md_ctrl->pm_entity->id = PM_ENTITY_ID_CTRL1;
1198         else
1199                 md_ctrl->pm_entity->id = PM_ENTITY_ID_CTRL2;
1200
1201         md_ctrl->pm_entity->suspend = t7xx_cldma_suspend;
1202         md_ctrl->pm_entity->suspend_late = t7xx_cldma_suspend_late;
1203         md_ctrl->pm_entity->resume = t7xx_cldma_resume;
1204         md_ctrl->pm_entity->resume_early = t7xx_cldma_resume_early;
1205
1206         return t7xx_pci_pm_entity_register(md_ctrl->t7xx_dev, md_ctrl->pm_entity);
1207 }
1208
1209 static int t7xx_cldma_pm_uninit(struct cldma_ctrl *md_ctrl)
1210 {
1211         if (!md_ctrl->pm_entity)
1212                 return -EINVAL;
1213
1214         t7xx_pci_pm_entity_unregister(md_ctrl->t7xx_dev, md_ctrl->pm_entity);
1215         kfree(md_ctrl->pm_entity);
1216         md_ctrl->pm_entity = NULL;
1217         return 0;
1218 }
1219
1220 void t7xx_cldma_hif_hw_init(struct cldma_ctrl *md_ctrl)
1221 {
1222         struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
1223         unsigned long flags;
1224
1225         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
1226         t7xx_cldma_hw_stop(hw_info, MTK_TX);
1227         t7xx_cldma_hw_stop(hw_info, MTK_RX);
1228         t7xx_cldma_hw_rx_done(hw_info, EMPTY_STATUS_BITMASK | TXRX_STATUS_BITMASK);
1229         t7xx_cldma_hw_tx_done(hw_info, EMPTY_STATUS_BITMASK | TXRX_STATUS_BITMASK);
1230         t7xx_cldma_hw_init(hw_info);
1231         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
1232 }
1233
1234 static irqreturn_t t7xx_cldma_isr_handler(int irq, void *data)
1235 {
1236         struct cldma_ctrl *md_ctrl = data;
1237         u32 interrupt;
1238
1239         interrupt = md_ctrl->hw_info.phy_interrupt_id;
1240         t7xx_pcie_mac_clear_int(md_ctrl->t7xx_dev, interrupt);
1241         t7xx_cldma_irq_work_cb(md_ctrl);
1242         t7xx_pcie_mac_clear_int_status(md_ctrl->t7xx_dev, interrupt);
1243         t7xx_pcie_mac_set_int(md_ctrl->t7xx_dev, interrupt);
1244         return IRQ_HANDLED;
1245 }
1246
1247 static void t7xx_cldma_destroy_wqs(struct cldma_ctrl *md_ctrl)
1248 {
1249         int i;
1250
1251         for (i = 0; i < CLDMA_TXQ_NUM; i++) {
1252                 if (md_ctrl->txq[i].worker) {
1253                         destroy_workqueue(md_ctrl->txq[i].worker);
1254                         md_ctrl->txq[i].worker = NULL;
1255                 }
1256         }
1257
1258         for (i = 0; i < CLDMA_RXQ_NUM; i++) {
1259                 if (md_ctrl->rxq[i].worker) {
1260                         destroy_workqueue(md_ctrl->rxq[i].worker);
1261                         md_ctrl->rxq[i].worker = NULL;
1262                 }
1263         }
1264 }
1265
1266 /**
1267  * t7xx_cldma_init() - Initialize CLDMA.
1268  * @md_ctrl: CLDMA context structure.
1269  *
1270  * Allocate and initialize device power management entity.
1271  * Initialize HIF TX/RX queue structure.
1272  * Register CLDMA callback ISR with PCIe driver.
1273  *
1274  * Return:
1275  * * 0          - Success.
1276  * * -ERROR     - Error code from failure sub-initializations.
1277  */
1278 int t7xx_cldma_init(struct cldma_ctrl *md_ctrl)
1279 {
1280         struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
1281         int ret, i;
1282
1283         md_ctrl->txq_active = 0;
1284         md_ctrl->rxq_active = 0;
1285         md_ctrl->is_late_init = false;
1286
1287         ret = t7xx_cldma_pm_init(md_ctrl);
1288         if (ret)
1289                 return ret;
1290
1291         spin_lock_init(&md_ctrl->cldma_lock);
1292
1293         for (i = 0; i < CLDMA_TXQ_NUM; i++) {
1294                 md_cd_queue_struct_init(&md_ctrl->txq[i], md_ctrl, MTK_TX, i);
1295                 md_ctrl->txq[i].worker =
1296                         alloc_workqueue("md_hif%d_tx%d_worker",
1297                                         WQ_UNBOUND | WQ_MEM_RECLAIM | (i ? 0 : WQ_HIGHPRI),
1298                                         1, md_ctrl->hif_id, i);
1299                 if (!md_ctrl->txq[i].worker)
1300                         goto err_workqueue;
1301
1302                 INIT_WORK(&md_ctrl->txq[i].cldma_work, t7xx_cldma_tx_done);
1303         }
1304
1305         for (i = 0; i < CLDMA_RXQ_NUM; i++) {
1306                 md_cd_queue_struct_init(&md_ctrl->rxq[i], md_ctrl, MTK_RX, i);
1307                 INIT_WORK(&md_ctrl->rxq[i].cldma_work, t7xx_cldma_rx_done);
1308
1309                 md_ctrl->rxq[i].worker = alloc_workqueue("md_hif%d_rx%d_worker",
1310                                                          WQ_UNBOUND | WQ_MEM_RECLAIM,
1311                                                          1, md_ctrl->hif_id, i);
1312                 if (!md_ctrl->rxq[i].worker)
1313                         goto err_workqueue;
1314         }
1315
1316         t7xx_pcie_mac_clear_int(md_ctrl->t7xx_dev, hw_info->phy_interrupt_id);
1317         md_ctrl->t7xx_dev->intr_handler[hw_info->phy_interrupt_id] = t7xx_cldma_isr_handler;
1318         md_ctrl->t7xx_dev->intr_thread[hw_info->phy_interrupt_id] = NULL;
1319         md_ctrl->t7xx_dev->callback_param[hw_info->phy_interrupt_id] = md_ctrl;
1320         t7xx_pcie_mac_clear_int_status(md_ctrl->t7xx_dev, hw_info->phy_interrupt_id);
1321         return 0;
1322
1323 err_workqueue:
1324         t7xx_cldma_destroy_wqs(md_ctrl);
1325         t7xx_cldma_pm_uninit(md_ctrl);
1326         return -ENOMEM;
1327 }
1328
1329 void t7xx_cldma_switch_cfg(struct cldma_ctrl *md_ctrl)
1330 {
1331         t7xx_cldma_late_release(md_ctrl);
1332         t7xx_cldma_late_init(md_ctrl);
1333 }
1334
1335 void t7xx_cldma_exit(struct cldma_ctrl *md_ctrl)
1336 {
1337         t7xx_cldma_stop(md_ctrl);
1338         t7xx_cldma_late_release(md_ctrl);
1339         t7xx_cldma_destroy_wqs(md_ctrl);
1340         t7xx_cldma_pm_uninit(md_ctrl);
1341 }