drivers/net/wireless/mediatek/mt76/dma.c

   1 // SPDX-License-Identifier: ISC
   2 /*
   3  * Copyright (C) 2016 Felix Fietkau <[email protected]>
   4  */
   5
   6 #include <linux/dma-mapping.h>
   7 #include "mt76.h"
   8 #include "dma.h"
   9
  10 static struct mt76_txwi_cache *
  11 mt76_alloc_txwi(struct mt76_dev *dev)
  12 {
  13         struct mt76_txwi_cache *t;
  14         dma_addr_t addr;
  15         u8 *txwi;
  16         int size;
  17
  18         size = L1_CACHE_ALIGN(dev->drv->txwi_size + sizeof(*t));
  19         txwi = devm_kzalloc(dev->dev, size, GFP_ATOMIC);
  20         if (!txwi)
  21                 return NULL;
  22
  23         addr = dma_map_single(dev->dev, txwi, dev->drv->txwi_size,
  24                               DMA_TO_DEVICE);
  25         t = (struct mt76_txwi_cache *)(txwi + dev->drv->txwi_size);
  26         t->dma_addr = addr;
  27
  28         return t;
  29 }
  30
  31 static struct mt76_txwi_cache *
  32 __mt76_get_txwi(struct mt76_dev *dev)
  33 {
  34         struct mt76_txwi_cache *t = NULL;
  35
  36         spin_lock(&dev->lock);
  37         if (!list_empty(&dev->txwi_cache)) {
  38                 t = list_first_entry(&dev->txwi_cache, struct mt76_txwi_cache,
  39                                      list);
  40                 list_del(&t->list);
  41         }
  42         spin_unlock(&dev->lock);
  43
  44         return t;
  45 }
  46
  47 static struct mt76_txwi_cache *
  48 mt76_get_txwi(struct mt76_dev *dev)
  49 {
  50         struct mt76_txwi_cache *t = __mt76_get_txwi(dev);
  51
  52         if (t)
  53                 return t;
  54
  55         return mt76_alloc_txwi(dev);
  56 }
  57
  58 void
  59 mt76_put_txwi(struct mt76_dev *dev, struct mt76_txwi_cache *t)
  60 {
  61         if (!t)
  62                 return;
  63
  64         spin_lock(&dev->lock);
  65         list_add(&t->list, &dev->txwi_cache);
  66         spin_unlock(&dev->lock);
  67 }
  68 EXPORT_SYMBOL_GPL(mt76_put_txwi);
  69
  70 static void
  71 mt76_free_pending_txwi(struct mt76_dev *dev)
  72 {
  73         struct mt76_txwi_cache *t;
  74
  75         local_bh_disable();
  76         while ((t = __mt76_get_txwi(dev)) != NULL)
  77                 dma_unmap_single(dev->dev, t->dma_addr, dev->drv->txwi_size,
  78                                  DMA_TO_DEVICE);
  79         local_bh_enable();
  80 }
  81
  82 static void
  83 mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q)
  84 {
  85         writel(q->desc_dma, &q->regs->desc_base);
  86         writel(q->ndesc, &q->regs->ring_size);
  87         q->head = readl(&q->regs->dma_idx);
  88         q->tail = q->head;
  89 }
  90
  91 static void
  92 mt76_dma_queue_reset(struct mt76_dev *dev, struct mt76_queue *q)
  93 {
  94         int i;
  95
  96         if (!q)
  97                 return;
  98
  99         /* clear descriptors */
 100         for (i = 0; i < q->ndesc; i++)
 101                 q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE);
 102
 103         writel(0, &q->regs->cpu_idx);
 104         writel(0, &q->regs->dma_idx);
 105         mt76_dma_sync_idx(dev, q);
 106 }
 107
 108 static int
 109 mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q,
 110                      int idx, int n_desc, int bufsize,
 111                      u32 ring_base)
 112 {
 113         int size;
 114
 115         spin_lock_init(&q->lock);
 116         spin_lock_init(&q->cleanup_lock);
 117
 118         q->regs = dev->mmio.regs + ring_base + idx * MT_RING_SIZE;
 119         q->ndesc = n_desc;
 120         q->buf_size = bufsize;
 121         q->hw_idx = idx;
 122
 123         size = q->ndesc * sizeof(struct mt76_desc);
 124         q->desc = dmam_alloc_coherent(dev->dev, size, &q->desc_dma, GFP_KERNEL);
 125         if (!q->desc)
 126                 return -ENOMEM;
 127
 128         size = q->ndesc * sizeof(*q->entry);
 129         q->entry = devm_kzalloc(dev->dev, size, GFP_KERNEL);
 130         if (!q->entry)
 131                 return -ENOMEM;
 132
 133         mt76_dma_queue_reset(dev, q);
 134
 135         return 0;
 136 }
 137
 138 static int
 139 mt76_dma_add_buf(struct mt76_dev *dev, struct mt76_queue *q,
 140                  struct mt76_queue_buf *buf, int nbufs, u32 info,
 141                  struct sk_buff *skb, void *txwi)
 142 {
 143         struct mt76_queue_entry *entry;
 144         struct mt76_desc *desc;
 145         u32 ctrl;
 146         int i, idx = -1;
 147
 148         if (txwi) {
 149                 q->entry[q->head].txwi = DMA_DUMMY_DATA;
 150                 q->entry[q->head].skip_buf0 = true;
 151         }
 152
 153         for (i = 0; i < nbufs; i += 2, buf += 2) {
 154                 u32 buf0 = buf[0].addr, buf1 = 0;
 155
 156                 idx = q->head;
 157                 q->head = (q->head + 1) % q->ndesc;
 158
 159                 desc = &q->desc[idx];
 160                 entry = &q->entry[idx];
 161
 162                 if (buf[0].skip_unmap)
 163                         entry->skip_buf0 = true;
 164                 entry->skip_buf1 = i == nbufs - 1;
 165
 166                 entry->dma_addr[0] = buf[0].addr;
 167                 entry->dma_len[0] = buf[0].len;
 168
 169                 ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len);
 170                 if (i < nbufs - 1) {
 171                         entry->dma_addr[1] = buf[1].addr;
 172                         entry->dma_len[1] = buf[1].len;
 173                         buf1 = buf[1].addr;
 174                         ctrl |= FIELD_PREP(MT_DMA_CTL_SD_LEN1, buf[1].len);
 175                         if (buf[1].skip_unmap)
 176                                 entry->skip_buf1 = true;
 177                 }
 178
 179                 if (i == nbufs - 1)
 180                         ctrl |= MT_DMA_CTL_LAST_SEC0;
 181                 else if (i == nbufs - 2)
 182                         ctrl |= MT_DMA_CTL_LAST_SEC1;
 183
 184                 WRITE_ONCE(desc->buf0, cpu_to_le32(buf0));
 185                 WRITE_ONCE(desc->buf1, cpu_to_le32(buf1));
 186                 WRITE_ONCE(desc->info, cpu_to_le32(info));
 187                 WRITE_ONCE(desc->ctrl, cpu_to_le32(ctrl));
 188
 189                 q->queued++;
 190         }
 191
 192         q->entry[idx].txwi = txwi;
 193         q->entry[idx].skb = skb;
 194
 195         return idx;
 196 }
 197
 198 static void
 199 mt76_dma_tx_cleanup_idx(struct mt76_dev *dev, struct mt76_queue *q, int idx,
 200                         struct mt76_queue_entry *prev_e)
 201 {
 202         struct mt76_queue_entry *e = &q->entry[idx];
 203
 204         if (!e->skip_buf0)
 205                 dma_unmap_single(dev->dev, e->dma_addr[0], e->dma_len[0],
 206                                  DMA_TO_DEVICE);
 207
 208         if (!e->skip_buf1)
 209                 dma_unmap_single(dev->dev, e->dma_addr[1], e->dma_len[1],
 210                                  DMA_TO_DEVICE);
 211
 212         if (e->txwi == DMA_DUMMY_DATA)
 213                 e->txwi = NULL;
 214
 215         if (e->skb == DMA_DUMMY_DATA)
 216                 e->skb = NULL;
 217
 218         *prev_e = *e;
 219         memset(e, 0, sizeof(*e));
 220 }
 221
 222 static void
 223 mt76_dma_kick_queue(struct mt76_dev *dev, struct mt76_queue *q)
 224 {
 225         wmb();
 226         writel(q->head, &q->regs->cpu_idx);
 227 }
 228
 229 static void
 230 mt76_dma_tx_cleanup(struct mt76_dev *dev, struct mt76_queue *q, bool flush)
 231 {
 232         struct mt76_queue_entry entry;
 233         int last;
 234
 235         if (!q)
 236                 return;
 237
 238         spin_lock_bh(&q->cleanup_lock);
 239         if (flush)
 240                 last = -1;
 241         else
 242                 last = readl(&q->regs->dma_idx);
 243
 244         while (q->queued > 0 && q->tail != last) {
 245                 mt76_dma_tx_cleanup_idx(dev, q, q->tail, &entry);
 246                 mt76_queue_tx_complete(dev, q, &entry);
 247
 248                 if (entry.txwi) {
 249                         if (!(dev->drv->drv_flags & MT_DRV_TXWI_NO_FREE))
 250                                 mt76_put_txwi(dev, entry.txwi);
 251                 }
 252
 253                 if (!flush && q->tail == last)
 254                         last = readl(&q->regs->dma_idx);
 255
 256         }
 257         spin_unlock_bh(&q->cleanup_lock);
 258
 259         if (flush) {
 260                 spin_lock_bh(&q->lock);
 261                 mt76_dma_sync_idx(dev, q);
 262                 mt76_dma_kick_queue(dev, q);
 263                 spin_unlock_bh(&q->lock);
 264         }
 265
 266         if (!q->queued)
 267                 wake_up(&dev->tx_wait);
 268 }
 269
 270 static void *
 271 mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx,
 272                  int *len, u32 *info, bool *more)
 273 {
 274         struct mt76_queue_entry *e = &q->entry[idx];
 275         struct mt76_desc *desc = &q->desc[idx];
 276         dma_addr_t buf_addr;
 277         void *buf = e->buf;
 278         int buf_len = SKB_WITH_OVERHEAD(q->buf_size);
 279
 280         buf_addr = e->dma_addr[0];
 281         if (len) {
 282                 u32 ctl = le32_to_cpu(READ_ONCE(desc->ctrl));
 283                 *len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctl);
 284                 *more = !(ctl & MT_DMA_CTL_LAST_SEC0);
 285         }
 286
 287         if (info)
 288                 *info = le32_to_cpu(desc->info);
 289
 290         dma_unmap_single(dev->dev, buf_addr, buf_len, DMA_FROM_DEVICE);
 291         e->buf = NULL;
 292
 293         return buf;
 294 }
 295
 296 static void *
 297 mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush,
 298                  int *len, u32 *info, bool *more)
 299 {
 300         int idx = q->tail;
 301
 302         *more = false;
 303         if (!q->queued)
 304                 return NULL;
 305
 306         if (flush)
 307                 q->desc[idx].ctrl |= cpu_to_le32(MT_DMA_CTL_DMA_DONE);
 308         else if (!(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_DMA_DONE)))
 309                 return NULL;
 310
 311         q->tail = (q->tail + 1) % q->ndesc;
 312         q->queued--;
 313
 314         return mt76_dma_get_buf(dev, q, idx, len, info, more);
 315 }
 316
 317 static int
 318 mt76_dma_tx_queue_skb_raw(struct mt76_dev *dev, struct mt76_queue *q,
 319                           struct sk_buff *skb, u32 tx_info)
 320 {
 321         struct mt76_queue_buf buf = {};
 322         dma_addr_t addr;
 323
 324         if (q->queued + 1 >= q->ndesc - 1)
 325                 goto error;
 326
 327         addr = dma_map_single(dev->dev, skb->data, skb->len,
 328                               DMA_TO_DEVICE);
 329         if (unlikely(dma_mapping_error(dev->dev, addr)))
 330                 goto error;
 331
 332         buf.addr = addr;
 333         buf.len = skb->len;
 334
 335         spin_lock_bh(&q->lock);
 336         mt76_dma_add_buf(dev, q, &buf, 1, tx_info, skb, NULL);
 337         mt76_dma_kick_queue(dev, q);
 338         spin_unlock_bh(&q->lock);
 339
 340         return 0;
 341
 342 error:
 343         dev_kfree_skb(skb);
 344         return -ENOMEM;
 345 }
 346
 347 static int
 348 mt76_dma_tx_queue_skb(struct mt76_dev *dev, struct mt76_queue *q,
 349                       struct sk_buff *skb, struct mt76_wcid *wcid,
 350                       struct ieee80211_sta *sta)
 351 {
 352         struct mt76_tx_info tx_info = {
 353                 .skb = skb,
 354         };
 355         struct ieee80211_hw *hw;
 356         int len, n = 0, ret = -ENOMEM;
 357         struct mt76_txwi_cache *t;
 358         struct sk_buff *iter;
 359         dma_addr_t addr;
 360         u8 *txwi;
 361
 362         t = mt76_get_txwi(dev);
 363         if (!t) {
 364                 hw = mt76_tx_status_get_hw(dev, skb);
 365                 ieee80211_free_txskb(hw, skb);
 366                 return -ENOMEM;
 367         }
 368         txwi = mt76_get_txwi_ptr(dev, t);
 369
 370         skb->prev = skb->next = NULL;
 371         if (dev->drv->drv_flags & MT_DRV_TX_ALIGNED4_SKBS)
 372                 mt76_insert_hdr_pad(skb);
 373
 374         len = skb_headlen(skb);
 375         addr = dma_map_single(dev->dev, skb->data, len, DMA_TO_DEVICE);
 376         if (unlikely(dma_mapping_error(dev->dev, addr)))
 377                 goto free;
 378
 379         tx_info.buf[n].addr = t->dma_addr;
 380         tx_info.buf[n++].len = dev->drv->txwi_size;
 381         tx_info.buf[n].addr = addr;
 382         tx_info.buf[n++].len = len;
 383
 384         skb_walk_frags(skb, iter) {
 385                 if (n == ARRAY_SIZE(tx_info.buf))
 386                         goto unmap;
 387
 388                 addr = dma_map_single(dev->dev, iter->data, iter->len,
 389                                       DMA_TO_DEVICE);
 390                 if (unlikely(dma_mapping_error(dev->dev, addr)))
 391                         goto unmap;
 392
 393                 tx_info.buf[n].addr = addr;
 394                 tx_info.buf[n++].len = iter->len;
 395         }
 396         tx_info.nbuf = n;
 397
 398         if (q->queued + (tx_info.nbuf + 1) / 2 >= q->ndesc - 1) {
 399                 ret = -ENOMEM;
 400                 goto unmap;
 401         }
 402
 403         dma_sync_single_for_cpu(dev->dev, t->dma_addr, dev->drv->txwi_size,
 404                                 DMA_TO_DEVICE);
 405         ret = dev->drv->tx_prepare_skb(dev, txwi, q->qid, wcid, sta, &tx_info);
 406         dma_sync_single_for_device(dev->dev, t->dma_addr, dev->drv->txwi_size,
 407                                    DMA_TO_DEVICE);
 408         if (ret < 0)
 409                 goto unmap;
 410
 411         return mt76_dma_add_buf(dev, q, tx_info.buf, tx_info.nbuf,
 412                                 tx_info.info, tx_info.skb, t);
 413
 414 unmap:
 415         for (n--; n > 0; n--)
 416                 dma_unmap_single(dev->dev, tx_info.buf[n].addr,
 417                                  tx_info.buf[n].len, DMA_TO_DEVICE);
 418
 419 free:
 420 #ifdef CONFIG_NL80211_TESTMODE
 421         /* fix tx_done accounting on queue overflow */
 422         if (mt76_is_testmode_skb(dev, skb, &hw)) {
 423                 struct mt76_phy *phy = hw->priv;
 424
 425                 if (tx_info.skb == phy->test.tx_skb)
 426                         phy->test.tx_done--;
 427         }
 428 #endif
 429
 430         dev_kfree_skb(tx_info.skb);
 431         mt76_put_txwi(dev, t);
 432         return ret;
 433 }
 434
 435 static int
 436 mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q)
 437 {
 438         dma_addr_t addr;
 439         void *buf;
 440         int frames = 0;
 441         int len = SKB_WITH_OVERHEAD(q->buf_size);
 442         int offset = q->buf_offset;
 443
 444         spin_lock_bh(&q->lock);
 445
 446         while (q->queued < q->ndesc - 1) {
 447                 struct mt76_queue_buf qbuf;
 448
 449                 buf = page_frag_alloc(&q->rx_page, q->buf_size, GFP_ATOMIC);
 450                 if (!buf)
 451                         break;
 452
 453                 addr = dma_map_single(dev->dev, buf, len, DMA_FROM_DEVICE);
 454                 if (unlikely(dma_mapping_error(dev->dev, addr))) {
 455                         skb_free_frag(buf);
 456                         break;
 457                 }
 458
 459                 qbuf.addr = addr + offset;
 460                 qbuf.len = len - offset;
 461                 mt76_dma_add_buf(dev, q, &qbuf, 1, 0, buf, NULL);
 462                 frames++;
 463         }
 464
 465         if (frames)
 466                 mt76_dma_kick_queue(dev, q);
 467
 468         spin_unlock_bh(&q->lock);
 469
 470         return frames;
 471 }
 472
 473 static void
 474 mt76_dma_rx_cleanup(struct mt76_dev *dev, struct mt76_queue *q)
 475 {
 476         struct page *page;
 477         void *buf;
 478         bool more;
 479
 480         spin_lock_bh(&q->lock);
 481         do {
 482                 buf = mt76_dma_dequeue(dev, q, true, NULL, NULL, &more);
 483                 if (!buf)
 484                         break;
 485
 486                 skb_free_frag(buf);
 487         } while (1);
 488         spin_unlock_bh(&q->lock);
 489
 490         if (!q->rx_page.va)
 491                 return;
 492
 493         page = virt_to_page(q->rx_page.va);
 494         __page_frag_cache_drain(page, q->rx_page.pagecnt_bias);
 495         memset(&q->rx_page, 0, sizeof(q->rx_page));
 496 }
 497
 498 static void
 499 mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid)
 500 {
 501         struct mt76_queue *q = &dev->q_rx[qid];
 502         int i;
 503
 504         for (i = 0; i < q->ndesc; i++)
 505                 q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE);
 506
 507         mt76_dma_rx_cleanup(dev, q);
 508         mt76_dma_sync_idx(dev, q);
 509         mt76_dma_rx_fill(dev, q);
 510
 511         if (!q->rx_head)
 512                 return;
 513
 514         dev_kfree_skb(q->rx_head);
 515         q->rx_head = NULL;
 516 }
 517
 518 static void
 519 mt76_add_fragment(struct mt76_dev *dev, struct mt76_queue *q, void *data,
 520                   int len, bool more)
 521 {
 522         struct sk_buff *skb = q->rx_head;
 523         struct skb_shared_info *shinfo = skb_shinfo(skb);
 524         int nr_frags = shinfo->nr_frags;
 525
 526         if (nr_frags < ARRAY_SIZE(shinfo->frags)) {
 527                 struct page *page = virt_to_head_page(data);
 528                 int offset = data - page_address(page) + q->buf_offset;
 529
 530                 skb_add_rx_frag(skb, nr_frags, page, offset, len, q->buf_size);
 531         } else {
 532                 skb_free_frag(data);
 533         }
 534
 535         if (more)
 536                 return;
 537
 538         q->rx_head = NULL;
 539         if (nr_frags < ARRAY_SIZE(shinfo->frags))
 540                 dev->drv->rx_skb(dev, q - dev->q_rx, skb);
 541         else
 542                 dev_kfree_skb(skb);
 543 }
 544
 545 static int
 546 mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget)
 547 {
 548         int len, data_len, done = 0;
 549         struct sk_buff *skb;
 550         unsigned char *data;
 551         bool more;
 552
 553         while (done < budget) {
 554                 u32 info;
 555
 556                 data = mt76_dma_dequeue(dev, q, false, &len, &info, &more);
 557                 if (!data)
 558                         break;
 559
 560                 if (q->rx_head)
 561                         data_len = q->buf_size;
 562                 else
 563                         data_len = SKB_WITH_OVERHEAD(q->buf_size);
 564
 565                 if (data_len < len + q->buf_offset) {
 566                         dev_kfree_skb(q->rx_head);
 567                         q->rx_head = NULL;
 568
 569                         skb_free_frag(data);
 570                         continue;
 571                 }
 572
 573                 if (q->rx_head) {
 574                         mt76_add_fragment(dev, q, data, len, more);
 575                         continue;
 576                 }
 577
 578                 skb = build_skb(data, q->buf_size);
 579                 if (!skb) {
 580                         skb_free_frag(data);
 581                         continue;
 582                 }
 583                 skb_reserve(skb, q->buf_offset);
 584
 585                 if (q == &dev->q_rx[MT_RXQ_MCU]) {
 586                         u32 *rxfce = (u32 *)skb->cb;
 587                         *rxfce = info;
 588                 }
 589
 590                 __skb_put(skb, len);
 591                 done++;
 592
 593                 if (more) {
 594                         q->rx_head = skb;
 595                         continue;
 596                 }
 597
 598                 dev->drv->rx_skb(dev, q - dev->q_rx, skb);
 599         }
 600
 601         mt76_dma_rx_fill(dev, q);
 602         return done;
 603 }
 604
 605 int mt76_dma_rx_poll(struct napi_struct *napi, int budget)
 606 {
 607         struct mt76_dev *dev;
 608         int qid, done = 0, cur;
 609
 610         dev = container_of(napi->dev, struct mt76_dev, napi_dev);
 611         qid = napi - dev->napi;
 612
 613         rcu_read_lock();
 614
 615         do {
 616                 cur = mt76_dma_rx_process(dev, &dev->q_rx[qid], budget - done);
 617                 mt76_rx_poll_complete(dev, qid, napi);
 618                 done += cur;
 619         } while (cur && done < budget);
 620
 621         rcu_read_unlock();
 622
 623         if (done < budget && napi_complete(napi))
 624                 dev->drv->rx_poll_complete(dev, qid);
 625
 626         return done;
 627 }
 628 EXPORT_SYMBOL_GPL(mt76_dma_rx_poll);
 629
 630 static int
 631 mt76_dma_init(struct mt76_dev *dev,
 632               int (*poll)(struct napi_struct *napi, int budget))
 633 {
 634         int i;
 635
 636         init_dummy_netdev(&dev->napi_dev);
 637         init_dummy_netdev(&dev->tx_napi_dev);
 638         snprintf(dev->napi_dev.name, sizeof(dev->napi_dev.name), "%s",
 639                  wiphy_name(dev->hw->wiphy));
 640         dev->napi_dev.threaded = 1;
 641
 642         mt76_for_each_q_rx(dev, i) {
 643                 netif_napi_add(&dev->napi_dev, &dev->napi[i], poll, 64);
 644                 mt76_dma_rx_fill(dev, &dev->q_rx[i]);
 645                 napi_enable(&dev->napi[i]);
 646         }
 647
 648         return 0;
 649 }
 650
 651 static const struct mt76_queue_ops mt76_dma_ops = {
 652         .init = mt76_dma_init,
 653         .alloc = mt76_dma_alloc_queue,
 654         .reset_q = mt76_dma_queue_reset,
 655         .tx_queue_skb_raw = mt76_dma_tx_queue_skb_raw,
 656         .tx_queue_skb = mt76_dma_tx_queue_skb,
 657         .tx_cleanup = mt76_dma_tx_cleanup,
 658         .rx_cleanup = mt76_dma_rx_cleanup,
 659         .rx_reset = mt76_dma_rx_reset,
 660         .kick = mt76_dma_kick_queue,
 661 };
 662
 663 void mt76_dma_attach(struct mt76_dev *dev)
 664 {
 665         dev->queue_ops = &mt76_dma_ops;
 666 }
 667 EXPORT_SYMBOL_GPL(mt76_dma_attach);
 668
 669 void mt76_dma_cleanup(struct mt76_dev *dev)
 670 {
 671         int i;
 672
 673         mt76_worker_disable(&dev->tx_worker);
 674         netif_napi_del(&dev->tx_napi);
 675
 676         for (i = 0; i < ARRAY_SIZE(dev->phy.q_tx); i++) {
 677                 mt76_dma_tx_cleanup(dev, dev->phy.q_tx[i], true);
 678                 if (dev->phy2)
 679                         mt76_dma_tx_cleanup(dev, dev->phy2->q_tx[i], true);
 680         }
 681
 682         for (i = 0; i < ARRAY_SIZE(dev->q_mcu); i++)
 683                 mt76_dma_tx_cleanup(dev, dev->q_mcu[i], true);
 684
 685         mt76_for_each_q_rx(dev, i) {
 686                 netif_napi_del(&dev->napi[i]);
 687                 mt76_dma_rx_cleanup(dev, &dev->q_rx[i]);
 688         }
 689
 690         mt76_free_pending_txwi(dev);
 691 }
 692 EXPORT_SYMBOL_GPL(mt76_dma_cleanup);