drivers/net/wireless/ath/ath11k/ce.c

   1 // SPDX-License-Identifier: BSD-3-Clause-Clear
   2 /*
   3  * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved.
   4  */
   5
   6 #include "dp_rx.h"
   7 #include "debug.h"
   8 #include "hif.h"
   9
  10 const struct ce_attr ath11k_host_ce_config_ipq8074[] = {
  11         /* CE0: host->target HTC control and raw streams */
  12         {
  13                 .flags = CE_ATTR_FLAGS,
  14                 .src_nentries = 16,
  15                 .src_sz_max = 2048,
  16                 .dest_nentries = 0,
  17         },
  18
  19         /* CE1: target->host HTT + HTC control */
  20         {
  21                 .flags = CE_ATTR_FLAGS,
  22                 .src_nentries = 0,
  23                 .src_sz_max = 2048,
  24                 .dest_nentries = 512,
  25                 .recv_cb = ath11k_htc_rx_completion_handler,
  26         },
  27
  28         /* CE2: target->host WMI */
  29         {
  30                 .flags = CE_ATTR_FLAGS,
  31                 .src_nentries = 0,
  32                 .src_sz_max = 2048,
  33                 .dest_nentries = 512,
  34                 .recv_cb = ath11k_htc_rx_completion_handler,
  35         },
  36
  37         /* CE3: host->target WMI (mac0) */
  38         {
  39                 .flags = CE_ATTR_FLAGS,
  40                 .src_nentries = 32,
  41                 .src_sz_max = 2048,
  42                 .dest_nentries = 0,
  43         },
  44
  45         /* CE4: host->target HTT */
  46         {
  47                 .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
  48                 .src_nentries = 2048,
  49                 .src_sz_max = 256,
  50                 .dest_nentries = 0,
  51         },
  52
  53         /* CE5: target->host pktlog */
  54         {
  55                 .flags = CE_ATTR_FLAGS,
  56                 .src_nentries = 0,
  57                 .src_sz_max = 2048,
  58                 .dest_nentries = 512,
  59                 .recv_cb = ath11k_dp_htt_htc_t2h_msg_handler,
  60         },
  61
  62         /* CE6: target autonomous hif_memcpy */
  63         {
  64                 .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
  65                 .src_nentries = 0,
  66                 .src_sz_max = 0,
  67                 .dest_nentries = 0,
  68         },
  69
  70         /* CE7: host->target WMI (mac1) */
  71         {
  72                 .flags = CE_ATTR_FLAGS,
  73                 .src_nentries = 32,
  74                 .src_sz_max = 2048,
  75                 .dest_nentries = 0,
  76         },
  77
  78         /* CE8: target autonomous hif_memcpy */
  79         {
  80                 .flags = CE_ATTR_FLAGS,
  81                 .src_nentries = 0,
  82                 .src_sz_max = 0,
  83                 .dest_nentries = 0,
  84         },
  85
  86         /* CE9: host->target WMI (mac2) */
  87         {
  88                 .flags = CE_ATTR_FLAGS,
  89                 .src_nentries = 32,
  90                 .src_sz_max = 2048,
  91                 .dest_nentries = 0,
  92         },
  93
  94         /* CE10: target->host HTT */
  95         {
  96                 .flags = CE_ATTR_FLAGS,
  97                 .src_nentries = 0,
  98                 .src_sz_max = 2048,
  99                 .dest_nentries = 512,
 100                 .recv_cb = ath11k_htc_rx_completion_handler,
 101         },
 102
 103         /* CE11: Not used */
 104         {
 105                 .flags = CE_ATTR_FLAGS,
 106                 .src_nentries = 0,
 107                 .src_sz_max = 0,
 108                 .dest_nentries = 0,
 109         },
 110 };
 111
 112 const struct ce_attr ath11k_host_ce_config_qca6390[] = {
 113         /* CE0: host->target HTC control and raw streams */
 114         {
 115                 .flags = CE_ATTR_FLAGS,
 116                 .src_nentries = 16,
 117                 .src_sz_max = 2048,
 118                 .dest_nentries = 0,
 119         },
 120
 121         /* CE1: target->host HTT + HTC control */
 122         {
 123                 .flags = CE_ATTR_FLAGS,
 124                 .src_nentries = 0,
 125                 .src_sz_max = 2048,
 126                 .dest_nentries = 512,
 127                 .recv_cb = ath11k_htc_rx_completion_handler,
 128         },
 129
 130         /* CE2: target->host WMI */
 131         {
 132                 .flags = CE_ATTR_FLAGS,
 133                 .src_nentries = 0,
 134                 .src_sz_max = 2048,
 135                 .dest_nentries = 512,
 136                 .recv_cb = ath11k_htc_rx_completion_handler,
 137         },
 138
 139         /* CE3: host->target WMI (mac0) */
 140         {
 141                 .flags = CE_ATTR_FLAGS,
 142                 .src_nentries = 32,
 143                 .src_sz_max = 2048,
 144                 .dest_nentries = 0,
 145         },
 146
 147         /* CE4: host->target HTT */
 148         {
 149                 .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
 150                 .src_nentries = 2048,
 151                 .src_sz_max = 256,
 152                 .dest_nentries = 0,
 153         },
 154
 155         /* CE5: target->host pktlog */
 156         {
 157                 .flags = CE_ATTR_FLAGS,
 158                 .src_nentries = 0,
 159                 .src_sz_max = 2048,
 160                 .dest_nentries = 512,
 161                 .recv_cb = ath11k_dp_htt_htc_t2h_msg_handler,
 162         },
 163
 164         /* CE6: target autonomous hif_memcpy */
 165         {
 166                 .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
 167                 .src_nentries = 0,
 168                 .src_sz_max = 0,
 169                 .dest_nentries = 0,
 170         },
 171
 172         /* CE7: host->target WMI (mac1) */
 173         {
 174                 .flags = CE_ATTR_FLAGS,
 175                 .src_nentries = 32,
 176                 .src_sz_max = 2048,
 177                 .dest_nentries = 0,
 178         },
 179
 180         /* CE8: target autonomous hif_memcpy */
 181         {
 182                 .flags = CE_ATTR_FLAGS,
 183                 .src_nentries = 0,
 184                 .src_sz_max = 0,
 185                 .dest_nentries = 0,
 186         },
 187
 188 };
 189
 190 const struct ce_attr ath11k_host_ce_config_qcn9074[] = {
 191         /* CE0: host->target HTC control and raw streams */
 192         {
 193                 .flags = CE_ATTR_FLAGS,
 194                 .src_nentries = 16,
 195                 .src_sz_max = 2048,
 196                 .dest_nentries = 0,
 197         },
 198
 199         /* CE1: target->host HTT + HTC control */
 200         {
 201                 .flags = CE_ATTR_FLAGS,
 202                 .src_nentries = 0,
 203                 .src_sz_max = 2048,
 204                 .dest_nentries = 512,
 205                 .recv_cb = ath11k_htc_rx_completion_handler,
 206         },
 207
 208         /* CE2: target->host WMI */
 209         {
 210                 .flags = CE_ATTR_FLAGS,
 211                 .src_nentries = 0,
 212                 .src_sz_max = 2048,
 213                 .dest_nentries = 32,
 214                 .recv_cb = ath11k_htc_rx_completion_handler,
 215         },
 216
 217         /* CE3: host->target WMI (mac0) */
 218         {
 219                 .flags = CE_ATTR_FLAGS,
 220                 .src_nentries = 32,
 221                 .src_sz_max = 2048,
 222                 .dest_nentries = 0,
 223         },
 224
 225         /* CE4: host->target HTT */
 226         {
 227                 .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
 228                 .src_nentries = 2048,
 229                 .src_sz_max = 256,
 230                 .dest_nentries = 0,
 231         },
 232
 233         /* CE5: target->host pktlog */
 234         {
 235                 .flags = CE_ATTR_FLAGS,
 236                 .src_nentries = 0,
 237                 .src_sz_max = 2048,
 238                 .dest_nentries = 512,
 239                 .recv_cb = ath11k_dp_htt_htc_t2h_msg_handler,
 240         },
 241 };
 242
 243 static bool ath11k_ce_need_shadow_fix(int ce_id)
 244 {
 245         /* only ce4 needs shadow workaroud*/
 246         if (ce_id == 4)
 247                 return true;
 248         return false;
 249 }
 250
 251 void ath11k_ce_stop_shadow_timers(struct ath11k_base *ab)
 252 {
 253         int i;
 254
 255         if (!ab->hw_params.supports_shadow_regs)
 256                 return;
 257
 258         for (i = 0; i < ab->hw_params.ce_count; i++)
 259                 if (ath11k_ce_need_shadow_fix(i))
 260                         ath11k_dp_shadow_stop_timer(ab, &ab->ce.hp_timer[i]);
 261 }
 262
 263 static int ath11k_ce_rx_buf_enqueue_pipe(struct ath11k_ce_pipe *pipe,
 264                                          struct sk_buff *skb, dma_addr_t paddr)
 265 {
 266         struct ath11k_base *ab = pipe->ab;
 267         struct ath11k_ce_ring *ring = pipe->dest_ring;
 268         struct hal_srng *srng;
 269         unsigned int write_index;
 270         unsigned int nentries_mask = ring->nentries_mask;
 271         u32 *desc;
 272         int ret;
 273
 274         lockdep_assert_held(&ab->ce.ce_lock);
 275
 276         write_index = ring->write_index;
 277
 278         srng = &ab->hal.srng_list[ring->hal_ring_id];
 279
 280         spin_lock_bh(&srng->lock);
 281
 282         ath11k_hal_srng_access_begin(ab, srng);
 283
 284         if (unlikely(ath11k_hal_srng_src_num_free(ab, srng, false) < 1)) {
 285                 ret = -ENOSPC;
 286                 goto exit;
 287         }
 288
 289         desc = ath11k_hal_srng_src_get_next_entry(ab, srng);
 290         if (!desc) {
 291                 ret = -ENOSPC;
 292                 goto exit;
 293         }
 294
 295         ath11k_hal_ce_dst_set_desc(desc, paddr);
 296
 297         ring->skb[write_index] = skb;
 298         write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
 299         ring->write_index = write_index;
 300
 301         pipe->rx_buf_needed--;
 302
 303         ret = 0;
 304 exit:
 305         ath11k_hal_srng_access_end(ab, srng);
 306
 307         spin_unlock_bh(&srng->lock);
 308
 309         return ret;
 310 }
 311
 312 static int ath11k_ce_rx_post_pipe(struct ath11k_ce_pipe *pipe)
 313 {
 314         struct ath11k_base *ab = pipe->ab;
 315         struct sk_buff *skb;
 316         dma_addr_t paddr;
 317         int ret = 0;
 318
 319         if (!(pipe->dest_ring || pipe->status_ring))
 320                 return 0;
 321
 322         spin_lock_bh(&ab->ce.ce_lock);
 323         while (pipe->rx_buf_needed) {
 324                 skb = dev_alloc_skb(pipe->buf_sz);
 325                 if (!skb) {
 326                         ret = -ENOMEM;
 327                         goto exit;
 328                 }
 329
 330                 WARN_ON_ONCE(!IS_ALIGNED((unsigned long)skb->data, 4));
 331
 332                 paddr = dma_map_single(ab->dev, skb->data,
 333                                        skb->len + skb_tailroom(skb),
 334                                        DMA_FROM_DEVICE);
 335                 if (unlikely(dma_mapping_error(ab->dev, paddr))) {
 336                         ath11k_warn(ab, "failed to dma map ce rx buf\n");
 337                         dev_kfree_skb_any(skb);
 338                         ret = -EIO;
 339                         goto exit;
 340                 }
 341
 342                 ATH11K_SKB_RXCB(skb)->paddr = paddr;
 343
 344                 ret = ath11k_ce_rx_buf_enqueue_pipe(pipe, skb, paddr);
 345
 346                 if (ret) {
 347                         ath11k_warn(ab, "failed to enqueue rx buf: %d\n", ret);
 348                         dma_unmap_single(ab->dev, paddr,
 349                                          skb->len + skb_tailroom(skb),
 350                                          DMA_FROM_DEVICE);
 351                         dev_kfree_skb_any(skb);
 352                         goto exit;
 353                 }
 354         }
 355
 356 exit:
 357         spin_unlock_bh(&ab->ce.ce_lock);
 358         return ret;
 359 }
 360
 361 static int ath11k_ce_completed_recv_next(struct ath11k_ce_pipe *pipe,
 362                                          struct sk_buff **skb, int *nbytes)
 363 {
 364         struct ath11k_base *ab = pipe->ab;
 365         struct hal_srng *srng;
 366         unsigned int sw_index;
 367         unsigned int nentries_mask;
 368         u32 *desc;
 369         int ret = 0;
 370
 371         spin_lock_bh(&ab->ce.ce_lock);
 372
 373         sw_index = pipe->dest_ring->sw_index;
 374         nentries_mask = pipe->dest_ring->nentries_mask;
 375
 376         srng = &ab->hal.srng_list[pipe->status_ring->hal_ring_id];
 377
 378         spin_lock_bh(&srng->lock);
 379
 380         ath11k_hal_srng_access_begin(ab, srng);
 381
 382         desc = ath11k_hal_srng_dst_get_next_entry(ab, srng);
 383         if (!desc) {
 384                 ret = -EIO;
 385                 goto err;
 386         }
 387
 388         *nbytes = ath11k_hal_ce_dst_status_get_length(desc);
 389         if (*nbytes == 0) {
 390                 ret = -EIO;
 391                 goto err;
 392         }
 393
 394         *skb = pipe->dest_ring->skb[sw_index];
 395         pipe->dest_ring->skb[sw_index] = NULL;
 396
 397         sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
 398         pipe->dest_ring->sw_index = sw_index;
 399
 400         pipe->rx_buf_needed++;
 401 err:
 402         ath11k_hal_srng_access_end(ab, srng);
 403
 404         spin_unlock_bh(&srng->lock);
 405
 406         spin_unlock_bh(&ab->ce.ce_lock);
 407
 408         return ret;
 409 }
 410
 411 static void ath11k_ce_recv_process_cb(struct ath11k_ce_pipe *pipe)
 412 {
 413         struct ath11k_base *ab = pipe->ab;
 414         struct sk_buff *skb;
 415         struct sk_buff_head list;
 416         unsigned int nbytes, max_nbytes;
 417         int ret;
 418
 419         __skb_queue_head_init(&list);
 420         while (ath11k_ce_completed_recv_next(pipe, &skb, &nbytes) == 0) {
 421                 max_nbytes = skb->len + skb_tailroom(skb);
 422                 dma_unmap_single(ab->dev, ATH11K_SKB_RXCB(skb)->paddr,
 423                                  max_nbytes, DMA_FROM_DEVICE);
 424
 425                 if (unlikely(max_nbytes < nbytes)) {
 426                         ath11k_warn(ab, "rxed more than expected (nbytes %d, max %d)",
 427                                     nbytes, max_nbytes);
 428                         dev_kfree_skb_any(skb);
 429                         continue;
 430                 }
 431
 432                 skb_put(skb, nbytes);
 433                 __skb_queue_tail(&list, skb);
 434         }
 435
 436         while ((skb = __skb_dequeue(&list))) {
 437                 ath11k_dbg(ab, ATH11K_DBG_AHB, "rx ce pipe %d len %d\n",
 438                            pipe->pipe_num, skb->len);
 439                 pipe->recv_cb(ab, skb);
 440         }
 441
 442         ret = ath11k_ce_rx_post_pipe(pipe);
 443         if (ret && ret != -ENOSPC) {
 444                 ath11k_warn(ab, "failed to post rx buf to pipe: %d err: %d\n",
 445                             pipe->pipe_num, ret);
 446                 mod_timer(&ab->rx_replenish_retry,
 447                           jiffies + ATH11K_CE_RX_POST_RETRY_JIFFIES);
 448         }
 449 }
 450
 451 static struct sk_buff *ath11k_ce_completed_send_next(struct ath11k_ce_pipe *pipe)
 452 {
 453         struct ath11k_base *ab = pipe->ab;
 454         struct hal_srng *srng;
 455         unsigned int sw_index;
 456         unsigned int nentries_mask;
 457         struct sk_buff *skb;
 458         u32 *desc;
 459
 460         spin_lock_bh(&ab->ce.ce_lock);
 461
 462         sw_index = pipe->src_ring->sw_index;
 463         nentries_mask = pipe->src_ring->nentries_mask;
 464
 465         srng = &ab->hal.srng_list[pipe->src_ring->hal_ring_id];
 466
 467         spin_lock_bh(&srng->lock);
 468
 469         ath11k_hal_srng_access_begin(ab, srng);
 470
 471         desc = ath11k_hal_srng_src_reap_next(ab, srng);
 472         if (!desc) {
 473                 skb = ERR_PTR(-EIO);
 474                 goto err_unlock;
 475         }
 476
 477         skb = pipe->src_ring->skb[sw_index];
 478
 479         pipe->src_ring->skb[sw_index] = NULL;
 480
 481         sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
 482         pipe->src_ring->sw_index = sw_index;
 483
 484 err_unlock:
 485         spin_unlock_bh(&srng->lock);
 486
 487         spin_unlock_bh(&ab->ce.ce_lock);
 488
 489         return skb;
 490 }
 491
 492 static void ath11k_ce_send_done_cb(struct ath11k_ce_pipe *pipe)
 493 {
 494         struct ath11k_base *ab = pipe->ab;
 495         struct sk_buff *skb;
 496
 497         while (!IS_ERR(skb = ath11k_ce_completed_send_next(pipe))) {
 498                 if (!skb)
 499                         continue;
 500
 501                 dma_unmap_single(ab->dev, ATH11K_SKB_CB(skb)->paddr, skb->len,
 502                                  DMA_TO_DEVICE);
 503                 dev_kfree_skb_any(skb);
 504         }
 505 }
 506
 507 static void ath11k_ce_srng_msi_ring_params_setup(struct ath11k_base *ab, u32 ce_id,
 508                                                  struct hal_srng_params *ring_params)
 509 {
 510         u32 msi_data_start;
 511         u32 msi_data_count, msi_data_idx;
 512         u32 msi_irq_start;
 513         u32 addr_lo;
 514         u32 addr_hi;
 515         int ret;
 516
 517         ret = ath11k_get_user_msi_vector(ab, "CE",
 518                                          &msi_data_count, &msi_data_start,
 519                                          &msi_irq_start);
 520
 521         if (ret)
 522                 return;
 523
 524         ath11k_get_msi_address(ab, &addr_lo, &addr_hi);
 525         ath11k_get_ce_msi_idx(ab, ce_id, &msi_data_idx);
 526
 527         ring_params->msi_addr = addr_lo;
 528         ring_params->msi_addr |= (dma_addr_t)(((uint64_t)addr_hi) << 32);
 529         ring_params->msi_data = (msi_data_idx % msi_data_count) + msi_data_start;
 530         ring_params->flags |= HAL_SRNG_FLAGS_MSI_INTR;
 531 }
 532
 533 static int ath11k_ce_init_ring(struct ath11k_base *ab,
 534                                struct ath11k_ce_ring *ce_ring,
 535                                int ce_id, enum hal_ring_type type)
 536 {
 537         struct hal_srng_params params = { 0 };
 538         int ret;
 539
 540         params.ring_base_paddr = ce_ring->base_addr_ce_space;
 541         params.ring_base_vaddr = ce_ring->base_addr_owner_space;
 542         params.num_entries = ce_ring->nentries;
 543
 544         if (!(CE_ATTR_DIS_INTR & ab->hw_params.host_ce_config[ce_id].flags))
 545                 ath11k_ce_srng_msi_ring_params_setup(ab, ce_id, &params);
 546
 547         switch (type) {
 548         case HAL_CE_SRC:
 549                 if (!(CE_ATTR_DIS_INTR & ab->hw_params.host_ce_config[ce_id].flags))
 550                         params.intr_batch_cntr_thres_entries = 1;
 551                 break;
 552         case HAL_CE_DST:
 553                 params.max_buffer_len = ab->hw_params.host_ce_config[ce_id].src_sz_max;
 554                 if (!(ab->hw_params.host_ce_config[ce_id].flags & CE_ATTR_DIS_INTR)) {
 555                         params.intr_timer_thres_us = 1024;
 556                         params.flags |= HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN;
 557                         params.low_threshold = ce_ring->nentries - 3;
 558                 }
 559                 break;
 560         case HAL_CE_DST_STATUS:
 561                 if (!(ab->hw_params.host_ce_config[ce_id].flags & CE_ATTR_DIS_INTR)) {
 562                         params.intr_batch_cntr_thres_entries = 1;
 563                         params.intr_timer_thres_us = 0x1000;
 564                 }
 565                 break;
 566         default:
 567                 ath11k_warn(ab, "Invalid CE ring type %d\n", type);
 568                 return -EINVAL;
 569         }
 570
 571         /* TODO: Init other params needed by HAL to init the ring */
 572
 573         ret = ath11k_hal_srng_setup(ab, type, ce_id, 0, &params);
 574         if (ret < 0) {
 575                 ath11k_warn(ab, "failed to setup srng: %d ring_id %d\n",
 576                             ret, ce_id);
 577                 return ret;
 578         }
 579
 580         ce_ring->hal_ring_id = ret;
 581
 582         if (ab->hw_params.supports_shadow_regs &&
 583             ath11k_ce_need_shadow_fix(ce_id))
 584                 ath11k_dp_shadow_init_timer(ab, &ab->ce.hp_timer[ce_id],
 585                                             ATH11K_SHADOW_CTRL_TIMER_INTERVAL,
 586                                             ce_ring->hal_ring_id);
 587
 588         return 0;
 589 }
 590
 591 static struct ath11k_ce_ring *
 592 ath11k_ce_alloc_ring(struct ath11k_base *ab, int nentries, int desc_sz)
 593 {
 594         struct ath11k_ce_ring *ce_ring;
 595         dma_addr_t base_addr;
 596
 597         ce_ring = kzalloc(struct_size(ce_ring, skb, nentries), GFP_KERNEL);
 598         if (ce_ring == NULL)
 599                 return ERR_PTR(-ENOMEM);
 600
 601         ce_ring->nentries = nentries;
 602         ce_ring->nentries_mask = nentries - 1;
 603
 604         /* Legacy platforms that do not support cache
 605          * coherent DMA are unsupported
 606          */
 607         ce_ring->base_addr_owner_space_unaligned =
 608                 dma_alloc_coherent(ab->dev,
 609                                    nentries * desc_sz + CE_DESC_RING_ALIGN,
 610                                    &base_addr, GFP_KERNEL);
 611         if (!ce_ring->base_addr_owner_space_unaligned) {
 612                 kfree(ce_ring);
 613                 return ERR_PTR(-ENOMEM);
 614         }
 615
 616         ce_ring->base_addr_ce_space_unaligned = base_addr;
 617
 618         ce_ring->base_addr_owner_space = PTR_ALIGN(
 619                         ce_ring->base_addr_owner_space_unaligned,
 620                         CE_DESC_RING_ALIGN);
 621         ce_ring->base_addr_ce_space = ALIGN(
 622                         ce_ring->base_addr_ce_space_unaligned,
 623                         CE_DESC_RING_ALIGN);
 624
 625         return ce_ring;
 626 }
 627
 628 static int ath11k_ce_alloc_pipe(struct ath11k_base *ab, int ce_id)
 629 {
 630         struct ath11k_ce_pipe *pipe = &ab->ce.ce_pipe[ce_id];
 631         const struct ce_attr *attr = &ab->hw_params.host_ce_config[ce_id];
 632         struct ath11k_ce_ring *ring;
 633         int nentries;
 634         int desc_sz;
 635
 636         pipe->attr_flags = attr->flags;
 637
 638         if (attr->src_nentries) {
 639                 pipe->send_cb = ath11k_ce_send_done_cb;
 640                 nentries = roundup_pow_of_two(attr->src_nentries);
 641                 desc_sz = ath11k_hal_ce_get_desc_size(HAL_CE_DESC_SRC);
 642                 ring = ath11k_ce_alloc_ring(ab, nentries, desc_sz);
 643                 if (IS_ERR(ring))
 644                         return PTR_ERR(ring);
 645                 pipe->src_ring = ring;
 646         }
 647
 648         if (attr->dest_nentries) {
 649                 pipe->recv_cb = attr->recv_cb;
 650                 nentries = roundup_pow_of_two(attr->dest_nentries);
 651                 desc_sz = ath11k_hal_ce_get_desc_size(HAL_CE_DESC_DST);
 652                 ring = ath11k_ce_alloc_ring(ab, nentries, desc_sz);
 653                 if (IS_ERR(ring))
 654                         return PTR_ERR(ring);
 655                 pipe->dest_ring = ring;
 656
 657                 desc_sz = ath11k_hal_ce_get_desc_size(HAL_CE_DESC_DST_STATUS);
 658                 ring = ath11k_ce_alloc_ring(ab, nentries, desc_sz);
 659                 if (IS_ERR(ring))
 660                         return PTR_ERR(ring);
 661                 pipe->status_ring = ring;
 662         }
 663
 664         return 0;
 665 }
 666
 667 void ath11k_ce_per_engine_service(struct ath11k_base *ab, u16 ce_id)
 668 {
 669         struct ath11k_ce_pipe *pipe = &ab->ce.ce_pipe[ce_id];
 670
 671         if (pipe->send_cb)
 672                 pipe->send_cb(pipe);
 673
 674         if (pipe->recv_cb)
 675                 ath11k_ce_recv_process_cb(pipe);
 676 }
 677
 678 void ath11k_ce_poll_send_completed(struct ath11k_base *ab, u8 pipe_id)
 679 {
 680         struct ath11k_ce_pipe *pipe = &ab->ce.ce_pipe[pipe_id];
 681
 682         if ((pipe->attr_flags & CE_ATTR_DIS_INTR) && pipe->send_cb)
 683                 pipe->send_cb(pipe);
 684 }
 685 EXPORT_SYMBOL(ath11k_ce_per_engine_service);
 686
 687 int ath11k_ce_send(struct ath11k_base *ab, struct sk_buff *skb, u8 pipe_id,
 688                    u16 transfer_id)
 689 {
 690         struct ath11k_ce_pipe *pipe = &ab->ce.ce_pipe[pipe_id];
 691         struct hal_srng *srng;
 692         u32 *desc;
 693         unsigned int write_index, sw_index;
 694         unsigned int nentries_mask;
 695         int ret = 0;
 696         u8 byte_swap_data = 0;
 697         int num_used;
 698
 699         /* Check if some entries could be regained by handling tx completion if
 700          * the CE has interrupts disabled and the used entries is more than the
 701          * defined usage threshold.
 702          */
 703         if (pipe->attr_flags & CE_ATTR_DIS_INTR) {
 704                 spin_lock_bh(&ab->ce.ce_lock);
 705                 write_index = pipe->src_ring->write_index;
 706
 707                 sw_index = pipe->src_ring->sw_index;
 708
 709                 if (write_index >= sw_index)
 710                         num_used = write_index - sw_index;
 711                 else
 712                         num_used = pipe->src_ring->nentries - sw_index +
 713                                    write_index;
 714
 715                 spin_unlock_bh(&ab->ce.ce_lock);
 716
 717                 if (num_used > ATH11K_CE_USAGE_THRESHOLD)
 718                         ath11k_ce_poll_send_completed(ab, pipe->pipe_num);
 719         }
 720
 721         if (test_bit(ATH11K_FLAG_CRASH_FLUSH, &ab->dev_flags))
 722                 return -ESHUTDOWN;
 723
 724         spin_lock_bh(&ab->ce.ce_lock);
 725
 726         write_index = pipe->src_ring->write_index;
 727         nentries_mask = pipe->src_ring->nentries_mask;
 728
 729         srng = &ab->hal.srng_list[pipe->src_ring->hal_ring_id];
 730
 731         spin_lock_bh(&srng->lock);
 732
 733         ath11k_hal_srng_access_begin(ab, srng);
 734
 735         if (unlikely(ath11k_hal_srng_src_num_free(ab, srng, false) < 1)) {
 736                 ath11k_hal_srng_access_end(ab, srng);
 737                 ret = -ENOBUFS;
 738                 goto err_unlock;
 739         }
 740
 741         desc = ath11k_hal_srng_src_get_next_reaped(ab, srng);
 742         if (!desc) {
 743                 ath11k_hal_srng_access_end(ab, srng);
 744                 ret = -ENOBUFS;
 745                 goto err_unlock;
 746         }
 747
 748         if (pipe->attr_flags & CE_ATTR_BYTE_SWAP_DATA)
 749                 byte_swap_data = 1;
 750
 751         ath11k_hal_ce_src_set_desc(desc, ATH11K_SKB_CB(skb)->paddr,
 752                                    skb->len, transfer_id, byte_swap_data);
 753
 754         pipe->src_ring->skb[write_index] = skb;
 755         pipe->src_ring->write_index = CE_RING_IDX_INCR(nentries_mask,
 756                                                        write_index);
 757
 758         ath11k_hal_srng_access_end(ab, srng);
 759
 760         if (ath11k_ce_need_shadow_fix(pipe_id))
 761                 ath11k_dp_shadow_start_timer(ab, srng, &ab->ce.hp_timer[pipe_id]);
 762
 763         spin_unlock_bh(&srng->lock);
 764
 765         spin_unlock_bh(&ab->ce.ce_lock);
 766
 767         return 0;
 768
 769 err_unlock:
 770         spin_unlock_bh(&srng->lock);
 771
 772         spin_unlock_bh(&ab->ce.ce_lock);
 773
 774         return ret;
 775 }
 776
 777 static void ath11k_ce_rx_pipe_cleanup(struct ath11k_ce_pipe *pipe)
 778 {
 779         struct ath11k_base *ab = pipe->ab;
 780         struct ath11k_ce_ring *ring = pipe->dest_ring;
 781         struct sk_buff *skb;
 782         int i;
 783
 784         if (!(ring && pipe->buf_sz))
 785                 return;
 786
 787         for (i = 0; i < ring->nentries; i++) {
 788                 skb = ring->skb[i];
 789                 if (!skb)
 790                         continue;
 791
 792                 ring->skb[i] = NULL;
 793                 dma_unmap_single(ab->dev, ATH11K_SKB_RXCB(skb)->paddr,
 794                                  skb->len + skb_tailroom(skb), DMA_FROM_DEVICE);
 795                 dev_kfree_skb_any(skb);
 796         }
 797 }
 798
 799 static void ath11k_ce_shadow_config(struct ath11k_base *ab)
 800 {
 801         int i;
 802
 803         for (i = 0; i < ab->hw_params.ce_count; i++) {
 804                 if (ab->hw_params.host_ce_config[i].src_nentries)
 805                         ath11k_hal_srng_update_shadow_config(ab,
 806                                                              HAL_CE_SRC, i);
 807
 808                 if (ab->hw_params.host_ce_config[i].dest_nentries) {
 809                         ath11k_hal_srng_update_shadow_config(ab,
 810                                                              HAL_CE_DST, i);
 811
 812                         ath11k_hal_srng_update_shadow_config(ab,
 813                                                              HAL_CE_DST_STATUS, i);
 814                 }
 815         }
 816 }
 817
 818 void ath11k_ce_get_shadow_config(struct ath11k_base *ab,
 819                                  u32 **shadow_cfg, u32 *shadow_cfg_len)
 820 {
 821         if (!ab->hw_params.supports_shadow_regs)
 822                 return;
 823
 824         ath11k_hal_srng_get_shadow_config(ab, shadow_cfg, shadow_cfg_len);
 825
 826         /* shadow is already configured */
 827         if (*shadow_cfg_len)
 828                 return;
 829
 830         /* shadow isn't configured yet, configure now.
 831          * non-CE srngs are configured firstly, then
 832          * all CE srngs.
 833          */
 834         ath11k_hal_srng_shadow_config(ab);
 835         ath11k_ce_shadow_config(ab);
 836
 837         /* get the shadow configuration */
 838         ath11k_hal_srng_get_shadow_config(ab, shadow_cfg, shadow_cfg_len);
 839 }
 840 EXPORT_SYMBOL(ath11k_ce_get_shadow_config);
 841
 842 void ath11k_ce_cleanup_pipes(struct ath11k_base *ab)
 843 {
 844         struct ath11k_ce_pipe *pipe;
 845         int pipe_num;
 846
 847         ath11k_ce_stop_shadow_timers(ab);
 848
 849         for (pipe_num = 0; pipe_num < ab->hw_params.ce_count; pipe_num++) {
 850                 pipe = &ab->ce.ce_pipe[pipe_num];
 851                 ath11k_ce_rx_pipe_cleanup(pipe);
 852
 853                 /* Cleanup any src CE's which have interrupts disabled */
 854                 ath11k_ce_poll_send_completed(ab, pipe_num);
 855
 856                 /* NOTE: Should we also clean up tx buffer in all pipes? */
 857         }
 858 }
 859 EXPORT_SYMBOL(ath11k_ce_cleanup_pipes);
 860
 861 void ath11k_ce_rx_post_buf(struct ath11k_base *ab)
 862 {
 863         struct ath11k_ce_pipe *pipe;
 864         int i;
 865         int ret;
 866
 867         for (i = 0; i < ab->hw_params.ce_count; i++) {
 868                 pipe = &ab->ce.ce_pipe[i];
 869                 ret = ath11k_ce_rx_post_pipe(pipe);
 870                 if (ret) {
 871                         if (ret == -ENOSPC)
 872                                 continue;
 873
 874                         ath11k_warn(ab, "failed to post rx buf to pipe: %d err: %d\n",
 875                                     i, ret);
 876                         mod_timer(&ab->rx_replenish_retry,
 877                                   jiffies + ATH11K_CE_RX_POST_RETRY_JIFFIES);
 878
 879                         return;
 880                 }
 881         }
 882 }
 883 EXPORT_SYMBOL(ath11k_ce_rx_post_buf);
 884
 885 void ath11k_ce_rx_replenish_retry(struct timer_list *t)
 886 {
 887         struct ath11k_base *ab = from_timer(ab, t, rx_replenish_retry);
 888
 889         ath11k_ce_rx_post_buf(ab);
 890 }
 891
 892 int ath11k_ce_init_pipes(struct ath11k_base *ab)
 893 {
 894         struct ath11k_ce_pipe *pipe;
 895         int i;
 896         int ret;
 897
 898         ath11k_ce_get_shadow_config(ab, &ab->qmi.ce_cfg.shadow_reg_v2,
 899                                     &ab->qmi.ce_cfg.shadow_reg_v2_len);
 900
 901         for (i = 0; i < ab->hw_params.ce_count; i++) {
 902                 pipe = &ab->ce.ce_pipe[i];
 903
 904                 if (pipe->src_ring) {
 905                         ret = ath11k_ce_init_ring(ab, pipe->src_ring, i,
 906                                                   HAL_CE_SRC);
 907                         if (ret) {
 908                                 ath11k_warn(ab, "failed to init src ring: %d\n",
 909                                             ret);
 910                                 /* Should we clear any partial init */
 911                                 return ret;
 912                         }
 913
 914                         pipe->src_ring->write_index = 0;
 915                         pipe->src_ring->sw_index = 0;
 916                 }
 917
 918                 if (pipe->dest_ring) {
 919                         ret = ath11k_ce_init_ring(ab, pipe->dest_ring, i,
 920                                                   HAL_CE_DST);
 921                         if (ret) {
 922                                 ath11k_warn(ab, "failed to init dest ring: %d\n",
 923                                             ret);
 924                                 /* Should we clear any partial init */
 925                                 return ret;
 926                         }
 927
 928                         pipe->rx_buf_needed = pipe->dest_ring->nentries ?
 929                                               pipe->dest_ring->nentries - 2 : 0;
 930
 931                         pipe->dest_ring->write_index = 0;
 932                         pipe->dest_ring->sw_index = 0;
 933                 }
 934
 935                 if (pipe->status_ring) {
 936                         ret = ath11k_ce_init_ring(ab, pipe->status_ring, i,
 937                                                   HAL_CE_DST_STATUS);
 938                         if (ret) {
 939                                 ath11k_warn(ab, "failed to init dest status ing: %d\n",
 940                                             ret);
 941                                 /* Should we clear any partial init */
 942                                 return ret;
 943                         }
 944
 945                         pipe->status_ring->write_index = 0;
 946                         pipe->status_ring->sw_index = 0;
 947                 }
 948         }
 949
 950         return 0;
 951 }
 952
 953 void ath11k_ce_free_pipes(struct ath11k_base *ab)
 954 {
 955         struct ath11k_ce_pipe *pipe;
 956         int desc_sz;
 957         int i;
 958
 959         for (i = 0; i < ab->hw_params.ce_count; i++) {
 960                 pipe = &ab->ce.ce_pipe[i];
 961
 962                 if (ath11k_ce_need_shadow_fix(i))
 963                         ath11k_dp_shadow_stop_timer(ab, &ab->ce.hp_timer[i]);
 964
 965                 if (pipe->src_ring) {
 966                         desc_sz = ath11k_hal_ce_get_desc_size(HAL_CE_DESC_SRC);
 967                         dma_free_coherent(ab->dev,
 968                                           pipe->src_ring->nentries * desc_sz +
 969                                           CE_DESC_RING_ALIGN,
 970                                           pipe->src_ring->base_addr_owner_space,
 971                                           pipe->src_ring->base_addr_ce_space);
 972                         kfree(pipe->src_ring);
 973                         pipe->src_ring = NULL;
 974                 }
 975
 976                 if (pipe->dest_ring) {
 977                         desc_sz = ath11k_hal_ce_get_desc_size(HAL_CE_DESC_DST);
 978                         dma_free_coherent(ab->dev,
 979                                           pipe->dest_ring->nentries * desc_sz +
 980                                           CE_DESC_RING_ALIGN,
 981                                           pipe->dest_ring->base_addr_owner_space,
 982                                           pipe->dest_ring->base_addr_ce_space);
 983                         kfree(pipe->dest_ring);
 984                         pipe->dest_ring = NULL;
 985                 }
 986
 987                 if (pipe->status_ring) {
 988                         desc_sz =
 989                           ath11k_hal_ce_get_desc_size(HAL_CE_DESC_DST_STATUS);
 990                         dma_free_coherent(ab->dev,
 991                                           pipe->status_ring->nentries * desc_sz +
 992                                           CE_DESC_RING_ALIGN,
 993                                           pipe->status_ring->base_addr_owner_space,
 994                                           pipe->status_ring->base_addr_ce_space);
 995                         kfree(pipe->status_ring);
 996                         pipe->status_ring = NULL;
 997                 }
 998         }
 999 }
1000 EXPORT_SYMBOL(ath11k_ce_free_pipes);
1001
1002 int ath11k_ce_alloc_pipes(struct ath11k_base *ab)
1003 {
1004         struct ath11k_ce_pipe *pipe;
1005         int i;
1006         int ret;
1007         const struct ce_attr *attr;
1008
1009         spin_lock_init(&ab->ce.ce_lock);
1010
1011         for (i = 0; i < ab->hw_params.ce_count; i++) {
1012                 attr = &ab->hw_params.host_ce_config[i];
1013                 pipe = &ab->ce.ce_pipe[i];
1014                 pipe->pipe_num = i;
1015                 pipe->ab = ab;
1016                 pipe->buf_sz = attr->src_sz_max;
1017
1018                 ret = ath11k_ce_alloc_pipe(ab, i);
1019                 if (ret) {
1020                         /* Free any parial successful allocation */
1021                         ath11k_ce_free_pipes(ab);
1022                         return ret;
1023                 }
1024         }
1025
1026         return 0;
1027 }
1028 EXPORT_SYMBOL(ath11k_ce_alloc_pipes);
1029
1030 /* For Big Endian Host, Copy Engine byte_swap is enabled
1031  * When Copy Engine does byte_swap, need to byte swap again for the
1032  * Host to get/put buffer content in the correct byte order
1033  */
1034 void ath11k_ce_byte_swap(void *mem, u32 len)
1035 {
1036         int i;
1037
1038         if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) {
1039                 if (!mem)
1040                         return;
1041
1042                 for (i = 0; i < (len / 4); i++) {
1043                         *(u32 *)mem = swab32(*(u32 *)mem);
1044                         mem += 4;
1045                 }
1046         }
1047 }
1048
1049 int ath11k_ce_get_attr_flags(struct ath11k_base *ab, int ce_id)
1050 {
1051         if (ce_id >= ab->hw_params.ce_count)
1052                 return -EINVAL;
1053
1054         return ab->hw_params.host_ce_config[ce_id].flags;
1055 }
1056 EXPORT_SYMBOL(ath11k_ce_get_attr_flags);