drivers/net/ethernet/intel/idpf/idpf_controlq.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (C) 2023 Intel Corporation */
   3
   4 #include "idpf_controlq.h"
   5
   6 /**
   7  * idpf_ctlq_setup_regs - initialize control queue registers
   8  * @cq: pointer to the specific control queue
   9  * @q_create_info: structs containing info for each queue to be initialized
  10  */
  11 static void idpf_ctlq_setup_regs(struct idpf_ctlq_info *cq,
  12                                  struct idpf_ctlq_create_info *q_create_info)
  13 {
  14         /* set control queue registers in our local struct */
  15         cq->reg.head = q_create_info->reg.head;
  16         cq->reg.tail = q_create_info->reg.tail;
  17         cq->reg.len = q_create_info->reg.len;
  18         cq->reg.bah = q_create_info->reg.bah;
  19         cq->reg.bal = q_create_info->reg.bal;
  20         cq->reg.len_mask = q_create_info->reg.len_mask;
  21         cq->reg.len_ena_mask = q_create_info->reg.len_ena_mask;
  22         cq->reg.head_mask = q_create_info->reg.head_mask;
  23 }
  24
  25 /**
  26  * idpf_ctlq_init_regs - Initialize control queue registers
  27  * @hw: pointer to hw struct
  28  * @cq: pointer to the specific Control queue
  29  * @is_rxq: true if receive control queue, false otherwise
  30  *
  31  * Initialize registers. The caller is expected to have already initialized the
  32  * descriptor ring memory and buffer memory
  33  */
  34 static void idpf_ctlq_init_regs(struct idpf_hw *hw, struct idpf_ctlq_info *cq,
  35                                 bool is_rxq)
  36 {
  37         /* Update tail to post pre-allocated buffers for rx queues */
  38         if (is_rxq)
  39                 wr32(hw, cq->reg.tail, (u32)(cq->ring_size - 1));
  40
  41         /* For non-Mailbox control queues only TAIL need to be set */
  42         if (cq->q_id != -1)
  43                 return;
  44
  45         /* Clear Head for both send or receive */
  46         wr32(hw, cq->reg.head, 0);
  47
  48         /* set starting point */
  49         wr32(hw, cq->reg.bal, lower_32_bits(cq->desc_ring.pa));
  50         wr32(hw, cq->reg.bah, upper_32_bits(cq->desc_ring.pa));
  51         wr32(hw, cq->reg.len, (cq->ring_size | cq->reg.len_ena_mask));
  52 }
  53
  54 /**
  55  * idpf_ctlq_init_rxq_bufs - populate receive queue descriptors with buf
  56  * @cq: pointer to the specific Control queue
  57  *
  58  * Record the address of the receive queue DMA buffers in the descriptors.
  59  * The buffers must have been previously allocated.
  60  */
  61 static void idpf_ctlq_init_rxq_bufs(struct idpf_ctlq_info *cq)
  62 {
  63         int i;
  64
  65         for (i = 0; i < cq->ring_size; i++) {
  66                 struct idpf_ctlq_desc *desc = IDPF_CTLQ_DESC(cq, i);
  67                 struct idpf_dma_mem *bi = cq->bi.rx_buff[i];
  68
  69                 /* No buffer to post to descriptor, continue */
  70                 if (!bi)
  71                         continue;
  72
  73                 desc->flags =
  74                         cpu_to_le16(IDPF_CTLQ_FLAG_BUF | IDPF_CTLQ_FLAG_RD);
  75                 desc->opcode = 0;
  76                 desc->datalen = cpu_to_le16(bi->size);
  77                 desc->ret_val = 0;
  78                 desc->v_opcode_dtype = 0;
  79                 desc->v_retval = 0;
  80                 desc->params.indirect.addr_high =
  81                         cpu_to_le32(upper_32_bits(bi->pa));
  82                 desc->params.indirect.addr_low =
  83                         cpu_to_le32(lower_32_bits(bi->pa));
  84                 desc->params.indirect.param0 = 0;
  85                 desc->params.indirect.sw_cookie = 0;
  86                 desc->params.indirect.v_flags = 0;
  87         }
  88 }
  89
  90 /**
  91  * idpf_ctlq_shutdown - shutdown the CQ
  92  * @hw: pointer to hw struct
  93  * @cq: pointer to the specific Control queue
  94  *
  95  * The main shutdown routine for any controq queue
  96  */
  97 static void idpf_ctlq_shutdown(struct idpf_hw *hw, struct idpf_ctlq_info *cq)
  98 {
  99         mutex_lock(&cq->cq_lock);
 100
 101         /* free ring buffers and the ring itself */
 102         idpf_ctlq_dealloc_ring_res(hw, cq);
 103
 104         /* Set ring_size to 0 to indicate uninitialized queue */
 105         cq->ring_size = 0;
 106
 107         mutex_unlock(&cq->cq_lock);
 108         mutex_destroy(&cq->cq_lock);
 109 }
 110
 111 /**
 112  * idpf_ctlq_add - add one control queue
 113  * @hw: pointer to hardware struct
 114  * @qinfo: info for queue to be created
 115  * @cq_out: (output) double pointer to control queue to be created
 116  *
 117  * Allocate and initialize a control queue and add it to the control queue list.
 118  * The cq parameter will be allocated/initialized and passed back to the caller
 119  * if no errors occur.
 120  *
 121  * Note: idpf_ctlq_init must be called prior to any calls to idpf_ctlq_add
 122  */
 123 int idpf_ctlq_add(struct idpf_hw *hw,
 124                   struct idpf_ctlq_create_info *qinfo,
 125                   struct idpf_ctlq_info **cq_out)
 126 {
 127         struct idpf_ctlq_info *cq;
 128         bool is_rxq = false;
 129         int err;
 130
 131         cq = kzalloc(sizeof(*cq), GFP_KERNEL);
 132         if (!cq)
 133                 return -ENOMEM;
 134
 135         cq->cq_type = qinfo->type;
 136         cq->q_id = qinfo->id;
 137         cq->buf_size = qinfo->buf_size;
 138         cq->ring_size = qinfo->len;
 139
 140         cq->next_to_use = 0;
 141         cq->next_to_clean = 0;
 142         cq->next_to_post = cq->ring_size - 1;
 143
 144         switch (qinfo->type) {
 145         case IDPF_CTLQ_TYPE_MAILBOX_RX:
 146                 is_rxq = true;
 147                 fallthrough;
 148         case IDPF_CTLQ_TYPE_MAILBOX_TX:
 149                 err = idpf_ctlq_alloc_ring_res(hw, cq);
 150                 break;
 151         default:
 152                 err = -EBADR;
 153                 break;
 154         }
 155
 156         if (err)
 157                 goto init_free_q;
 158
 159         if (is_rxq) {
 160                 idpf_ctlq_init_rxq_bufs(cq);
 161         } else {
 162                 /* Allocate the array of msg pointers for TX queues */
 163                 cq->bi.tx_msg = kcalloc(qinfo->len,
 164                                         sizeof(struct idpf_ctlq_msg *),
 165                                         GFP_KERNEL);
 166                 if (!cq->bi.tx_msg) {
 167                         err = -ENOMEM;
 168                         goto init_dealloc_q_mem;
 169                 }
 170         }
 171
 172         idpf_ctlq_setup_regs(cq, qinfo);
 173
 174         idpf_ctlq_init_regs(hw, cq, is_rxq);
 175
 176         mutex_init(&cq->cq_lock);
 177
 178         list_add(&cq->cq_list, &hw->cq_list_head);
 179
 180         *cq_out = cq;
 181
 182         return 0;
 183
 184 init_dealloc_q_mem:
 185         /* free ring buffers and the ring itself */
 186         idpf_ctlq_dealloc_ring_res(hw, cq);
 187 init_free_q:
 188         kfree(cq);
 189
 190         return err;
 191 }
 192
 193 /**
 194  * idpf_ctlq_remove - deallocate and remove specified control queue
 195  * @hw: pointer to hardware struct
 196  * @cq: pointer to control queue to be removed
 197  */
 198 void idpf_ctlq_remove(struct idpf_hw *hw,
 199                       struct idpf_ctlq_info *cq)
 200 {
 201         list_del(&cq->cq_list);
 202         idpf_ctlq_shutdown(hw, cq);
 203         kfree(cq);
 204 }
 205
 206 /**
 207  * idpf_ctlq_init - main initialization routine for all control queues
 208  * @hw: pointer to hardware struct
 209  * @num_q: number of queues to initialize
 210  * @q_info: array of structs containing info for each queue to be initialized
 211  *
 212  * This initializes any number and any type of control queues. This is an all
 213  * or nothing routine; if one fails, all previously allocated queues will be
 214  * destroyed. This must be called prior to using the individual add/remove
 215  * APIs.
 216  */
 217 int idpf_ctlq_init(struct idpf_hw *hw, u8 num_q,
 218                    struct idpf_ctlq_create_info *q_info)
 219 {
 220         struct idpf_ctlq_info *cq, *tmp;
 221         int err;
 222         int i;
 223
 224         INIT_LIST_HEAD(&hw->cq_list_head);
 225
 226         for (i = 0; i < num_q; i++) {
 227                 struct idpf_ctlq_create_info *qinfo = q_info + i;
 228
 229                 err = idpf_ctlq_add(hw, qinfo, &cq);
 230                 if (err)
 231                         goto init_destroy_qs;
 232         }
 233
 234         return 0;
 235
 236 init_destroy_qs:
 237         list_for_each_entry_safe(cq, tmp, &hw->cq_list_head, cq_list)
 238                 idpf_ctlq_remove(hw, cq);
 239
 240         return err;
 241 }
 242
 243 /**
 244  * idpf_ctlq_deinit - destroy all control queues
 245  * @hw: pointer to hw struct
 246  */
 247 void idpf_ctlq_deinit(struct idpf_hw *hw)
 248 {
 249         struct idpf_ctlq_info *cq, *tmp;
 250
 251         list_for_each_entry_safe(cq, tmp, &hw->cq_list_head, cq_list)
 252                 idpf_ctlq_remove(hw, cq);
 253 }
 254
 255 /**
 256  * idpf_ctlq_send - send command to Control Queue (CTQ)
 257  * @hw: pointer to hw struct
 258  * @cq: handle to control queue struct to send on
 259  * @num_q_msg: number of messages to send on control queue
 260  * @q_msg: pointer to array of queue messages to be sent
 261  *
 262  * The caller is expected to allocate DMAable buffers and pass them to the
 263  * send routine via the q_msg struct / control queue specific data struct.
 264  * The control queue will hold a reference to each send message until
 265  * the completion for that message has been cleaned.
 266  */
 267 int idpf_ctlq_send(struct idpf_hw *hw, struct idpf_ctlq_info *cq,
 268                    u16 num_q_msg, struct idpf_ctlq_msg q_msg[])
 269 {
 270         struct idpf_ctlq_desc *desc;
 271         int num_desc_avail;
 272         int err = 0;
 273         int i;
 274
 275         mutex_lock(&cq->cq_lock);
 276
 277         /* Ensure there are enough descriptors to send all messages */
 278         num_desc_avail = IDPF_CTLQ_DESC_UNUSED(cq);
 279         if (num_desc_avail == 0 || num_desc_avail < num_q_msg) {
 280                 err = -ENOSPC;
 281                 goto err_unlock;
 282         }
 283
 284         for (i = 0; i < num_q_msg; i++) {
 285                 struct idpf_ctlq_msg *msg = &q_msg[i];
 286
 287                 desc = IDPF_CTLQ_DESC(cq, cq->next_to_use);
 288
 289                 desc->opcode = cpu_to_le16(msg->opcode);
 290                 desc->pfid_vfid = cpu_to_le16(msg->func_id);
 291
 292                 desc->v_opcode_dtype = cpu_to_le32(msg->cookie.mbx.chnl_opcode);
 293                 desc->v_retval = cpu_to_le32(msg->cookie.mbx.chnl_retval);
 294
 295                 desc->flags = cpu_to_le16((msg->host_id & IDPF_HOST_ID_MASK) <<
 296                                           IDPF_CTLQ_FLAG_HOST_ID_S);
 297                 if (msg->data_len) {
 298                         struct idpf_dma_mem *buff = msg->ctx.indirect.payload;
 299
 300                         desc->datalen |= cpu_to_le16(msg->data_len);
 301                         desc->flags |= cpu_to_le16(IDPF_CTLQ_FLAG_BUF);
 302                         desc->flags |= cpu_to_le16(IDPF_CTLQ_FLAG_RD);
 303
 304                         /* Update the address values in the desc with the pa
 305                          * value for respective buffer
 306                          */
 307                         desc->params.indirect.addr_high =
 308                                 cpu_to_le32(upper_32_bits(buff->pa));
 309                         desc->params.indirect.addr_low =
 310                                 cpu_to_le32(lower_32_bits(buff->pa));
 311
 312                         memcpy(&desc->params, msg->ctx.indirect.context,
 313                                IDPF_INDIRECT_CTX_SIZE);
 314                 } else {
 315                         memcpy(&desc->params, msg->ctx.direct,
 316                                IDPF_DIRECT_CTX_SIZE);
 317                 }
 318
 319                 /* Store buffer info */
 320                 cq->bi.tx_msg[cq->next_to_use] = msg;
 321
 322                 (cq->next_to_use)++;
 323                 if (cq->next_to_use == cq->ring_size)
 324                         cq->next_to_use = 0;
 325         }
 326
 327         /* Force memory write to complete before letting hardware
 328          * know that there are new descriptors to fetch.
 329          */
 330         dma_wmb();
 331
 332         wr32(hw, cq->reg.tail, cq->next_to_use);
 333
 334 err_unlock:
 335         mutex_unlock(&cq->cq_lock);
 336
 337         return err;
 338 }
 339
 340 /**
 341  * idpf_ctlq_clean_sq - reclaim send descriptors on HW write back for the
 342  * requested queue
 343  * @cq: pointer to the specific Control queue
 344  * @clean_count: (input|output) number of descriptors to clean as input, and
 345  * number of descriptors actually cleaned as output
 346  * @msg_status: (output) pointer to msg pointer array to be populated; needs
 347  * to be allocated by caller
 348  *
 349  * Returns an array of message pointers associated with the cleaned
 350  * descriptors. The pointers are to the original ctlq_msgs sent on the cleaned
 351  * descriptors.  The status will be returned for each; any messages that failed
 352  * to send will have a non-zero status. The caller is expected to free original
 353  * ctlq_msgs and free or reuse the DMA buffers.
 354  */
 355 int idpf_ctlq_clean_sq(struct idpf_ctlq_info *cq, u16 *clean_count,
 356                        struct idpf_ctlq_msg *msg_status[])
 357 {
 358         struct idpf_ctlq_desc *desc;
 359         u16 i, num_to_clean;
 360         u16 ntc, desc_err;
 361
 362         if (*clean_count == 0)
 363                 return 0;
 364         if (*clean_count > cq->ring_size)
 365                 return -EBADR;
 366
 367         mutex_lock(&cq->cq_lock);
 368
 369         ntc = cq->next_to_clean;
 370
 371         num_to_clean = *clean_count;
 372
 373         for (i = 0; i < num_to_clean; i++) {
 374                 /* Fetch next descriptor and check if marked as done */
 375                 desc = IDPF_CTLQ_DESC(cq, ntc);
 376                 if (!(le16_to_cpu(desc->flags) & IDPF_CTLQ_FLAG_DD))
 377                         break;
 378
 379                 /* strip off FW internal code */
 380                 desc_err = le16_to_cpu(desc->ret_val) & 0xff;
 381
 382                 msg_status[i] = cq->bi.tx_msg[ntc];
 383                 msg_status[i]->status = desc_err;
 384
 385                 cq->bi.tx_msg[ntc] = NULL;
 386
 387                 /* Zero out any stale data */
 388                 memset(desc, 0, sizeof(*desc));
 389
 390                 ntc++;
 391                 if (ntc == cq->ring_size)
 392                         ntc = 0;
 393         }
 394
 395         cq->next_to_clean = ntc;
 396
 397         mutex_unlock(&cq->cq_lock);
 398
 399         /* Return number of descriptors actually cleaned */
 400         *clean_count = i;
 401
 402         return 0;
 403 }
 404
 405 /**
 406  * idpf_ctlq_post_rx_buffs - post buffers to descriptor ring
 407  * @hw: pointer to hw struct
 408  * @cq: pointer to control queue handle
 409  * @buff_count: (input|output) input is number of buffers caller is trying to
 410  * return; output is number of buffers that were not posted
 411  * @buffs: array of pointers to dma mem structs to be given to hardware
 412  *
 413  * Caller uses this function to return DMA buffers to the descriptor ring after
 414  * consuming them; buff_count will be the number of buffers.
 415  *
 416  * Note: this function needs to be called after a receive call even
 417  * if there are no DMA buffers to be returned, i.e. buff_count = 0,
 418  * buffs = NULL to support direct commands
 419  */
 420 int idpf_ctlq_post_rx_buffs(struct idpf_hw *hw, struct idpf_ctlq_info *cq,
 421                             u16 *buff_count, struct idpf_dma_mem **buffs)
 422 {
 423         struct idpf_ctlq_desc *desc;
 424         u16 ntp = cq->next_to_post;
 425         bool buffs_avail = false;
 426         u16 tbp = ntp + 1;
 427         int i = 0;
 428
 429         if (*buff_count > cq->ring_size)
 430                 return -EBADR;
 431
 432         if (*buff_count > 0)
 433                 buffs_avail = true;
 434
 435         mutex_lock(&cq->cq_lock);
 436
 437         if (tbp >= cq->ring_size)
 438                 tbp = 0;
 439
 440         if (tbp == cq->next_to_clean)
 441                 /* Nothing to do */
 442                 goto post_buffs_out;
 443
 444         /* Post buffers for as many as provided or up until the last one used */
 445         while (ntp != cq->next_to_clean) {
 446                 desc = IDPF_CTLQ_DESC(cq, ntp);
 447
 448                 if (cq->bi.rx_buff[ntp])
 449                         goto fill_desc;
 450                 if (!buffs_avail) {
 451                         /* If the caller hasn't given us any buffers or
 452                          * there are none left, search the ring itself
 453                          * for an available buffer to move to this
 454                          * entry starting at the next entry in the ring
 455                          */
 456                         tbp = ntp + 1;
 457
 458                         /* Wrap ring if necessary */
 459                         if (tbp >= cq->ring_size)
 460                                 tbp = 0;
 461
 462                         while (tbp != cq->next_to_clean) {
 463                                 if (cq->bi.rx_buff[tbp]) {
 464                                         cq->bi.rx_buff[ntp] =
 465                                                 cq->bi.rx_buff[tbp];
 466                                         cq->bi.rx_buff[tbp] = NULL;
 467
 468                                         /* Found a buffer, no need to
 469                                          * search anymore
 470                                          */
 471                                         break;
 472                                 }
 473
 474                                 /* Wrap ring if necessary */
 475                                 tbp++;
 476                                 if (tbp >= cq->ring_size)
 477                                         tbp = 0;
 478                         }
 479
 480                         if (tbp == cq->next_to_clean)
 481                                 goto post_buffs_out;
 482                 } else {
 483                         /* Give back pointer to DMA buffer */
 484                         cq->bi.rx_buff[ntp] = buffs[i];
 485                         i++;
 486
 487                         if (i >= *buff_count)
 488                                 buffs_avail = false;
 489                 }
 490
 491 fill_desc:
 492                 desc->flags =
 493                         cpu_to_le16(IDPF_CTLQ_FLAG_BUF | IDPF_CTLQ_FLAG_RD);
 494
 495                 /* Post buffers to descriptor */
 496                 desc->datalen = cpu_to_le16(cq->bi.rx_buff[ntp]->size);
 497                 desc->params.indirect.addr_high =
 498                         cpu_to_le32(upper_32_bits(cq->bi.rx_buff[ntp]->pa));
 499                 desc->params.indirect.addr_low =
 500                         cpu_to_le32(lower_32_bits(cq->bi.rx_buff[ntp]->pa));
 501
 502                 ntp++;
 503                 if (ntp == cq->ring_size)
 504                         ntp = 0;
 505         }
 506
 507 post_buffs_out:
 508         /* Only update tail if buffers were actually posted */
 509         if (cq->next_to_post != ntp) {
 510                 if (ntp)
 511                         /* Update next_to_post to ntp - 1 since current ntp
 512                          * will not have a buffer
 513                          */
 514                         cq->next_to_post = ntp - 1;
 515                 else
 516                         /* Wrap to end of end ring since current ntp is 0 */
 517                         cq->next_to_post = cq->ring_size - 1;
 518
 519                 dma_wmb();
 520
 521                 wr32(hw, cq->reg.tail, cq->next_to_post);
 522         }
 523
 524         mutex_unlock(&cq->cq_lock);
 525
 526         /* return the number of buffers that were not posted */
 527         *buff_count = *buff_count - i;
 528
 529         return 0;
 530 }
 531
 532 /**
 533  * idpf_ctlq_recv - receive control queue message call back
 534  * @cq: pointer to control queue handle to receive on
 535  * @num_q_msg: (input|output) input number of messages that should be received;
 536  * output number of messages actually received
 537  * @q_msg: (output) array of received control queue messages on this q;
 538  * needs to be pre-allocated by caller for as many messages as requested
 539  *
 540  * Called by interrupt handler or polling mechanism. Caller is expected
 541  * to free buffers
 542  */
 543 int idpf_ctlq_recv(struct idpf_ctlq_info *cq, u16 *num_q_msg,
 544                    struct idpf_ctlq_msg *q_msg)
 545 {
 546         u16 num_to_clean, ntc, flags;
 547         struct idpf_ctlq_desc *desc;
 548         int err = 0;
 549         u16 i;
 550
 551         /* take the lock before we start messing with the ring */
 552         mutex_lock(&cq->cq_lock);
 553
 554         ntc = cq->next_to_clean;
 555
 556         num_to_clean = *num_q_msg;
 557
 558         for (i = 0; i < num_to_clean; i++) {
 559                 /* Fetch next descriptor and check if marked as done */
 560                 desc = IDPF_CTLQ_DESC(cq, ntc);
 561                 flags = le16_to_cpu(desc->flags);
 562
 563                 if (!(flags & IDPF_CTLQ_FLAG_DD))
 564                         break;
 565
 566                 q_msg[i].vmvf_type = (flags &
 567                                       (IDPF_CTLQ_FLAG_FTYPE_VM |
 568                                        IDPF_CTLQ_FLAG_FTYPE_PF)) >>
 569                                        IDPF_CTLQ_FLAG_FTYPE_S;
 570
 571                 if (flags & IDPF_CTLQ_FLAG_ERR)
 572                         err  = -EBADMSG;
 573
 574                 q_msg[i].cookie.mbx.chnl_opcode =
 575                                 le32_to_cpu(desc->v_opcode_dtype);
 576                 q_msg[i].cookie.mbx.chnl_retval =
 577                                 le32_to_cpu(desc->v_retval);
 578
 579                 q_msg[i].opcode = le16_to_cpu(desc->opcode);
 580                 q_msg[i].data_len = le16_to_cpu(desc->datalen);
 581                 q_msg[i].status = le16_to_cpu(desc->ret_val);
 582
 583                 if (desc->datalen) {
 584                         memcpy(q_msg[i].ctx.indirect.context,
 585                                &desc->params.indirect, IDPF_INDIRECT_CTX_SIZE);
 586
 587                         /* Assign pointer to dma buffer to ctlq_msg array
 588                          * to be given to upper layer
 589                          */
 590                         q_msg[i].ctx.indirect.payload = cq->bi.rx_buff[ntc];
 591
 592                         /* Zero out pointer to DMA buffer info;
 593                          * will be repopulated by post buffers API
 594                          */
 595                         cq->bi.rx_buff[ntc] = NULL;
 596                 } else {
 597                         memcpy(q_msg[i].ctx.direct, desc->params.raw,
 598                                IDPF_DIRECT_CTX_SIZE);
 599                 }
 600
 601                 /* Zero out stale data in descriptor */
 602                 memset(desc, 0, sizeof(struct idpf_ctlq_desc));
 603
 604                 ntc++;
 605                 if (ntc == cq->ring_size)
 606                         ntc = 0;
 607         }
 608
 609         cq->next_to_clean = ntc;
 610
 611         mutex_unlock(&cq->cq_lock);
 612
 613         *num_q_msg = i;
 614         if (*num_q_msg == 0)
 615                 err = -ENOMSG;
 616
 617         return err;
 618 }