drivers/net/ethernet/pensando/ionic/ionic_dev.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/types.h>
   6 #include <linux/errno.h>
   7 #include <linux/io.h>
   8 #include <linux/slab.h>
   9 #include <linux/etherdevice.h>
  10 #include "ionic.h"
  11 #include "ionic_dev.h"
  12 #include "ionic_lif.h"
  13
  14 static void ionic_watchdog_cb(struct timer_list *t)
  15 {
  16         struct ionic *ionic = from_timer(ionic, t, watchdog_timer);
  17         struct ionic_lif *lif = ionic->lif;
  18         int hb;
  19
  20         mod_timer(&ionic->watchdog_timer,
  21                   round_jiffies(jiffies + ionic->watchdog_period));
  22
  23         if (!lif)
  24                 return;
  25
  26         hb = ionic_heartbeat_check(ionic);
  27         dev_dbg(ionic->dev, "%s: hb %d running %d UP %d\n",
  28                 __func__, hb, netif_running(lif->netdev),
  29                 test_bit(IONIC_LIF_F_UP, lif->state));
  30
  31         if (hb >= 0 &&
  32             !test_bit(IONIC_LIF_F_FW_RESET, lif->state))
  33                 ionic_link_status_check_request(lif, CAN_NOT_SLEEP);
  34 }
  35
  36 void ionic_init_devinfo(struct ionic *ionic)
  37 {
  38         struct ionic_dev *idev = &ionic->idev;
  39
  40         idev->dev_info.asic_type = ioread8(&idev->dev_info_regs->asic_type);
  41         idev->dev_info.asic_rev = ioread8(&idev->dev_info_regs->asic_rev);
  42
  43         memcpy_fromio(idev->dev_info.fw_version,
  44                       idev->dev_info_regs->fw_version,
  45                       IONIC_DEVINFO_FWVERS_BUFLEN);
  46
  47         memcpy_fromio(idev->dev_info.serial_num,
  48                       idev->dev_info_regs->serial_num,
  49                       IONIC_DEVINFO_SERIAL_BUFLEN);
  50
  51         idev->dev_info.fw_version[IONIC_DEVINFO_FWVERS_BUFLEN] = 0;
  52         idev->dev_info.serial_num[IONIC_DEVINFO_SERIAL_BUFLEN] = 0;
  53
  54         dev_dbg(ionic->dev, "fw_version %s\n", idev->dev_info.fw_version);
  55 }
  56
  57 int ionic_dev_setup(struct ionic *ionic)
  58 {
  59         struct ionic_dev_bar *bar = ionic->bars;
  60         unsigned int num_bars = ionic->num_bars;
  61         struct ionic_dev *idev = &ionic->idev;
  62         struct device *dev = ionic->dev;
  63         u32 sig;
  64
  65         /* BAR0: dev_cmd and interrupts */
  66         if (num_bars < 1) {
  67                 dev_err(dev, "No bars found, aborting\n");
  68                 return -EFAULT;
  69         }
  70
  71         if (bar->len < IONIC_BAR0_SIZE) {
  72                 dev_err(dev, "Resource bar size %lu too small, aborting\n",
  73                         bar->len);
  74                 return -EFAULT;
  75         }
  76
  77         idev->dev_info_regs = bar->vaddr + IONIC_BAR0_DEV_INFO_REGS_OFFSET;
  78         idev->dev_cmd_regs = bar->vaddr + IONIC_BAR0_DEV_CMD_REGS_OFFSET;
  79         idev->intr_status = bar->vaddr + IONIC_BAR0_INTR_STATUS_OFFSET;
  80         idev->intr_ctrl = bar->vaddr + IONIC_BAR0_INTR_CTRL_OFFSET;
  81
  82         idev->hwstamp_regs = &idev->dev_info_regs->hwstamp;
  83
  84         sig = ioread32(&idev->dev_info_regs->signature);
  85         if (sig != IONIC_DEV_INFO_SIGNATURE) {
  86                 dev_err(dev, "Incompatible firmware signature %x", sig);
  87                 return -EFAULT;
  88         }
  89
  90         ionic_init_devinfo(ionic);
  91
  92         /* BAR1: doorbells */
  93         bar++;
  94         if (num_bars < 2) {
  95                 dev_err(dev, "Doorbell bar missing, aborting\n");
  96                 return -EFAULT;
  97         }
  98
  99         timer_setup(&ionic->watchdog_timer, ionic_watchdog_cb, 0);
 100         ionic->watchdog_period = IONIC_WATCHDOG_SECS * HZ;
 101
 102         /* set times to ensure the first check will proceed */
 103         atomic_long_set(&idev->last_check_time, jiffies - 2 * HZ);
 104         idev->last_hb_time = jiffies - 2 * ionic->watchdog_period;
 105         /* init as ready, so no transition if the first check succeeds */
 106         idev->last_fw_hb = 0;
 107         idev->fw_hb_ready = true;
 108         idev->fw_status_ready = true;
 109
 110         mod_timer(&ionic->watchdog_timer,
 111                   round_jiffies(jiffies + ionic->watchdog_period));
 112
 113         idev->db_pages = bar->vaddr;
 114         idev->phy_db_pages = bar->bus_addr;
 115
 116         return 0;
 117 }
 118
 119 /* Devcmd Interface */
 120 int ionic_heartbeat_check(struct ionic *ionic)
 121 {
 122         struct ionic_dev *idev = &ionic->idev;
 123         unsigned long check_time, last_check_time;
 124         bool fw_status_ready, fw_hb_ready;
 125         u8 fw_status;
 126         u32 fw_hb;
 127
 128         /* wait a least one second before testing again */
 129         check_time = jiffies;
 130         last_check_time = atomic_long_read(&idev->last_check_time);
 131 do_check_time:
 132         if (time_before(check_time, last_check_time + HZ))
 133                 return 0;
 134         if (!atomic_long_try_cmpxchg_relaxed(&idev->last_check_time,
 135                                              &last_check_time, check_time)) {
 136                 /* if called concurrently, only the first should proceed. */
 137                 dev_dbg(ionic->dev, "%s: do_check_time again\n", __func__);
 138                 goto do_check_time;
 139         }
 140
 141         /* firmware is useful only if the running bit is set and
 142          * fw_status != 0xff (bad PCI read)
 143          */
 144         fw_status = ioread8(&idev->dev_info_regs->fw_status);
 145         fw_status_ready = (fw_status != 0xff) && (fw_status & IONIC_FW_STS_F_RUNNING);
 146
 147         /* is this a transition? */
 148         if (fw_status_ready != idev->fw_status_ready) {
 149                 struct ionic_lif *lif = ionic->lif;
 150                 bool trigger = false;
 151
 152                 idev->fw_status_ready = fw_status_ready;
 153
 154                 if (!fw_status_ready) {
 155                         dev_info(ionic->dev, "FW stopped %u\n", fw_status);
 156                         if (lif && !test_bit(IONIC_LIF_F_FW_RESET, lif->state))
 157                                 trigger = true;
 158                 } else {
 159                         dev_info(ionic->dev, "FW running %u\n", fw_status);
 160                         if (lif && test_bit(IONIC_LIF_F_FW_RESET, lif->state))
 161                                 trigger = true;
 162                 }
 163
 164                 if (trigger) {
 165                         struct ionic_deferred_work *work;
 166
 167                         work = kzalloc(sizeof(*work), GFP_ATOMIC);
 168                         if (work) {
 169                                 work->type = IONIC_DW_TYPE_LIF_RESET;
 170                                 work->fw_status = fw_status_ready;
 171                                 ionic_lif_deferred_enqueue(&lif->deferred, work);
 172                         }
 173                 }
 174         }
 175
 176         if (!fw_status_ready)
 177                 return -ENXIO;
 178
 179         /* wait at least one watchdog period since the last heartbeat */
 180         last_check_time = idev->last_hb_time;
 181         if (time_before(check_time, last_check_time + ionic->watchdog_period))
 182                 return 0;
 183
 184         fw_hb = ioread32(&idev->dev_info_regs->fw_heartbeat);
 185         fw_hb_ready = fw_hb != idev->last_fw_hb;
 186
 187         /* early FW version had no heartbeat, so fake it */
 188         if (!fw_hb_ready && !fw_hb)
 189                 fw_hb_ready = true;
 190
 191         dev_dbg(ionic->dev, "%s: fw_hb %u last_fw_hb %u ready %u\n",
 192                 __func__, fw_hb, idev->last_fw_hb, fw_hb_ready);
 193
 194         idev->last_fw_hb = fw_hb;
 195
 196         /* log a transition */
 197         if (fw_hb_ready != idev->fw_hb_ready) {
 198                 idev->fw_hb_ready = fw_hb_ready;
 199                 if (!fw_hb_ready)
 200                         dev_info(ionic->dev, "FW heartbeat stalled at %d\n", fw_hb);
 201                 else
 202                         dev_info(ionic->dev, "FW heartbeat restored at %d\n", fw_hb);
 203         }
 204
 205         if (!fw_hb_ready)
 206                 return -ENXIO;
 207
 208         idev->last_hb_time = check_time;
 209
 210         return 0;
 211 }
 212
 213 u8 ionic_dev_cmd_status(struct ionic_dev *idev)
 214 {
 215         return ioread8(&idev->dev_cmd_regs->comp.comp.status);
 216 }
 217
 218 bool ionic_dev_cmd_done(struct ionic_dev *idev)
 219 {
 220         return ioread32(&idev->dev_cmd_regs->done) & IONIC_DEV_CMD_DONE;
 221 }
 222
 223 void ionic_dev_cmd_comp(struct ionic_dev *idev, union ionic_dev_cmd_comp *comp)
 224 {
 225         memcpy_fromio(comp, &idev->dev_cmd_regs->comp, sizeof(*comp));
 226 }
 227
 228 void ionic_dev_cmd_go(struct ionic_dev *idev, union ionic_dev_cmd *cmd)
 229 {
 230         memcpy_toio(&idev->dev_cmd_regs->cmd, cmd, sizeof(*cmd));
 231         iowrite32(0, &idev->dev_cmd_regs->done);
 232         iowrite32(1, &idev->dev_cmd_regs->doorbell);
 233 }
 234
 235 /* Device commands */
 236 void ionic_dev_cmd_identify(struct ionic_dev *idev, u8 ver)
 237 {
 238         union ionic_dev_cmd cmd = {
 239                 .identify.opcode = IONIC_CMD_IDENTIFY,
 240                 .identify.ver = ver,
 241         };
 242
 243         ionic_dev_cmd_go(idev, &cmd);
 244 }
 245
 246 void ionic_dev_cmd_init(struct ionic_dev *idev)
 247 {
 248         union ionic_dev_cmd cmd = {
 249                 .init.opcode = IONIC_CMD_INIT,
 250                 .init.type = 0,
 251         };
 252
 253         ionic_dev_cmd_go(idev, &cmd);
 254 }
 255
 256 void ionic_dev_cmd_reset(struct ionic_dev *idev)
 257 {
 258         union ionic_dev_cmd cmd = {
 259                 .reset.opcode = IONIC_CMD_RESET,
 260         };
 261
 262         ionic_dev_cmd_go(idev, &cmd);
 263 }
 264
 265 /* Port commands */
 266 void ionic_dev_cmd_port_identify(struct ionic_dev *idev)
 267 {
 268         union ionic_dev_cmd cmd = {
 269                 .port_init.opcode = IONIC_CMD_PORT_IDENTIFY,
 270                 .port_init.index = 0,
 271         };
 272
 273         ionic_dev_cmd_go(idev, &cmd);
 274 }
 275
 276 void ionic_dev_cmd_port_init(struct ionic_dev *idev)
 277 {
 278         union ionic_dev_cmd cmd = {
 279                 .port_init.opcode = IONIC_CMD_PORT_INIT,
 280                 .port_init.index = 0,
 281                 .port_init.info_pa = cpu_to_le64(idev->port_info_pa),
 282         };
 283
 284         ionic_dev_cmd_go(idev, &cmd);
 285 }
 286
 287 void ionic_dev_cmd_port_reset(struct ionic_dev *idev)
 288 {
 289         union ionic_dev_cmd cmd = {
 290                 .port_reset.opcode = IONIC_CMD_PORT_RESET,
 291                 .port_reset.index = 0,
 292         };
 293
 294         ionic_dev_cmd_go(idev, &cmd);
 295 }
 296
 297 void ionic_dev_cmd_port_state(struct ionic_dev *idev, u8 state)
 298 {
 299         union ionic_dev_cmd cmd = {
 300                 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
 301                 .port_setattr.index = 0,
 302                 .port_setattr.attr = IONIC_PORT_ATTR_STATE,
 303                 .port_setattr.state = state,
 304         };
 305
 306         ionic_dev_cmd_go(idev, &cmd);
 307 }
 308
 309 void ionic_dev_cmd_port_speed(struct ionic_dev *idev, u32 speed)
 310 {
 311         union ionic_dev_cmd cmd = {
 312                 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
 313                 .port_setattr.index = 0,
 314                 .port_setattr.attr = IONIC_PORT_ATTR_SPEED,
 315                 .port_setattr.speed = cpu_to_le32(speed),
 316         };
 317
 318         ionic_dev_cmd_go(idev, &cmd);
 319 }
 320
 321 void ionic_dev_cmd_port_autoneg(struct ionic_dev *idev, u8 an_enable)
 322 {
 323         union ionic_dev_cmd cmd = {
 324                 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
 325                 .port_setattr.index = 0,
 326                 .port_setattr.attr = IONIC_PORT_ATTR_AUTONEG,
 327                 .port_setattr.an_enable = an_enable,
 328         };
 329
 330         ionic_dev_cmd_go(idev, &cmd);
 331 }
 332
 333 void ionic_dev_cmd_port_fec(struct ionic_dev *idev, u8 fec_type)
 334 {
 335         union ionic_dev_cmd cmd = {
 336                 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
 337                 .port_setattr.index = 0,
 338                 .port_setattr.attr = IONIC_PORT_ATTR_FEC,
 339                 .port_setattr.fec_type = fec_type,
 340         };
 341
 342         ionic_dev_cmd_go(idev, &cmd);
 343 }
 344
 345 void ionic_dev_cmd_port_pause(struct ionic_dev *idev, u8 pause_type)
 346 {
 347         union ionic_dev_cmd cmd = {
 348                 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
 349                 .port_setattr.index = 0,
 350                 .port_setattr.attr = IONIC_PORT_ATTR_PAUSE,
 351                 .port_setattr.pause_type = pause_type,
 352         };
 353
 354         ionic_dev_cmd_go(idev, &cmd);
 355 }
 356
 357 /* VF commands */
 358 int ionic_set_vf_config(struct ionic *ionic, int vf, u8 attr, u8 *data)
 359 {
 360         union ionic_dev_cmd cmd = {
 361                 .vf_setattr.opcode = IONIC_CMD_VF_SETATTR,
 362                 .vf_setattr.attr = attr,
 363                 .vf_setattr.vf_index = cpu_to_le16(vf),
 364         };
 365         int err;
 366
 367         switch (attr) {
 368         case IONIC_VF_ATTR_SPOOFCHK:
 369                 cmd.vf_setattr.spoofchk = *data;
 370                 dev_dbg(ionic->dev, "%s: vf %d spoof %d\n",
 371                         __func__, vf, *data);
 372                 break;
 373         case IONIC_VF_ATTR_TRUST:
 374                 cmd.vf_setattr.trust = *data;
 375                 dev_dbg(ionic->dev, "%s: vf %d trust %d\n",
 376                         __func__, vf, *data);
 377                 break;
 378         case IONIC_VF_ATTR_LINKSTATE:
 379                 cmd.vf_setattr.linkstate = *data;
 380                 dev_dbg(ionic->dev, "%s: vf %d linkstate %d\n",
 381                         __func__, vf, *data);
 382                 break;
 383         case IONIC_VF_ATTR_MAC:
 384                 ether_addr_copy(cmd.vf_setattr.macaddr, data);
 385                 dev_dbg(ionic->dev, "%s: vf %d macaddr %pM\n",
 386                         __func__, vf, data);
 387                 break;
 388         case IONIC_VF_ATTR_VLAN:
 389                 cmd.vf_setattr.vlanid = cpu_to_le16(*(u16 *)data);
 390                 dev_dbg(ionic->dev, "%s: vf %d vlan %d\n",
 391                         __func__, vf, *(u16 *)data);
 392                 break;
 393         case IONIC_VF_ATTR_RATE:
 394                 cmd.vf_setattr.maxrate = cpu_to_le32(*(u32 *)data);
 395                 dev_dbg(ionic->dev, "%s: vf %d maxrate %d\n",
 396                         __func__, vf, *(u32 *)data);
 397                 break;
 398         case IONIC_VF_ATTR_STATSADDR:
 399                 cmd.vf_setattr.stats_pa = cpu_to_le64(*(u64 *)data);
 400                 dev_dbg(ionic->dev, "%s: vf %d stats_pa 0x%08llx\n",
 401                         __func__, vf, *(u64 *)data);
 402                 break;
 403         default:
 404                 return -EINVAL;
 405         }
 406
 407         mutex_lock(&ionic->dev_cmd_lock);
 408         ionic_dev_cmd_go(&ionic->idev, &cmd);
 409         err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
 410         mutex_unlock(&ionic->dev_cmd_lock);
 411
 412         return err;
 413 }
 414
 415 /* LIF commands */
 416 void ionic_dev_cmd_queue_identify(struct ionic_dev *idev,
 417                                   u16 lif_type, u8 qtype, u8 qver)
 418 {
 419         union ionic_dev_cmd cmd = {
 420                 .q_identify.opcode = IONIC_CMD_Q_IDENTIFY,
 421                 .q_identify.lif_type = cpu_to_le16(lif_type),
 422                 .q_identify.type = qtype,
 423                 .q_identify.ver = qver,
 424         };
 425
 426         ionic_dev_cmd_go(idev, &cmd);
 427 }
 428
 429 void ionic_dev_cmd_lif_identify(struct ionic_dev *idev, u8 type, u8 ver)
 430 {
 431         union ionic_dev_cmd cmd = {
 432                 .lif_identify.opcode = IONIC_CMD_LIF_IDENTIFY,
 433                 .lif_identify.type = type,
 434                 .lif_identify.ver = ver,
 435         };
 436
 437         ionic_dev_cmd_go(idev, &cmd);
 438 }
 439
 440 void ionic_dev_cmd_lif_init(struct ionic_dev *idev, u16 lif_index,
 441                             dma_addr_t info_pa)
 442 {
 443         union ionic_dev_cmd cmd = {
 444                 .lif_init.opcode = IONIC_CMD_LIF_INIT,
 445                 .lif_init.index = cpu_to_le16(lif_index),
 446                 .lif_init.info_pa = cpu_to_le64(info_pa),
 447         };
 448
 449         ionic_dev_cmd_go(idev, &cmd);
 450 }
 451
 452 void ionic_dev_cmd_lif_reset(struct ionic_dev *idev, u16 lif_index)
 453 {
 454         union ionic_dev_cmd cmd = {
 455                 .lif_init.opcode = IONIC_CMD_LIF_RESET,
 456                 .lif_init.index = cpu_to_le16(lif_index),
 457         };
 458
 459         ionic_dev_cmd_go(idev, &cmd);
 460 }
 461
 462 void ionic_dev_cmd_adminq_init(struct ionic_dev *idev, struct ionic_qcq *qcq,
 463                                u16 lif_index, u16 intr_index)
 464 {
 465         struct ionic_queue *q = &qcq->q;
 466         struct ionic_cq *cq = &qcq->cq;
 467
 468         union ionic_dev_cmd cmd = {
 469                 .q_init.opcode = IONIC_CMD_Q_INIT,
 470                 .q_init.lif_index = cpu_to_le16(lif_index),
 471                 .q_init.type = q->type,
 472                 .q_init.ver = qcq->q.lif->qtype_info[q->type].version,
 473                 .q_init.index = cpu_to_le32(q->index),
 474                 .q_init.flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
 475                                             IONIC_QINIT_F_ENA),
 476                 .q_init.pid = cpu_to_le16(q->pid),
 477                 .q_init.intr_index = cpu_to_le16(intr_index),
 478                 .q_init.ring_size = ilog2(q->num_descs),
 479                 .q_init.ring_base = cpu_to_le64(q->base_pa),
 480                 .q_init.cq_ring_base = cpu_to_le64(cq->base_pa),
 481         };
 482
 483         ionic_dev_cmd_go(idev, &cmd);
 484 }
 485
 486 int ionic_db_page_num(struct ionic_lif *lif, int pid)
 487 {
 488         return (lif->hw_index * lif->dbid_count) + pid;
 489 }
 490
 491 int ionic_cq_init(struct ionic_lif *lif, struct ionic_cq *cq,
 492                   struct ionic_intr_info *intr,
 493                   unsigned int num_descs, size_t desc_size)
 494 {
 495         unsigned int ring_size;
 496
 497         if (desc_size == 0 || !is_power_of_2(num_descs))
 498                 return -EINVAL;
 499
 500         ring_size = ilog2(num_descs);
 501         if (ring_size < 2 || ring_size > 16)
 502                 return -EINVAL;
 503
 504         cq->lif = lif;
 505         cq->bound_intr = intr;
 506         cq->num_descs = num_descs;
 507         cq->desc_size = desc_size;
 508         cq->tail_idx = 0;
 509         cq->done_color = 1;
 510
 511         return 0;
 512 }
 513
 514 void ionic_cq_map(struct ionic_cq *cq, void *base, dma_addr_t base_pa)
 515 {
 516         struct ionic_cq_info *cur;
 517         unsigned int i;
 518
 519         cq->base = base;
 520         cq->base_pa = base_pa;
 521
 522         for (i = 0, cur = cq->info; i < cq->num_descs; i++, cur++)
 523                 cur->cq_desc = base + (i * cq->desc_size);
 524 }
 525
 526 void ionic_cq_bind(struct ionic_cq *cq, struct ionic_queue *q)
 527 {
 528         cq->bound_q = q;
 529 }
 530
 531 unsigned int ionic_cq_service(struct ionic_cq *cq, unsigned int work_to_do,
 532                               ionic_cq_cb cb, ionic_cq_done_cb done_cb,
 533                               void *done_arg)
 534 {
 535         struct ionic_cq_info *cq_info;
 536         unsigned int work_done = 0;
 537
 538         if (work_to_do == 0)
 539                 return 0;
 540
 541         cq_info = &cq->info[cq->tail_idx];
 542         while (cb(cq, cq_info)) {
 543                 if (cq->tail_idx == cq->num_descs - 1)
 544                         cq->done_color = !cq->done_color;
 545                 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
 546                 cq_info = &cq->info[cq->tail_idx];
 547                 DEBUG_STATS_CQE_CNT(cq);
 548
 549                 if (++work_done >= work_to_do)
 550                         break;
 551         }
 552
 553         if (work_done && done_cb)
 554                 done_cb(done_arg);
 555
 556         return work_done;
 557 }
 558
 559 int ionic_q_init(struct ionic_lif *lif, struct ionic_dev *idev,
 560                  struct ionic_queue *q, unsigned int index, const char *name,
 561                  unsigned int num_descs, size_t desc_size,
 562                  size_t sg_desc_size, unsigned int pid)
 563 {
 564         unsigned int ring_size;
 565
 566         if (desc_size == 0 || !is_power_of_2(num_descs))
 567                 return -EINVAL;
 568
 569         ring_size = ilog2(num_descs);
 570         if (ring_size < 2 || ring_size > 16)
 571                 return -EINVAL;
 572
 573         q->lif = lif;
 574         q->idev = idev;
 575         q->index = index;
 576         q->num_descs = num_descs;
 577         q->desc_size = desc_size;
 578         q->sg_desc_size = sg_desc_size;
 579         q->tail_idx = 0;
 580         q->head_idx = 0;
 581         q->pid = pid;
 582
 583         snprintf(q->name, sizeof(q->name), "L%d-%s%u", lif->index, name, index);
 584
 585         return 0;
 586 }
 587
 588 void ionic_q_map(struct ionic_queue *q, void *base, dma_addr_t base_pa)
 589 {
 590         struct ionic_desc_info *cur;
 591         unsigned int i;
 592
 593         q->base = base;
 594         q->base_pa = base_pa;
 595
 596         for (i = 0, cur = q->info; i < q->num_descs; i++, cur++)
 597                 cur->desc = base + (i * q->desc_size);
 598 }
 599
 600 void ionic_q_sg_map(struct ionic_queue *q, void *base, dma_addr_t base_pa)
 601 {
 602         struct ionic_desc_info *cur;
 603         unsigned int i;
 604
 605         q->sg_base = base;
 606         q->sg_base_pa = base_pa;
 607
 608         for (i = 0, cur = q->info; i < q->num_descs; i++, cur++)
 609                 cur->sg_desc = base + (i * q->sg_desc_size);
 610 }
 611
 612 void ionic_q_post(struct ionic_queue *q, bool ring_doorbell, ionic_desc_cb cb,
 613                   void *cb_arg)
 614 {
 615         struct ionic_desc_info *desc_info;
 616         struct ionic_lif *lif = q->lif;
 617         struct device *dev = q->dev;
 618
 619         desc_info = &q->info[q->head_idx];
 620         desc_info->cb = cb;
 621         desc_info->cb_arg = cb_arg;
 622
 623         q->head_idx = (q->head_idx + 1) & (q->num_descs - 1);
 624
 625         dev_dbg(dev, "lif=%d qname=%s qid=%d qtype=%d p_index=%d ringdb=%d\n",
 626                 q->lif->index, q->name, q->hw_type, q->hw_index,
 627                 q->head_idx, ring_doorbell);
 628
 629         if (ring_doorbell)
 630                 ionic_dbell_ring(lif->kern_dbpage, q->hw_type,
 631                                  q->dbval | q->head_idx);
 632 }
 633
 634 static bool ionic_q_is_posted(struct ionic_queue *q, unsigned int pos)
 635 {
 636         unsigned int mask, tail, head;
 637
 638         mask = q->num_descs - 1;
 639         tail = q->tail_idx;
 640         head = q->head_idx;
 641
 642         return ((pos - tail) & mask) < ((head - tail) & mask);
 643 }
 644
 645 void ionic_q_service(struct ionic_queue *q, struct ionic_cq_info *cq_info,
 646                      unsigned int stop_index)
 647 {
 648         struct ionic_desc_info *desc_info;
 649         ionic_desc_cb cb;
 650         void *cb_arg;
 651         u16 index;
 652
 653         /* check for empty queue */
 654         if (q->tail_idx == q->head_idx)
 655                 return;
 656
 657         /* stop index must be for a descriptor that is not yet completed */
 658         if (unlikely(!ionic_q_is_posted(q, stop_index)))
 659                 dev_err(q->dev,
 660                         "ionic stop is not posted %s stop %u tail %u head %u\n",
 661                         q->name, stop_index, q->tail_idx, q->head_idx);
 662
 663         do {
 664                 desc_info = &q->info[q->tail_idx];
 665                 index = q->tail_idx;
 666                 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
 667
 668                 cb = desc_info->cb;
 669                 cb_arg = desc_info->cb_arg;
 670
 671                 desc_info->cb = NULL;
 672                 desc_info->cb_arg = NULL;
 673
 674                 if (cb)
 675                         cb(q, desc_info, cq_info, cb_arg);
 676         } while (index != stop_index);
 677 }