drivers/vdpa/solidrun/snet_main.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * SolidRun DPU driver for control plane
   4  *
   5  * Copyright (C) 2022-2023 SolidRun
   6  *
   7  * Author: Alvaro Karsz <[email protected]>
   8  *
   9  */
  10 #include <linux/iopoll.h>
  11
  12 #include "snet_vdpa.h"
  13
  14 /* SNET DPU device ID */
  15 #define SNET_DEVICE_ID          0x1000
  16 /* SNET signature */
  17 #define SNET_SIGNATURE          0xD0D06363
  18 /* Max. config version that we can work with */
  19 #define SNET_CFG_VERSION        0x2
  20 /* Queue align */
  21 #define SNET_QUEUE_ALIGNMENT    PAGE_SIZE
  22 /* Kick value to notify that new data is available */
  23 #define SNET_KICK_VAL           0x1
  24 #define SNET_CONFIG_OFF         0x0
  25 /* How long we are willing to wait for a SNET device */
  26 #define SNET_DETECT_TIMEOUT     5000000
  27 /* How long should we wait for the DPU to read our config */
  28 #define SNET_READ_CFG_TIMEOUT   3000000
  29 /* Size of configs written to the DPU */
  30 #define SNET_GENERAL_CFG_LEN    36
  31 #define SNET_GENERAL_CFG_VQ_LEN 40
  32
  33 static struct snet *vdpa_to_snet(struct vdpa_device *vdpa)
  34 {
  35         return container_of(vdpa, struct snet, vdpa);
  36 }
  37
  38 static irqreturn_t snet_cfg_irq_hndlr(int irq, void *data)
  39 {
  40         struct snet *snet = data;
  41         /* Call callback if any */
  42         if (likely(snet->cb.callback))
  43                 return snet->cb.callback(snet->cb.private);
  44
  45         return IRQ_HANDLED;
  46 }
  47
  48 static irqreturn_t snet_vq_irq_hndlr(int irq, void *data)
  49 {
  50         struct snet_vq *vq = data;
  51         /* Call callback if any */
  52         if (likely(vq->cb.callback))
  53                 return vq->cb.callback(vq->cb.private);
  54
  55         return IRQ_HANDLED;
  56 }
  57
  58 static void snet_free_irqs(struct snet *snet)
  59 {
  60         struct psnet *psnet = snet->psnet;
  61         struct pci_dev *pdev;
  62         u32 i;
  63
  64         /* Which Device allcoated the IRQs? */
  65         if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF))
  66                 pdev = snet->pdev->physfn;
  67         else
  68                 pdev = snet->pdev;
  69
  70         /* Free config's IRQ */
  71         if (snet->cfg_irq != -1) {
  72                 devm_free_irq(&pdev->dev, snet->cfg_irq, snet);
  73                 snet->cfg_irq = -1;
  74         }
  75         /* Free VQ IRQs */
  76         for (i = 0; i < snet->cfg->vq_num; i++) {
  77                 if (snet->vqs[i] && snet->vqs[i]->irq != -1) {
  78                         devm_free_irq(&pdev->dev, snet->vqs[i]->irq, snet->vqs[i]);
  79                         snet->vqs[i]->irq = -1;
  80                 }
  81         }
  82
  83         /* IRQ vectors are freed when the pci remove callback is called */
  84 }
  85
  86 static int snet_set_vq_address(struct vdpa_device *vdev, u16 idx, u64 desc_area,
  87                                u64 driver_area, u64 device_area)
  88 {
  89         struct snet *snet = vdpa_to_snet(vdev);
  90         /* save received parameters in vqueue sturct */
  91         snet->vqs[idx]->desc_area = desc_area;
  92         snet->vqs[idx]->driver_area = driver_area;
  93         snet->vqs[idx]->device_area = device_area;
  94
  95         return 0;
  96 }
  97
  98 static void snet_set_vq_num(struct vdpa_device *vdev, u16 idx, u32 num)
  99 {
 100         struct snet *snet = vdpa_to_snet(vdev);
 101         /* save num in vqueue */
 102         snet->vqs[idx]->num = num;
 103 }
 104
 105 static void snet_kick_vq(struct vdpa_device *vdev, u16 idx)
 106 {
 107         struct snet *snet = vdpa_to_snet(vdev);
 108         /* not ready - ignore */
 109         if (unlikely(!snet->vqs[idx]->ready))
 110                 return;
 111
 112         iowrite32(SNET_KICK_VAL, snet->vqs[idx]->kick_ptr);
 113 }
 114
 115 static void snet_kick_vq_with_data(struct vdpa_device *vdev, u32 data)
 116 {
 117         struct snet *snet = vdpa_to_snet(vdev);
 118         u16 idx = data & 0xFFFF;
 119
 120         /* not ready - ignore */
 121         if (unlikely(!snet->vqs[idx]->ready))
 122                 return;
 123
 124         iowrite32((data & 0xFFFF0000) | SNET_KICK_VAL, snet->vqs[idx]->kick_ptr);
 125 }
 126
 127 static void snet_set_vq_cb(struct vdpa_device *vdev, u16 idx, struct vdpa_callback *cb)
 128 {
 129         struct snet *snet = vdpa_to_snet(vdev);
 130
 131         snet->vqs[idx]->cb.callback = cb->callback;
 132         snet->vqs[idx]->cb.private = cb->private;
 133 }
 134
 135 static void snet_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready)
 136 {
 137         struct snet *snet = vdpa_to_snet(vdev);
 138
 139         snet->vqs[idx]->ready = ready;
 140 }
 141
 142 static bool snet_get_vq_ready(struct vdpa_device *vdev, u16 idx)
 143 {
 144         struct snet *snet = vdpa_to_snet(vdev);
 145
 146         return snet->vqs[idx]->ready;
 147 }
 148
 149 static bool snet_vq_state_is_initial(struct snet *snet, const struct vdpa_vq_state *state)
 150 {
 151         if (SNET_HAS_FEATURE(snet, VIRTIO_F_RING_PACKED)) {
 152                 const struct vdpa_vq_state_packed *p = &state->packed;
 153
 154                 if (p->last_avail_counter == 1 && p->last_used_counter == 1 &&
 155                     p->last_avail_idx == 0 && p->last_used_idx == 0)
 156                         return true;
 157         } else {
 158                 const struct vdpa_vq_state_split *s = &state->split;
 159
 160                 if (s->avail_index == 0)
 161                         return true;
 162         }
 163
 164         return false;
 165 }
 166
 167 static int snet_set_vq_state(struct vdpa_device *vdev, u16 idx, const struct vdpa_vq_state *state)
 168 {
 169         struct snet *snet = vdpa_to_snet(vdev);
 170
 171         /* We can set any state for config version 2+ */
 172         if (SNET_CFG_VER(snet, 2)) {
 173                 memcpy(&snet->vqs[idx]->vq_state, state, sizeof(*state));
 174                 return 0;
 175         }
 176
 177         /* Older config - we can't set the VQ state.
 178          * Return 0 only if this is the initial state we use in the DPU.
 179          */
 180         if (snet_vq_state_is_initial(snet, state))
 181                 return 0;
 182
 183         return -EOPNOTSUPP;
 184 }
 185
 186 static int snet_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa_vq_state *state)
 187 {
 188         struct snet *snet = vdpa_to_snet(vdev);
 189
 190         return snet_read_vq_state(snet, idx, state);
 191 }
 192
 193 static int snet_get_vq_irq(struct vdpa_device *vdev, u16 idx)
 194 {
 195         struct snet *snet = vdpa_to_snet(vdev);
 196
 197         return snet->vqs[idx]->irq;
 198 }
 199
 200 static u32 snet_get_vq_align(struct vdpa_device *vdev)
 201 {
 202         return (u32)SNET_QUEUE_ALIGNMENT;
 203 }
 204
 205 static int snet_reset_dev(struct snet *snet)
 206 {
 207         struct pci_dev *pdev = snet->pdev;
 208         int ret = 0;
 209         u32 i;
 210
 211         /* If status is 0, nothing to do */
 212         if (!snet->status)
 213                 return 0;
 214
 215         /* If DPU started, destroy it */
 216         if (snet->status & VIRTIO_CONFIG_S_DRIVER_OK)
 217                 ret = snet_destroy_dev(snet);
 218
 219         /* Clear VQs */
 220         for (i = 0; i < snet->cfg->vq_num; i++) {
 221                 if (!snet->vqs[i])
 222                         continue;
 223                 snet->vqs[i]->cb.callback = NULL;
 224                 snet->vqs[i]->cb.private = NULL;
 225                 snet->vqs[i]->desc_area = 0;
 226                 snet->vqs[i]->device_area = 0;
 227                 snet->vqs[i]->driver_area = 0;
 228                 snet->vqs[i]->ready = false;
 229         }
 230
 231         /* Clear config callback */
 232         snet->cb.callback = NULL;
 233         snet->cb.private = NULL;
 234         /* Free IRQs */
 235         snet_free_irqs(snet);
 236         /* Reset status */
 237         snet->status = 0;
 238         snet->dpu_ready = false;
 239
 240         if (ret)
 241                 SNET_WARN(pdev, "Incomplete reset to SNET[%u] device, err: %d\n", snet->sid, ret);
 242         else
 243                 SNET_DBG(pdev, "Reset SNET[%u] device\n", snet->sid);
 244
 245         return 0;
 246 }
 247
 248 static int snet_reset(struct vdpa_device *vdev)
 249 {
 250         struct snet *snet = vdpa_to_snet(vdev);
 251
 252         return snet_reset_dev(snet);
 253 }
 254
 255 static size_t snet_get_config_size(struct vdpa_device *vdev)
 256 {
 257         struct snet *snet = vdpa_to_snet(vdev);
 258
 259         return (size_t)snet->cfg->cfg_size;
 260 }
 261
 262 static u64 snet_get_features(struct vdpa_device *vdev)
 263 {
 264         struct snet *snet = vdpa_to_snet(vdev);
 265
 266         return snet->cfg->features;
 267 }
 268
 269 static int snet_set_drv_features(struct vdpa_device *vdev, u64 features)
 270 {
 271         struct snet *snet = vdpa_to_snet(vdev);
 272
 273         snet->negotiated_features = snet->cfg->features & features;
 274         return 0;
 275 }
 276
 277 static u64 snet_get_drv_features(struct vdpa_device *vdev)
 278 {
 279         struct snet *snet = vdpa_to_snet(vdev);
 280
 281         return snet->negotiated_features;
 282 }
 283
 284 static u16 snet_get_vq_num_max(struct vdpa_device *vdev)
 285 {
 286         struct snet *snet = vdpa_to_snet(vdev);
 287
 288         return (u16)snet->cfg->vq_size;
 289 }
 290
 291 static void snet_set_config_cb(struct vdpa_device *vdev, struct vdpa_callback *cb)
 292 {
 293         struct snet *snet = vdpa_to_snet(vdev);
 294
 295         snet->cb.callback = cb->callback;
 296         snet->cb.private = cb->private;
 297 }
 298
 299 static u32 snet_get_device_id(struct vdpa_device *vdev)
 300 {
 301         struct snet *snet = vdpa_to_snet(vdev);
 302
 303         return snet->cfg->virtio_id;
 304 }
 305
 306 static u32 snet_get_vendor_id(struct vdpa_device *vdev)
 307 {
 308         return (u32)PCI_VENDOR_ID_SOLIDRUN;
 309 }
 310
 311 static u8 snet_get_status(struct vdpa_device *vdev)
 312 {
 313         struct snet *snet = vdpa_to_snet(vdev);
 314
 315         return snet->status;
 316 }
 317
 318 static int snet_write_conf(struct snet *snet)
 319 {
 320         u32 off, i, tmp;
 321         int ret;
 322
 323         /* No need to write the config twice */
 324         if (snet->dpu_ready)
 325                 return true;
 326
 327         /* Snet data :
 328          *
 329          * General data: SNET_GENERAL_CFG_LEN bytes long
 330          *  0             0x4       0x8        0xC               0x10      0x14        0x1C     0x24
 331          *  | MAGIC NUMBER | CFG VER | SNET SID | NUMBER OF QUEUES | IRQ IDX | FEATURES |  RSVD  |
 332          *
 333          * For every VQ: SNET_GENERAL_CFG_VQ_LEN bytes long
 334          * 0                          0x4        0x8
 335          * |  VQ SID  AND  QUEUE SIZE | IRQ Index |
 336          * |             DESC AREA                |
 337          * |            DEVICE AREA               |
 338          * |            DRIVER AREA               |
 339          * |    VQ STATE (CFG 2+)     |   RSVD    |
 340          *
 341          * Magic number should be written last, this is the DPU indication that the data is ready
 342          */
 343
 344         /* Init offset */
 345         off = snet->psnet->cfg.host_cfg_off;
 346
 347         /* Ignore magic number for now */
 348         off += 4;
 349         snet_write32(snet, off, snet->psnet->negotiated_cfg_ver);
 350         off += 4;
 351         snet_write32(snet, off, snet->sid);
 352         off += 4;
 353         snet_write32(snet, off, snet->cfg->vq_num);
 354         off += 4;
 355         snet_write32(snet, off, snet->cfg_irq_idx);
 356         off += 4;
 357         snet_write64(snet, off, snet->negotiated_features);
 358         off += 8;
 359         /* Ignore reserved */
 360         off += 8;
 361         /* Write VQs */
 362         for (i = 0 ; i < snet->cfg->vq_num ; i++) {
 363                 tmp = (i << 16) | (snet->vqs[i]->num & 0xFFFF);
 364                 snet_write32(snet, off, tmp);
 365                 off += 4;
 366                 snet_write32(snet, off, snet->vqs[i]->irq_idx);
 367                 off += 4;
 368                 snet_write64(snet, off, snet->vqs[i]->desc_area);
 369                 off += 8;
 370                 snet_write64(snet, off, snet->vqs[i]->device_area);
 371                 off += 8;
 372                 snet_write64(snet, off, snet->vqs[i]->driver_area);
 373                 off += 8;
 374                 /* Write VQ state if config version is 2+ */
 375                 if (SNET_CFG_VER(snet, 2))
 376                         snet_write32(snet, off, *(u32 *)&snet->vqs[i]->vq_state);
 377                 off += 4;
 378
 379                 /* Ignore reserved */
 380                 off += 4;
 381         }
 382
 383         /* Write magic number - data is ready */
 384         snet_write32(snet, snet->psnet->cfg.host_cfg_off, SNET_SIGNATURE);
 385
 386         /* The DPU will ACK the config by clearing the signature */
 387         ret = readx_poll_timeout(ioread32, snet->bar + snet->psnet->cfg.host_cfg_off,
 388                                  tmp, !tmp, 10, SNET_READ_CFG_TIMEOUT);
 389         if (ret) {
 390                 SNET_ERR(snet->pdev, "Timeout waiting for the DPU to read the config\n");
 391                 return false;
 392         }
 393
 394         /* set DPU flag */
 395         snet->dpu_ready = true;
 396
 397         return true;
 398 }
 399
 400 static int snet_request_irqs(struct pci_dev *pdev, struct snet *snet)
 401 {
 402         int ret, i, irq;
 403
 404         /* Request config IRQ */
 405         irq = pci_irq_vector(pdev, snet->cfg_irq_idx);
 406         ret = devm_request_irq(&pdev->dev, irq, snet_cfg_irq_hndlr, 0,
 407                                snet->cfg_irq_name, snet);
 408         if (ret) {
 409                 SNET_ERR(pdev, "Failed to request IRQ\n");
 410                 return ret;
 411         }
 412         snet->cfg_irq = irq;
 413
 414         /* Request IRQ for every VQ */
 415         for (i = 0; i < snet->cfg->vq_num; i++) {
 416                 irq = pci_irq_vector(pdev, snet->vqs[i]->irq_idx);
 417                 ret = devm_request_irq(&pdev->dev, irq, snet_vq_irq_hndlr, 0,
 418                                        snet->vqs[i]->irq_name, snet->vqs[i]);
 419                 if (ret) {
 420                         SNET_ERR(pdev, "Failed to request IRQ\n");
 421                         return ret;
 422                 }
 423                 snet->vqs[i]->irq = irq;
 424         }
 425         return 0;
 426 }
 427
 428 static void snet_set_status(struct vdpa_device *vdev, u8 status)
 429 {
 430         struct snet *snet = vdpa_to_snet(vdev);
 431         struct psnet *psnet = snet->psnet;
 432         struct pci_dev *pdev = snet->pdev;
 433         int ret;
 434         bool pf_irqs;
 435
 436         if (status == snet->status)
 437                 return;
 438
 439         if ((status & VIRTIO_CONFIG_S_DRIVER_OK) &&
 440             !(snet->status & VIRTIO_CONFIG_S_DRIVER_OK)) {
 441                 /* Request IRQs */
 442                 pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF);
 443                 ret = snet_request_irqs(pf_irqs ? pdev->physfn : pdev, snet);
 444                 if (ret)
 445                         goto set_err;
 446
 447                 /* Write config to the DPU */
 448                 if (snet_write_conf(snet)) {
 449                         SNET_INFO(pdev, "Create SNET[%u] device\n", snet->sid);
 450                 } else {
 451                         snet_free_irqs(snet);
 452                         goto set_err;
 453                 }
 454         }
 455
 456         /* Save the new status */
 457         snet->status = status;
 458         return;
 459
 460 set_err:
 461         snet->status |= VIRTIO_CONFIG_S_FAILED;
 462 }
 463
 464 static void snet_get_config(struct vdpa_device *vdev, unsigned int offset,
 465                             void *buf, unsigned int len)
 466 {
 467         struct snet *snet = vdpa_to_snet(vdev);
 468         void __iomem *cfg_ptr = snet->cfg->virtio_cfg + offset;
 469         u8 *buf_ptr = buf;
 470         u32 i;
 471
 472         /* check for offset error */
 473         if (offset + len > snet->cfg->cfg_size)
 474                 return;
 475
 476         /* Write into buffer */
 477         for (i = 0; i < len; i++)
 478                 *buf_ptr++ = ioread8(cfg_ptr + i);
 479 }
 480
 481 static void snet_set_config(struct vdpa_device *vdev, unsigned int offset,
 482                             const void *buf, unsigned int len)
 483 {
 484         struct snet *snet = vdpa_to_snet(vdev);
 485         void __iomem *cfg_ptr = snet->cfg->virtio_cfg + offset;
 486         const u8 *buf_ptr = buf;
 487         u32 i;
 488
 489         /* check for offset error */
 490         if (offset + len > snet->cfg->cfg_size)
 491                 return;
 492
 493         /* Write into PCI BAR */
 494         for (i = 0; i < len; i++)
 495                 iowrite8(*buf_ptr++, cfg_ptr + i);
 496 }
 497
 498 static int snet_suspend(struct vdpa_device *vdev)
 499 {
 500         struct snet *snet = vdpa_to_snet(vdev);
 501         int ret;
 502
 503         ret = snet_suspend_dev(snet);
 504         if (ret)
 505                 SNET_ERR(snet->pdev, "SNET[%u] suspend failed, err: %d\n", snet->sid, ret);
 506         else
 507                 SNET_DBG(snet->pdev, "Suspend SNET[%u] device\n", snet->sid);
 508
 509         return ret;
 510 }
 511
 512 static int snet_resume(struct vdpa_device *vdev)
 513 {
 514         struct snet *snet = vdpa_to_snet(vdev);
 515         int ret;
 516
 517         ret = snet_resume_dev(snet);
 518         if (ret)
 519                 SNET_ERR(snet->pdev, "SNET[%u] resume failed, err: %d\n", snet->sid, ret);
 520         else
 521                 SNET_DBG(snet->pdev, "Resume SNET[%u] device\n", snet->sid);
 522
 523         return ret;
 524 }
 525
 526 static const struct vdpa_config_ops snet_config_ops = {
 527         .set_vq_address         = snet_set_vq_address,
 528         .set_vq_num             = snet_set_vq_num,
 529         .kick_vq                = snet_kick_vq,
 530         .kick_vq_with_data      = snet_kick_vq_with_data,
 531         .set_vq_cb              = snet_set_vq_cb,
 532         .set_vq_ready           = snet_set_vq_ready,
 533         .get_vq_ready           = snet_get_vq_ready,
 534         .set_vq_state           = snet_set_vq_state,
 535         .get_vq_state           = snet_get_vq_state,
 536         .get_vq_irq             = snet_get_vq_irq,
 537         .get_vq_align           = snet_get_vq_align,
 538         .reset                  = snet_reset,
 539         .get_config_size        = snet_get_config_size,
 540         .get_device_features    = snet_get_features,
 541         .set_driver_features    = snet_set_drv_features,
 542         .get_driver_features    = snet_get_drv_features,
 543         .get_vq_num_min         = snet_get_vq_num_max,
 544         .get_vq_num_max         = snet_get_vq_num_max,
 545         .set_config_cb          = snet_set_config_cb,
 546         .get_device_id          = snet_get_device_id,
 547         .get_vendor_id          = snet_get_vendor_id,
 548         .get_status             = snet_get_status,
 549         .set_status             = snet_set_status,
 550         .get_config             = snet_get_config,
 551         .set_config             = snet_set_config,
 552         .suspend                = snet_suspend,
 553         .resume                 = snet_resume,
 554 };
 555
 556 static int psnet_open_pf_bar(struct pci_dev *pdev, struct psnet *psnet)
 557 {
 558         char *name;
 559         int ret, i, mask = 0;
 560         /* We don't know which BAR will be used to communicate..
 561          * We will map every bar with len > 0.
 562          *
 563          * Later, we will discover the BAR and unmap all other BARs.
 564          */
 565         for (i = 0; i < PCI_STD_NUM_BARS; i++) {
 566                 if (pci_resource_len(pdev, i))
 567                         mask |= (1 << i);
 568         }
 569
 570         /* No BAR can be used.. */
 571         if (!mask) {
 572                 SNET_ERR(pdev, "Failed to find a PCI BAR\n");
 573                 return -ENODEV;
 574         }
 575
 576         name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "psnet[%s]-bars", pci_name(pdev));
 577         if (!name)
 578                 return -ENOMEM;
 579
 580         ret = pcim_iomap_regions(pdev, mask, name);
 581         if (ret) {
 582                 SNET_ERR(pdev, "Failed to request and map PCI BARs\n");
 583                 return ret;
 584         }
 585
 586         for (i = 0; i < PCI_STD_NUM_BARS; i++) {
 587                 if (mask & (1 << i))
 588                         psnet->bars[i] = pcim_iomap_table(pdev)[i];
 589         }
 590
 591         return 0;
 592 }
 593
 594 static int snet_open_vf_bar(struct pci_dev *pdev, struct snet *snet)
 595 {
 596         char *name;
 597         int ret;
 598
 599         name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "snet[%s]-bars", pci_name(pdev));
 600         if (!name)
 601                 return -ENOMEM;
 602
 603         /* Request and map BAR */
 604         ret = pcim_iomap_regions(pdev, BIT(snet->psnet->cfg.vf_bar), name);
 605         if (ret) {
 606                 SNET_ERR(pdev, "Failed to request and map PCI BAR for a VF\n");
 607                 return ret;
 608         }
 609
 610         snet->bar = pcim_iomap_table(pdev)[snet->psnet->cfg.vf_bar];
 611
 612         return 0;
 613 }
 614
 615 static void snet_free_cfg(struct snet_cfg *cfg)
 616 {
 617         u32 i;
 618
 619         if (!cfg->devs)
 620                 return;
 621
 622         /* Free devices */
 623         for (i = 0; i < cfg->devices_num; i++) {
 624                 if (!cfg->devs[i])
 625                         break;
 626
 627                 kfree(cfg->devs[i]);
 628         }
 629         /* Free pointers to devices */
 630         kfree(cfg->devs);
 631 }
 632
 633 /* Detect which BAR is used for communication with the device. */
 634 static int psnet_detect_bar(struct psnet *psnet, u32 off)
 635 {
 636         unsigned long exit_time;
 637         int i;
 638
 639         exit_time = jiffies + usecs_to_jiffies(SNET_DETECT_TIMEOUT);
 640
 641         /* SNET DPU will write SNET's signature when the config is ready. */
 642         while (time_before(jiffies, exit_time)) {
 643                 for (i = 0; i < PCI_STD_NUM_BARS; i++) {
 644                         /* Is this BAR mapped? */
 645                         if (!psnet->bars[i])
 646                                 continue;
 647
 648                         if (ioread32(psnet->bars[i] + off) == SNET_SIGNATURE)
 649                                 return i;
 650                 }
 651                 usleep_range(1000, 10000);
 652         }
 653
 654         return -ENODEV;
 655 }
 656
 657 static void psnet_unmap_unused_bars(struct pci_dev *pdev, struct psnet *psnet)
 658 {
 659         int i, mask = 0;
 660
 661         for (i = 0; i < PCI_STD_NUM_BARS; i++) {
 662                 if (psnet->bars[i] && i != psnet->barno)
 663                         mask |= (1 << i);
 664         }
 665
 666         if (mask)
 667                 pcim_iounmap_regions(pdev, mask);
 668 }
 669
 670 /* Read SNET config from PCI BAR */
 671 static int psnet_read_cfg(struct pci_dev *pdev, struct psnet *psnet)
 672 {
 673         struct snet_cfg *cfg = &psnet->cfg;
 674         u32 i, off;
 675         int barno;
 676
 677         /* Move to where the config starts */
 678         off = SNET_CONFIG_OFF;
 679
 680         /* Find BAR used for communication */
 681         barno = psnet_detect_bar(psnet, off);
 682         if (barno < 0) {
 683                 SNET_ERR(pdev, "SNET config is not ready.\n");
 684                 return barno;
 685         }
 686
 687         /* Save used BAR number and unmap all other BARs */
 688         psnet->barno = barno;
 689         SNET_DBG(pdev, "Using BAR number %d\n", barno);
 690
 691         psnet_unmap_unused_bars(pdev, psnet);
 692
 693         /* load config from BAR */
 694         cfg->key = psnet_read32(psnet, off);
 695         off += 4;
 696         cfg->cfg_size = psnet_read32(psnet, off);
 697         off += 4;
 698         cfg->cfg_ver = psnet_read32(psnet, off);
 699         off += 4;
 700         /* The negotiated config version is the lower one between this driver's config
 701          * and the DPU's.
 702          */
 703         psnet->negotiated_cfg_ver = min_t(u32, cfg->cfg_ver, SNET_CFG_VERSION);
 704         SNET_DBG(pdev, "SNET config version %u\n", psnet->negotiated_cfg_ver);
 705
 706         cfg->vf_num = psnet_read32(psnet, off);
 707         off += 4;
 708         cfg->vf_bar = psnet_read32(psnet, off);
 709         off += 4;
 710         cfg->host_cfg_off = psnet_read32(psnet, off);
 711         off += 4;
 712         cfg->max_size_host_cfg = psnet_read32(psnet, off);
 713         off += 4;
 714         cfg->virtio_cfg_off = psnet_read32(psnet, off);
 715         off += 4;
 716         cfg->kick_off = psnet_read32(psnet, off);
 717         off += 4;
 718         cfg->hwmon_off = psnet_read32(psnet, off);
 719         off += 4;
 720         cfg->ctrl_off = psnet_read32(psnet, off);
 721         off += 4;
 722         cfg->flags = psnet_read32(psnet, off);
 723         off += 4;
 724         /* Ignore Reserved */
 725         off += sizeof(cfg->rsvd);
 726
 727         cfg->devices_num = psnet_read32(psnet, off);
 728         off += 4;
 729         /* Allocate memory to hold pointer to the devices */
 730         cfg->devs = kcalloc(cfg->devices_num, sizeof(void *), GFP_KERNEL);
 731         if (!cfg->devs)
 732                 return -ENOMEM;
 733
 734         /* Load device configuration from BAR */
 735         for (i = 0; i < cfg->devices_num; i++) {
 736                 cfg->devs[i] = kzalloc(sizeof(*cfg->devs[i]), GFP_KERNEL);
 737                 if (!cfg->devs[i]) {
 738                         snet_free_cfg(cfg);
 739                         return -ENOMEM;
 740                 }
 741                 /* Read device config */
 742                 cfg->devs[i]->virtio_id = psnet_read32(psnet, off);
 743                 off += 4;
 744                 cfg->devs[i]->vq_num = psnet_read32(psnet, off);
 745                 off += 4;
 746                 cfg->devs[i]->vq_size = psnet_read32(psnet, off);
 747                 off += 4;
 748                 cfg->devs[i]->vfid = psnet_read32(psnet, off);
 749                 off += 4;
 750                 cfg->devs[i]->features = psnet_read64(psnet, off);
 751                 off += 8;
 752                 /* Ignore Reserved */
 753                 off += sizeof(cfg->devs[i]->rsvd);
 754
 755                 cfg->devs[i]->cfg_size = psnet_read32(psnet, off);
 756                 off += 4;
 757
 758                 /* Is the config witten to the DPU going to be too big? */
 759                 if (SNET_GENERAL_CFG_LEN + SNET_GENERAL_CFG_VQ_LEN * cfg->devs[i]->vq_num >
 760                     cfg->max_size_host_cfg) {
 761                         SNET_ERR(pdev, "Failed to read SNET config, the config is too big..\n");
 762                         snet_free_cfg(cfg);
 763                         return -EINVAL;
 764                 }
 765         }
 766         return 0;
 767 }
 768
 769 static int psnet_alloc_irq_vector(struct pci_dev *pdev, struct psnet *psnet)
 770 {
 771         int ret = 0;
 772         u32 i, irq_num = 0;
 773
 774         /* Let's count how many IRQs we need, 1 for every VQ + 1 for config change */
 775         for (i = 0; i < psnet->cfg.devices_num; i++)
 776                 irq_num += psnet->cfg.devs[i]->vq_num + 1;
 777
 778         ret = pci_alloc_irq_vectors(pdev, irq_num, irq_num, PCI_IRQ_MSIX);
 779         if (ret != irq_num) {
 780                 SNET_ERR(pdev, "Failed to allocate IRQ vectors\n");
 781                 return ret;
 782         }
 783         SNET_DBG(pdev, "Allocated %u IRQ vectors from physical function\n", irq_num);
 784
 785         return 0;
 786 }
 787
 788 static int snet_alloc_irq_vector(struct pci_dev *pdev, struct snet_dev_cfg *snet_cfg)
 789 {
 790         int ret = 0;
 791         u32 irq_num;
 792
 793         /* We want 1 IRQ for every VQ + 1 for config change events */
 794         irq_num = snet_cfg->vq_num + 1;
 795
 796         ret = pci_alloc_irq_vectors(pdev, irq_num, irq_num, PCI_IRQ_MSIX);
 797         if (ret <= 0) {
 798                 SNET_ERR(pdev, "Failed to allocate IRQ vectors\n");
 799                 return ret;
 800         }
 801
 802         return 0;
 803 }
 804
 805 static void snet_free_vqs(struct snet *snet)
 806 {
 807         u32 i;
 808
 809         if (!snet->vqs)
 810                 return;
 811
 812         for (i = 0 ; i < snet->cfg->vq_num ; i++) {
 813                 if (!snet->vqs[i])
 814                         break;
 815
 816                 kfree(snet->vqs[i]);
 817         }
 818         kfree(snet->vqs);
 819 }
 820
 821 static int snet_build_vqs(struct snet *snet)
 822 {
 823         u32 i;
 824         /* Allocate the VQ pointers array */
 825         snet->vqs = kcalloc(snet->cfg->vq_num, sizeof(void *), GFP_KERNEL);
 826         if (!snet->vqs)
 827                 return -ENOMEM;
 828
 829         /* Allocate the VQs */
 830         for (i = 0; i < snet->cfg->vq_num; i++) {
 831                 snet->vqs[i] = kzalloc(sizeof(*snet->vqs[i]), GFP_KERNEL);
 832                 if (!snet->vqs[i]) {
 833                         snet_free_vqs(snet);
 834                         return -ENOMEM;
 835                 }
 836                 /* Reset IRQ num */
 837                 snet->vqs[i]->irq = -1;
 838                 /* VQ serial ID */
 839                 snet->vqs[i]->sid = i;
 840                 /* Kick address - every VQ gets 4B */
 841                 snet->vqs[i]->kick_ptr = snet->bar + snet->psnet->cfg.kick_off +
 842                                          snet->vqs[i]->sid * 4;
 843                 /* Clear kick address for this VQ */
 844                 iowrite32(0, snet->vqs[i]->kick_ptr);
 845         }
 846         return 0;
 847 }
 848
 849 static int psnet_get_next_irq_num(struct psnet *psnet)
 850 {
 851         int irq;
 852
 853         spin_lock(&psnet->lock);
 854         irq = psnet->next_irq++;
 855         spin_unlock(&psnet->lock);
 856
 857         return irq;
 858 }
 859
 860 static void snet_reserve_irq_idx(struct pci_dev *pdev, struct snet *snet)
 861 {
 862         struct psnet *psnet = snet->psnet;
 863         int  i;
 864
 865         /* one IRQ for every VQ, and one for config changes */
 866         snet->cfg_irq_idx = psnet_get_next_irq_num(psnet);
 867         snprintf(snet->cfg_irq_name, SNET_NAME_SIZE, "snet[%s]-cfg[%d]",
 868                  pci_name(pdev), snet->cfg_irq_idx);
 869
 870         for (i = 0; i < snet->cfg->vq_num; i++) {
 871                 /* Get next free IRQ ID */
 872                 snet->vqs[i]->irq_idx = psnet_get_next_irq_num(psnet);
 873                 /* Write IRQ name */
 874                 snprintf(snet->vqs[i]->irq_name, SNET_NAME_SIZE, "snet[%s]-vq[%d]",
 875                          pci_name(pdev), snet->vqs[i]->irq_idx);
 876         }
 877 }
 878
 879 /* Find a device config based on virtual function id */
 880 static struct snet_dev_cfg *snet_find_dev_cfg(struct snet_cfg *cfg, u32 vfid)
 881 {
 882         u32 i;
 883
 884         for (i = 0; i < cfg->devices_num; i++) {
 885                 if (cfg->devs[i]->vfid == vfid)
 886                         return cfg->devs[i];
 887         }
 888         /* Oppss.. no config found.. */
 889         return NULL;
 890 }
 891
 892 /* Probe function for a physical PCI function */
 893 static int snet_vdpa_probe_pf(struct pci_dev *pdev)
 894 {
 895         struct psnet *psnet;
 896         int ret = 0;
 897         bool pf_irqs = false;
 898
 899         ret = pcim_enable_device(pdev);
 900         if (ret) {
 901                 SNET_ERR(pdev, "Failed to enable PCI device\n");
 902                 return ret;
 903         }
 904
 905         /* Allocate a PCI physical function device */
 906         psnet = kzalloc(sizeof(*psnet), GFP_KERNEL);
 907         if (!psnet)
 908                 return -ENOMEM;
 909
 910         /* Init PSNET spinlock */
 911         spin_lock_init(&psnet->lock);
 912
 913         pci_set_master(pdev);
 914         pci_set_drvdata(pdev, psnet);
 915
 916         /* Open SNET MAIN BAR */
 917         ret = psnet_open_pf_bar(pdev, psnet);
 918         if (ret)
 919                 goto free_psnet;
 920
 921         /* Try to read SNET's config from PCI BAR */
 922         ret = psnet_read_cfg(pdev, psnet);
 923         if (ret)
 924                 goto free_psnet;
 925
 926         /* If SNET_CFG_FLAG_IRQ_PF flag is set, we should use
 927          * PF MSI-X vectors
 928          */
 929         pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF);
 930
 931         if (pf_irqs) {
 932                 ret = psnet_alloc_irq_vector(pdev, psnet);
 933                 if (ret)
 934                         goto free_cfg;
 935         }
 936
 937         SNET_DBG(pdev, "Enable %u virtual functions\n", psnet->cfg.vf_num);
 938         ret = pci_enable_sriov(pdev, psnet->cfg.vf_num);
 939         if (ret) {
 940                 SNET_ERR(pdev, "Failed to enable SR-IOV\n");
 941                 goto free_irq;
 942         }
 943
 944         /* Create HW monitor device */
 945         if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_HWMON)) {
 946 #if IS_ENABLED(CONFIG_HWMON)
 947                 psnet_create_hwmon(pdev);
 948 #else
 949                 SNET_WARN(pdev, "Can't start HWMON, CONFIG_HWMON is not enabled\n");
 950 #endif
 951         }
 952
 953         return 0;
 954
 955 free_irq:
 956         if (pf_irqs)
 957                 pci_free_irq_vectors(pdev);
 958 free_cfg:
 959         snet_free_cfg(&psnet->cfg);
 960 free_psnet:
 961         kfree(psnet);
 962         return ret;
 963 }
 964
 965 /* Probe function for a virtual PCI function */
 966 static int snet_vdpa_probe_vf(struct pci_dev *pdev)
 967 {
 968         struct pci_dev *pdev_pf = pdev->physfn;
 969         struct psnet *psnet = pci_get_drvdata(pdev_pf);
 970         struct snet_dev_cfg *dev_cfg;
 971         struct snet *snet;
 972         u32 vfid;
 973         int ret;
 974         bool pf_irqs = false;
 975
 976         /* Get virtual function id.
 977          * (the DPU counts the VFs from 1)
 978          */
 979         ret = pci_iov_vf_id(pdev);
 980         if (ret < 0) {
 981                 SNET_ERR(pdev, "Failed to find a VF id\n");
 982                 return ret;
 983         }
 984         vfid = ret + 1;
 985
 986         /* Find the snet_dev_cfg based on vfid */
 987         dev_cfg = snet_find_dev_cfg(&psnet->cfg, vfid);
 988         if (!dev_cfg) {
 989                 SNET_WARN(pdev, "Failed to find a VF config..\n");
 990                 return -ENODEV;
 991         }
 992
 993         /* Which PCI device should allocate the IRQs?
 994          * If the SNET_CFG_FLAG_IRQ_PF flag set, the PF device allocates the IRQs
 995          */
 996         pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF);
 997
 998         ret = pcim_enable_device(pdev);
 999         if (ret) {
1000                 SNET_ERR(pdev, "Failed to enable PCI VF device\n");
1001                 return ret;
1002         }
1003
1004         /* Request for MSI-X IRQs */
1005         if (!pf_irqs) {
1006                 ret = snet_alloc_irq_vector(pdev, dev_cfg);
1007                 if (ret)
1008                         return ret;
1009         }
1010
1011         /* Allocate vdpa device */
1012         snet = vdpa_alloc_device(struct snet, vdpa, &pdev->dev, &snet_config_ops, 1, 1, NULL,
1013                                  false);
1014         if (!snet) {
1015                 SNET_ERR(pdev, "Failed to allocate a vdpa device\n");
1016                 ret = -ENOMEM;
1017                 goto free_irqs;
1018         }
1019
1020         /* Init control mutex and spinlock */
1021         mutex_init(&snet->ctrl_lock);
1022         spin_lock_init(&snet->ctrl_spinlock);
1023
1024         /* Save pci device pointer */
1025         snet->pdev = pdev;
1026         snet->psnet = psnet;
1027         snet->cfg = dev_cfg;
1028         snet->dpu_ready = false;
1029         snet->sid = vfid;
1030         /* Reset IRQ value */
1031         snet->cfg_irq = -1;
1032
1033         ret = snet_open_vf_bar(pdev, snet);
1034         if (ret)
1035                 goto put_device;
1036
1037         /* Create a VirtIO config pointer */
1038         snet->cfg->virtio_cfg = snet->bar + snet->psnet->cfg.virtio_cfg_off;
1039
1040         /* Clear control registers */
1041         snet_ctrl_clear(snet);
1042
1043         pci_set_master(pdev);
1044         pci_set_drvdata(pdev, snet);
1045
1046         ret = snet_build_vqs(snet);
1047         if (ret)
1048                 goto put_device;
1049
1050         /* Reserve IRQ indexes,
1051          * The IRQs may be requested and freed multiple times,
1052          * but the indexes won't change.
1053          */
1054         snet_reserve_irq_idx(pf_irqs ? pdev_pf : pdev, snet);
1055
1056         /*set DMA device*/
1057         snet->vdpa.dma_dev = &pdev->dev;
1058
1059         /* Register VDPA device */
1060         ret = vdpa_register_device(&snet->vdpa, snet->cfg->vq_num);
1061         if (ret) {
1062                 SNET_ERR(pdev, "Failed to register vdpa device\n");
1063                 goto free_vqs;
1064         }
1065
1066         return 0;
1067
1068 free_vqs:
1069         snet_free_vqs(snet);
1070 put_device:
1071         put_device(&snet->vdpa.dev);
1072 free_irqs:
1073         if (!pf_irqs)
1074                 pci_free_irq_vectors(pdev);
1075         return ret;
1076 }
1077
1078 static int snet_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1079 {
1080         if (pdev->is_virtfn)
1081                 return snet_vdpa_probe_vf(pdev);
1082         else
1083                 return snet_vdpa_probe_pf(pdev);
1084 }
1085
1086 static void snet_vdpa_remove_pf(struct pci_dev *pdev)
1087 {
1088         struct psnet *psnet = pci_get_drvdata(pdev);
1089
1090         pci_disable_sriov(pdev);
1091         /* If IRQs are allocated from the PF, we should free the IRQs */
1092         if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF))
1093                 pci_free_irq_vectors(pdev);
1094
1095         snet_free_cfg(&psnet->cfg);
1096         kfree(psnet);
1097 }
1098
1099 static void snet_vdpa_remove_vf(struct pci_dev *pdev)
1100 {
1101         struct snet *snet = pci_get_drvdata(pdev);
1102         struct psnet *psnet = snet->psnet;
1103
1104         vdpa_unregister_device(&snet->vdpa);
1105         snet_free_vqs(snet);
1106         /* If IRQs are allocated from the VF, we should free the IRQs */
1107         if (!PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF))
1108                 pci_free_irq_vectors(pdev);
1109 }
1110
1111 static void snet_vdpa_remove(struct pci_dev *pdev)
1112 {
1113         if (pdev->is_virtfn)
1114                 snet_vdpa_remove_vf(pdev);
1115         else
1116                 snet_vdpa_remove_pf(pdev);
1117 }
1118
1119 static struct pci_device_id snet_driver_pci_ids[] = {
1120         { PCI_DEVICE_SUB(PCI_VENDOR_ID_SOLIDRUN, SNET_DEVICE_ID,
1121                          PCI_VENDOR_ID_SOLIDRUN, SNET_DEVICE_ID) },
1122         { 0 },
1123 };
1124
1125 MODULE_DEVICE_TABLE(pci, snet_driver_pci_ids);
1126
1127 static struct pci_driver snet_vdpa_driver = {
1128         .name           = "snet-vdpa-driver",
1129         .id_table       = snet_driver_pci_ids,
1130         .probe          = snet_vdpa_probe,
1131         .remove         = snet_vdpa_remove,
1132 };
1133
1134 module_pci_driver(snet_vdpa_driver);
1135
1136 MODULE_AUTHOR("Alvaro Karsz <[email protected]>");
1137 MODULE_DESCRIPTION("SolidRun vDPA driver");
1138 MODULE_LICENSE("GPL v2");