drivers/misc/mic/vop/vop_main.c

   1 /*
   2  * Intel MIC Platform Software Stack (MPSS)
   3  *
   4  * Copyright(c) 2016 Intel Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License, version 2, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  13  * General Public License for more details.
  14  *
  15  * The full GNU General Public License is included in this distribution in
  16  * the file called "COPYING".
  17  *
  18  * Adapted from:
  19  *
  20  * virtio for kvm on s390
  21  *
  22  * Copyright IBM Corp. 2008
  23  *
  24  * This program is free software; you can redistribute it and/or modify
  25  * it under the terms of the GNU General Public License (version 2 only)
  26  * as published by the Free Software Foundation.
  27  *
  28  *    Author(s): Christian Borntraeger <[email protected]>
  29  *
  30  * Intel Virtio Over PCIe (VOP) driver.
  31  *
  32  */
  33 #include <linux/delay.h>
  34 #include <linux/module.h>
  35 #include <linux/sched.h>
  36 #include <linux/dma-mapping.h>
  37 #include <linux/io-64-nonatomic-lo-hi.h>
  38
  39 #include "vop_main.h"
  40
  41 #define VOP_MAX_VRINGS 4
  42
  43 /*
  44  * _vop_vdev - Allocated per virtio device instance injected by the peer.
  45  *
  46  * @vdev: Virtio device
  47  * @desc: Virtio device page descriptor
  48  * @dc: Virtio device control
  49  * @vpdev: VOP device which is the parent for this virtio device
  50  * @vr: Buffer for accessing the VRING
  51  * @used_virt: Virtual address of used ring
  52  * @used: DMA address of used ring
  53  * @used_size: Size of the used buffer
  54  * @reset_done: Track whether VOP reset is complete
  55  * @virtio_cookie: Cookie returned upon requesting a interrupt
  56  * @c2h_vdev_db: The doorbell used by the guest to interrupt the host
  57  * @h2c_vdev_db: The doorbell used by the host to interrupt the guest
  58  * @dnode: The destination node
  59  */
  60 struct _vop_vdev {
  61         struct virtio_device vdev;
  62         struct mic_device_desc __iomem *desc;
  63         struct mic_device_ctrl __iomem *dc;
  64         struct vop_device *vpdev;
  65         void __iomem *vr[VOP_MAX_VRINGS];
  66         void *used_virt[VOP_MAX_VRINGS];
  67         dma_addr_t used[VOP_MAX_VRINGS];
  68         int used_size[VOP_MAX_VRINGS];
  69         struct completion reset_done;
  70         struct mic_irq *virtio_cookie;
  71         int c2h_vdev_db;
  72         int h2c_vdev_db;
  73         int dnode;
  74 };
  75
  76 #define to_vopvdev(vd) container_of(vd, struct _vop_vdev, vdev)
  77
  78 #define _vop_aligned_desc_size(d) __mic_align(_vop_desc_size(d), 8)
  79
  80 /* Helper API to obtain the parent of the virtio device */
  81 static inline struct device *_vop_dev(struct _vop_vdev *vdev)
  82 {
  83         return vdev->vdev.dev.parent;
  84 }
  85
  86 static inline unsigned _vop_desc_size(struct mic_device_desc __iomem *desc)
  87 {
  88         return sizeof(*desc)
  89                 + ioread8(&desc->num_vq) * sizeof(struct mic_vqconfig)
  90                 + ioread8(&desc->feature_len) * 2
  91                 + ioread8(&desc->config_len);
  92 }
  93
  94 static inline struct mic_vqconfig __iomem *
  95 _vop_vq_config(struct mic_device_desc __iomem *desc)
  96 {
  97         return (struct mic_vqconfig __iomem *)(desc + 1);
  98 }
  99
 100 static inline u8 __iomem *
 101 _vop_vq_features(struct mic_device_desc __iomem *desc)
 102 {
 103         return (u8 __iomem *)(_vop_vq_config(desc) + ioread8(&desc->num_vq));
 104 }
 105
 106 static inline u8 __iomem *
 107 _vop_vq_configspace(struct mic_device_desc __iomem *desc)
 108 {
 109         return _vop_vq_features(desc) + ioread8(&desc->feature_len) * 2;
 110 }
 111
 112 static inline unsigned
 113 _vop_total_desc_size(struct mic_device_desc __iomem *desc)
 114 {
 115         return _vop_aligned_desc_size(desc) + sizeof(struct mic_device_ctrl);
 116 }
 117
 118 /* This gets the device's feature bits. */
 119 static u64 vop_get_features(struct virtio_device *vdev)
 120 {
 121         unsigned int i, bits;
 122         u64 features = 0;
 123         struct mic_device_desc __iomem *desc = to_vopvdev(vdev)->desc;
 124         u8 __iomem *in_features = _vop_vq_features(desc);
 125         int feature_len = ioread8(&desc->feature_len);
 126
 127         bits = min_t(unsigned, feature_len, sizeof(vdev->features)) * 8;
 128         for (i = 0; i < bits; i++)
 129                 if (ioread8(&in_features[i / 8]) & (BIT(i % 8)))
 130                         features |= BIT_ULL(i);
 131
 132         return features;
 133 }
 134
 135 static void vop_transport_features(struct virtio_device *vdev)
 136 {
 137         /*
 138          * Packed ring isn't enabled on virtio_vop for now,
 139          * because virtio_vop uses vring_new_virtqueue() which
 140          * creates virtio rings on preallocated memory.
 141          */
 142         __virtio_clear_bit(vdev, VIRTIO_F_RING_PACKED);
 143 }
 144
 145 static int vop_finalize_features(struct virtio_device *vdev)
 146 {
 147         unsigned int i, bits;
 148         struct mic_device_desc __iomem *desc = to_vopvdev(vdev)->desc;
 149         u8 feature_len = ioread8(&desc->feature_len);
 150         /* Second half of bitmap is features we accept. */
 151         u8 __iomem *out_features =
 152                 _vop_vq_features(desc) + feature_len;
 153
 154         /* Give virtio_ring a chance to accept features. */
 155         vring_transport_features(vdev);
 156
 157         /* Give virtio_vop a chance to accept features. */
 158         vop_transport_features(vdev);
 159
 160         memset_io(out_features, 0, feature_len);
 161         bits = min_t(unsigned, feature_len,
 162                      sizeof(vdev->features)) * 8;
 163         for (i = 0; i < bits; i++) {
 164                 if (__virtio_test_bit(vdev, i))
 165                         iowrite8(ioread8(&out_features[i / 8]) | (1 << (i % 8)),
 166                                  &out_features[i / 8]);
 167         }
 168         return 0;
 169 }
 170
 171 /*
 172  * Reading and writing elements in config space
 173  */
 174 static void vop_get(struct virtio_device *vdev, unsigned int offset,
 175                     void *buf, unsigned len)
 176 {
 177         struct mic_device_desc __iomem *desc = to_vopvdev(vdev)->desc;
 178
 179         if (offset + len > ioread8(&desc->config_len))
 180                 return;
 181         memcpy_fromio(buf, _vop_vq_configspace(desc) + offset, len);
 182 }
 183
 184 static void vop_set(struct virtio_device *vdev, unsigned int offset,
 185                     const void *buf, unsigned len)
 186 {
 187         struct mic_device_desc __iomem *desc = to_vopvdev(vdev)->desc;
 188
 189         if (offset + len > ioread8(&desc->config_len))
 190                 return;
 191         memcpy_toio(_vop_vq_configspace(desc) + offset, buf, len);
 192 }
 193
 194 /*
 195  * The operations to get and set the status word just access the status
 196  * field of the device descriptor. set_status also interrupts the host
 197  * to tell about status changes.
 198  */
 199 static u8 vop_get_status(struct virtio_device *vdev)
 200 {
 201         return ioread8(&to_vopvdev(vdev)->desc->status);
 202 }
 203
 204 static void vop_set_status(struct virtio_device *dev, u8 status)
 205 {
 206         struct _vop_vdev *vdev = to_vopvdev(dev);
 207         struct vop_device *vpdev = vdev->vpdev;
 208
 209         if (!status)
 210                 return;
 211         iowrite8(status, &vdev->desc->status);
 212         vpdev->hw_ops->send_intr(vpdev, vdev->c2h_vdev_db);
 213 }
 214
 215 /* Inform host on a virtio device reset and wait for ack from host */
 216 static void vop_reset_inform_host(struct virtio_device *dev)
 217 {
 218         struct _vop_vdev *vdev = to_vopvdev(dev);
 219         struct mic_device_ctrl __iomem *dc = vdev->dc;
 220         struct vop_device *vpdev = vdev->vpdev;
 221         int retry;
 222
 223         iowrite8(0, &dc->host_ack);
 224         iowrite8(1, &dc->vdev_reset);
 225         vpdev->hw_ops->send_intr(vpdev, vdev->c2h_vdev_db);
 226
 227         /* Wait till host completes all card accesses and acks the reset */
 228         for (retry = 100; retry--;) {
 229                 if (ioread8(&dc->host_ack))
 230                         break;
 231                 msleep(100);
 232         }
 233
 234         dev_dbg(_vop_dev(vdev), "%s: retry: %d\n", __func__, retry);
 235
 236         /* Reset status to 0 in case we timed out */
 237         iowrite8(0, &vdev->desc->status);
 238 }
 239
 240 static void vop_reset(struct virtio_device *dev)
 241 {
 242         struct _vop_vdev *vdev = to_vopvdev(dev);
 243
 244         dev_dbg(_vop_dev(vdev), "%s: virtio id %d\n",
 245                 __func__, dev->id.device);
 246
 247         vop_reset_inform_host(dev);
 248         complete_all(&vdev->reset_done);
 249 }
 250
 251 /*
 252  * The virtio_ring code calls this API when it wants to notify the Host.
 253  */
 254 static bool vop_notify(struct virtqueue *vq)
 255 {
 256         struct _vop_vdev *vdev = vq->priv;
 257         struct vop_device *vpdev = vdev->vpdev;
 258
 259         vpdev->hw_ops->send_intr(vpdev, vdev->c2h_vdev_db);
 260         return true;
 261 }
 262
 263 static void vop_del_vq(struct virtqueue *vq, int n)
 264 {
 265         struct _vop_vdev *vdev = to_vopvdev(vq->vdev);
 266         struct vop_device *vpdev = vdev->vpdev;
 267
 268         dma_unmap_single(&vpdev->dev, vdev->used[n],
 269                          vdev->used_size[n], DMA_BIDIRECTIONAL);
 270         free_pages((unsigned long)vdev->used_virt[n],
 271                    get_order(vdev->used_size[n]));
 272         vring_del_virtqueue(vq);
 273         vpdev->hw_ops->unmap(vpdev, vdev->vr[n]);
 274         vdev->vr[n] = NULL;
 275 }
 276
 277 static void vop_del_vqs(struct virtio_device *dev)
 278 {
 279         struct _vop_vdev *vdev = to_vopvdev(dev);
 280         struct virtqueue *vq, *n;
 281         int idx = 0;
 282
 283         dev_dbg(_vop_dev(vdev), "%s\n", __func__);
 284
 285         list_for_each_entry_safe(vq, n, &dev->vqs, list)
 286                 vop_del_vq(vq, idx++);
 287 }
 288
 289 static struct virtqueue *vop_new_virtqueue(unsigned int index,
 290                                       unsigned int num,
 291                                       struct virtio_device *vdev,
 292                                       bool context,
 293                                       void *pages,
 294                                       bool (*notify)(struct virtqueue *vq),
 295                                       void (*callback)(struct virtqueue *vq),
 296                                       const char *name,
 297                                       void *used)
 298 {
 299         bool weak_barriers = false;
 300         struct vring vring;
 301
 302         vring_init(&vring, num, pages, MIC_VIRTIO_RING_ALIGN);
 303         vring.used = used;
 304
 305         return __vring_new_virtqueue(index, vring, vdev, weak_barriers, context,
 306                                      notify, callback, name);
 307 }
 308
 309 /*
 310  * This routine will assign vring's allocated in host/io memory. Code in
 311  * virtio_ring.c however continues to access this io memory as if it were local
 312  * memory without io accessors.
 313  */
 314 static struct virtqueue *vop_find_vq(struct virtio_device *dev,
 315                                      unsigned index,
 316                                      void (*callback)(struct virtqueue *vq),
 317                                      const char *name, bool ctx)
 318 {
 319         struct _vop_vdev *vdev = to_vopvdev(dev);
 320         struct vop_device *vpdev = vdev->vpdev;
 321         struct mic_vqconfig __iomem *vqconfig;
 322         struct mic_vqconfig config;
 323         struct virtqueue *vq;
 324         void __iomem *va;
 325         struct _mic_vring_info __iomem *info;
 326         void *used;
 327         int vr_size, _vr_size, err, magic;
 328         u8 type = ioread8(&vdev->desc->type);
 329
 330         if (index >= ioread8(&vdev->desc->num_vq))
 331                 return ERR_PTR(-ENOENT);
 332
 333         if (!name)
 334                 return ERR_PTR(-ENOENT);
 335
 336         /* First assign the vring's allocated in host memory */
 337         vqconfig = _vop_vq_config(vdev->desc) + index;
 338         memcpy_fromio(&config, vqconfig, sizeof(config));
 339         _vr_size = vring_size(le16_to_cpu(config.num), MIC_VIRTIO_RING_ALIGN);
 340         vr_size = PAGE_ALIGN(_vr_size + sizeof(struct _mic_vring_info));
 341         va = vpdev->hw_ops->remap(vpdev, le64_to_cpu(config.address), vr_size);
 342         if (!va)
 343                 return ERR_PTR(-ENOMEM);
 344         vdev->vr[index] = va;
 345         memset_io(va, 0x0, _vr_size);
 346
 347         info = va + _vr_size;
 348         magic = ioread32(&info->magic);
 349
 350         if (WARN(magic != MIC_MAGIC + type + index, "magic mismatch")) {
 351                 err = -EIO;
 352                 goto unmap;
 353         }
 354
 355         vdev->used_size[index] = PAGE_ALIGN(sizeof(__u16) * 3 +
 356                                              sizeof(struct vring_used_elem) *
 357                                              le16_to_cpu(config.num));
 358         used = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
 359                                         get_order(vdev->used_size[index]));
 360         vdev->used_virt[index] = used;
 361         if (!used) {
 362                 err = -ENOMEM;
 363                 dev_err(_vop_dev(vdev), "%s %d err %d\n",
 364                         __func__, __LINE__, err);
 365                 goto unmap;
 366         }
 367
 368         vq = vop_new_virtqueue(index, le16_to_cpu(config.num), dev, ctx,
 369                                (void __force *)va, vop_notify, callback,
 370                                name, used);
 371         if (!vq) {
 372                 err = -ENOMEM;
 373                 goto free_used;
 374         }
 375
 376         vdev->used[index] = dma_map_single(&vpdev->dev, used,
 377                                             vdev->used_size[index],
 378                                             DMA_BIDIRECTIONAL);
 379         if (dma_mapping_error(&vpdev->dev, vdev->used[index])) {
 380                 err = -ENOMEM;
 381                 dev_err(_vop_dev(vdev), "%s %d err %d\n",
 382                         __func__, __LINE__, err);
 383                 goto del_vq;
 384         }
 385         writeq(vdev->used[index], &vqconfig->used_address);
 386
 387         vq->priv = vdev;
 388         return vq;
 389 del_vq:
 390         vring_del_virtqueue(vq);
 391 free_used:
 392         free_pages((unsigned long)used,
 393                    get_order(vdev->used_size[index]));
 394 unmap:
 395         vpdev->hw_ops->unmap(vpdev, vdev->vr[index]);
 396         return ERR_PTR(err);
 397 }
 398
 399 static int vop_find_vqs(struct virtio_device *dev, unsigned nvqs,
 400                         struct virtqueue *vqs[],
 401                         vq_callback_t *callbacks[],
 402                         const char * const names[], const bool *ctx,
 403                         struct irq_affinity *desc)
 404 {
 405         struct _vop_vdev *vdev = to_vopvdev(dev);
 406         struct vop_device *vpdev = vdev->vpdev;
 407         struct mic_device_ctrl __iomem *dc = vdev->dc;
 408         int i, err, retry, queue_idx = 0;
 409
 410         /* We must have this many virtqueues. */
 411         if (nvqs > ioread8(&vdev->desc->num_vq))
 412                 return -ENOENT;
 413
 414         for (i = 0; i < nvqs; ++i) {
 415                 if (!names[i]) {
 416                         vqs[i] = NULL;
 417                         continue;
 418                 }
 419
 420                 dev_dbg(_vop_dev(vdev), "%s: %d: %s\n",
 421                         __func__, i, names[i]);
 422                 vqs[i] = vop_find_vq(dev, queue_idx++, callbacks[i], names[i],
 423                                      ctx ? ctx[i] : false);
 424                 if (IS_ERR(vqs[i])) {
 425                         err = PTR_ERR(vqs[i]);
 426                         goto error;
 427                 }
 428         }
 429
 430         iowrite8(1, &dc->used_address_updated);
 431         /*
 432          * Send an interrupt to the host to inform it that used
 433          * rings have been re-assigned.
 434          */
 435         vpdev->hw_ops->send_intr(vpdev, vdev->c2h_vdev_db);
 436         for (retry = 100; --retry;) {
 437                 if (!ioread8(&dc->used_address_updated))
 438                         break;
 439                 msleep(100);
 440         }
 441
 442         dev_dbg(_vop_dev(vdev), "%s: retry: %d\n", __func__, retry);
 443         if (!retry) {
 444                 err = -ENODEV;
 445                 goto error;
 446         }
 447
 448         return 0;
 449 error:
 450         vop_del_vqs(dev);
 451         return err;
 452 }
 453
 454 /*
 455  * The config ops structure as defined by virtio config
 456  */
 457 static struct virtio_config_ops vop_vq_config_ops = {
 458         .get_features = vop_get_features,
 459         .finalize_features = vop_finalize_features,
 460         .get = vop_get,
 461         .set = vop_set,
 462         .get_status = vop_get_status,
 463         .set_status = vop_set_status,
 464         .reset = vop_reset,
 465         .find_vqs = vop_find_vqs,
 466         .del_vqs = vop_del_vqs,
 467 };
 468
 469 static irqreturn_t vop_virtio_intr_handler(int irq, void *data)
 470 {
 471         struct _vop_vdev *vdev = data;
 472         struct vop_device *vpdev = vdev->vpdev;
 473         struct virtqueue *vq;
 474
 475         vpdev->hw_ops->ack_interrupt(vpdev, vdev->h2c_vdev_db);
 476         list_for_each_entry(vq, &vdev->vdev.vqs, list)
 477                 vring_interrupt(0, vq);
 478
 479         return IRQ_HANDLED;
 480 }
 481
 482 static void vop_virtio_release_dev(struct device *_d)
 483 {
 484         struct virtio_device *vdev =
 485                         container_of(_d, struct virtio_device, dev);
 486         struct _vop_vdev *vop_vdev =
 487                         container_of(vdev, struct _vop_vdev, vdev);
 488
 489         kfree(vop_vdev);
 490 }
 491
 492 /*
 493  * adds a new device and register it with virtio
 494  * appropriate drivers are loaded by the device model
 495  */
 496 static int _vop_add_device(struct mic_device_desc __iomem *d,
 497                            unsigned int offset, struct vop_device *vpdev,
 498                            int dnode)
 499 {
 500         struct _vop_vdev *vdev, *reg_dev = NULL;
 501         int ret;
 502         u8 type = ioread8(&d->type);
 503
 504         vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
 505         if (!vdev)
 506                 return -ENOMEM;
 507
 508         vdev->vpdev = vpdev;
 509         vdev->vdev.dev.parent = &vpdev->dev;
 510         vdev->vdev.dev.release = vop_virtio_release_dev;
 511         vdev->vdev.id.device = type;
 512         vdev->vdev.config = &vop_vq_config_ops;
 513         vdev->desc = d;
 514         vdev->dc = (void __iomem *)d + _vop_aligned_desc_size(d);
 515         vdev->dnode = dnode;
 516         vdev->vdev.priv = (void *)(unsigned long)dnode;
 517         init_completion(&vdev->reset_done);
 518
 519         vdev->h2c_vdev_db = vpdev->hw_ops->next_db(vpdev);
 520         vdev->virtio_cookie = vpdev->hw_ops->request_irq(vpdev,
 521                         vop_virtio_intr_handler, "virtio intr",
 522                         vdev, vdev->h2c_vdev_db);
 523         if (IS_ERR(vdev->virtio_cookie)) {
 524                 ret = PTR_ERR(vdev->virtio_cookie);
 525                 goto kfree;
 526         }
 527         iowrite8((u8)vdev->h2c_vdev_db, &vdev->dc->h2c_vdev_db);
 528         vdev->c2h_vdev_db = ioread8(&vdev->dc->c2h_vdev_db);
 529
 530         ret = register_virtio_device(&vdev->vdev);
 531         reg_dev = vdev;
 532         if (ret) {
 533                 dev_err(_vop_dev(vdev),
 534                         "Failed to register vop device %u type %u\n",
 535                         offset, type);
 536                 goto free_irq;
 537         }
 538         writeq((unsigned long)vdev, &vdev->dc->vdev);
 539         dev_dbg(_vop_dev(vdev), "%s: registered vop device %u type %u vdev %p\n",
 540                 __func__, offset, type, vdev);
 541
 542         return 0;
 543
 544 free_irq:
 545         vpdev->hw_ops->free_irq(vpdev, vdev->virtio_cookie, vdev);
 546 kfree:
 547         if (reg_dev)
 548                 put_device(&vdev->vdev.dev);
 549         else
 550                 kfree(vdev);
 551         return ret;
 552 }
 553
 554 /*
 555  * match for a vop device with a specific desc pointer
 556  */
 557 static int vop_match_desc(struct device *dev, void *data)
 558 {
 559         struct virtio_device *_dev = dev_to_virtio(dev);
 560         struct _vop_vdev *vdev = to_vopvdev(_dev);
 561
 562         return vdev->desc == (void __iomem *)data;
 563 }
 564
 565 static struct _vop_vdev *vop_dc_to_vdev(struct mic_device_ctrl *dc)
 566 {
 567         return (struct _vop_vdev *)(unsigned long)readq(&dc->vdev);
 568 }
 569
 570 static void _vop_handle_config_change(struct mic_device_desc __iomem *d,
 571                                       unsigned int offset,
 572                                       struct vop_device *vpdev)
 573 {
 574         struct mic_device_ctrl __iomem *dc
 575                 = (void __iomem *)d + _vop_aligned_desc_size(d);
 576         struct _vop_vdev *vdev = vop_dc_to_vdev(dc);
 577
 578         if (ioread8(&dc->config_change) != MIC_VIRTIO_PARAM_CONFIG_CHANGED)
 579                 return;
 580
 581         dev_dbg(&vpdev->dev, "%s %d\n", __func__, __LINE__);
 582         virtio_config_changed(&vdev->vdev);
 583         iowrite8(1, &dc->guest_ack);
 584 }
 585
 586 /*
 587  * removes a virtio device if a hot remove event has been
 588  * requested by the host.
 589  */
 590 static int _vop_remove_device(struct mic_device_desc __iomem *d,
 591                               unsigned int offset, struct vop_device *vpdev)
 592 {
 593         struct mic_device_ctrl __iomem *dc
 594                 = (void __iomem *)d + _vop_aligned_desc_size(d);
 595         struct _vop_vdev *vdev = vop_dc_to_vdev(dc);
 596         u8 status;
 597         int ret = -1;
 598
 599         if (ioread8(&dc->config_change) == MIC_VIRTIO_PARAM_DEV_REMOVE) {
 600                 struct device *dev = get_device(&vdev->vdev.dev);
 601
 602                 dev_dbg(&vpdev->dev,
 603                         "%s %d config_change %d type %d vdev %p\n",
 604                         __func__, __LINE__,
 605                         ioread8(&dc->config_change), ioread8(&d->type), vdev);
 606                 status = ioread8(&d->status);
 607                 reinit_completion(&vdev->reset_done);
 608                 unregister_virtio_device(&vdev->vdev);
 609                 vpdev->hw_ops->free_irq(vpdev, vdev->virtio_cookie, vdev);
 610                 iowrite8(-1, &dc->h2c_vdev_db);
 611                 if (status & VIRTIO_CONFIG_S_DRIVER_OK)
 612                         wait_for_completion(&vdev->reset_done);
 613                 put_device(dev);
 614                 iowrite8(1, &dc->guest_ack);
 615                 dev_dbg(&vpdev->dev, "%s %d guest_ack %d\n",
 616                         __func__, __LINE__, ioread8(&dc->guest_ack));
 617                 iowrite8(-1, &d->type);
 618                 ret = 0;
 619         }
 620         return ret;
 621 }
 622
 623 #define REMOVE_DEVICES true
 624
 625 static void _vop_scan_devices(void __iomem *dp, struct vop_device *vpdev,
 626                               bool remove, int dnode)
 627 {
 628         s8 type;
 629         unsigned int i;
 630         struct mic_device_desc __iomem *d;
 631         struct mic_device_ctrl __iomem *dc;
 632         struct device *dev;
 633         int ret;
 634
 635         for (i = sizeof(struct mic_bootparam);
 636                         i < MIC_DP_SIZE; i += _vop_total_desc_size(d)) {
 637                 d = dp + i;
 638                 dc = (void __iomem *)d + _vop_aligned_desc_size(d);
 639                 /*
 640                  * This read barrier is paired with the corresponding write
 641                  * barrier on the host which is inserted before adding or
 642                  * removing a virtio device descriptor, by updating the type.
 643                  */
 644                 rmb();
 645                 type = ioread8(&d->type);
 646
 647                 /* end of list */
 648                 if (type == 0)
 649                         break;
 650
 651                 if (type == -1)
 652                         continue;
 653
 654                 /* device already exists */
 655                 dev = device_find_child(&vpdev->dev, (void __force *)d,
 656                                         vop_match_desc);
 657                 if (dev) {
 658                         if (remove)
 659                                 iowrite8(MIC_VIRTIO_PARAM_DEV_REMOVE,
 660                                          &dc->config_change);
 661                         put_device(dev);
 662                         _vop_handle_config_change(d, i, vpdev);
 663                         ret = _vop_remove_device(d, i, vpdev);
 664                         if (remove) {
 665                                 iowrite8(0, &dc->config_change);
 666                                 iowrite8(0, &dc->guest_ack);
 667                         }
 668                         continue;
 669                 }
 670
 671                 /* new device */
 672                 dev_dbg(&vpdev->dev, "%s %d Adding new virtio device %p\n",
 673                         __func__, __LINE__, d);
 674                 if (!remove)
 675                         _vop_add_device(d, i, vpdev, dnode);
 676         }
 677 }
 678
 679 static void vop_scan_devices(struct vop_info *vi,
 680                              struct vop_device *vpdev, bool remove)
 681 {
 682         void __iomem *dp = vpdev->hw_ops->get_remote_dp(vpdev);
 683
 684         if (!dp)
 685                 return;
 686         mutex_lock(&vi->vop_mutex);
 687         _vop_scan_devices(dp, vpdev, remove, vpdev->dnode);
 688         mutex_unlock(&vi->vop_mutex);
 689 }
 690
 691 /*
 692  * vop_hotplug_device tries to find changes in the device page.
 693  */
 694 static void vop_hotplug_devices(struct work_struct *work)
 695 {
 696         struct vop_info *vi = container_of(work, struct vop_info,
 697                                              hotplug_work);
 698
 699         vop_scan_devices(vi, vi->vpdev, !REMOVE_DEVICES);
 700 }
 701
 702 /*
 703  * Interrupt handler for hot plug/config changes etc.
 704  */
 705 static irqreturn_t vop_extint_handler(int irq, void *data)
 706 {
 707         struct vop_info *vi = data;
 708         struct mic_bootparam __iomem *bp;
 709         struct vop_device *vpdev = vi->vpdev;
 710
 711         bp = vpdev->hw_ops->get_remote_dp(vpdev);
 712         dev_dbg(&vpdev->dev, "%s %d hotplug work\n",
 713                 __func__, __LINE__);
 714         vpdev->hw_ops->ack_interrupt(vpdev, ioread8(&bp->h2c_config_db));
 715         schedule_work(&vi->hotplug_work);
 716         return IRQ_HANDLED;
 717 }
 718
 719 static int vop_driver_probe(struct vop_device *vpdev)
 720 {
 721         struct vop_info *vi;
 722         int rc;
 723
 724         vi = kzalloc(sizeof(*vi), GFP_KERNEL);
 725         if (!vi) {
 726                 rc = -ENOMEM;
 727                 goto exit;
 728         }
 729         dev_set_drvdata(&vpdev->dev, vi);
 730         vi->vpdev = vpdev;
 731
 732         mutex_init(&vi->vop_mutex);
 733         INIT_WORK(&vi->hotplug_work, vop_hotplug_devices);
 734         if (vpdev->dnode) {
 735                 rc = vop_host_init(vi);
 736                 if (rc < 0)
 737                         goto free;
 738         } else {
 739                 struct mic_bootparam __iomem *bootparam;
 740
 741                 vop_scan_devices(vi, vpdev, !REMOVE_DEVICES);
 742
 743                 vi->h2c_config_db = vpdev->hw_ops->next_db(vpdev);
 744                 vi->cookie = vpdev->hw_ops->request_irq(vpdev,
 745                                                         vop_extint_handler,
 746                                                         "virtio_config_intr",
 747                                                         vi, vi->h2c_config_db);
 748                 if (IS_ERR(vi->cookie)) {
 749                         rc = PTR_ERR(vi->cookie);
 750                         goto free;
 751                 }
 752                 bootparam = vpdev->hw_ops->get_remote_dp(vpdev);
 753                 iowrite8(vi->h2c_config_db, &bootparam->h2c_config_db);
 754         }
 755         vop_init_debugfs(vi);
 756         return 0;
 757 free:
 758         kfree(vi);
 759 exit:
 760         return rc;
 761 }
 762
 763 static void vop_driver_remove(struct vop_device *vpdev)
 764 {
 765         struct vop_info *vi = dev_get_drvdata(&vpdev->dev);
 766
 767         if (vpdev->dnode) {
 768                 vop_host_uninit(vi);
 769         } else {
 770                 struct mic_bootparam __iomem *bootparam =
 771                         vpdev->hw_ops->get_remote_dp(vpdev);
 772                 if (bootparam)
 773                         iowrite8(-1, &bootparam->h2c_config_db);
 774                 vpdev->hw_ops->free_irq(vpdev, vi->cookie, vi);
 775                 flush_work(&vi->hotplug_work);
 776                 vop_scan_devices(vi, vpdev, REMOVE_DEVICES);
 777         }
 778         vop_exit_debugfs(vi);
 779         kfree(vi);
 780 }
 781
 782 static struct vop_device_id id_table[] = {
 783         { VOP_DEV_TRNSP, VOP_DEV_ANY_ID },
 784         { 0 },
 785 };
 786
 787 static struct vop_driver vop_driver = {
 788         .driver.name =  KBUILD_MODNAME,
 789         .driver.owner = THIS_MODULE,
 790         .id_table = id_table,
 791         .probe = vop_driver_probe,
 792         .remove = vop_driver_remove,
 793 };
 794
 795 module_vop_driver(vop_driver);
 796
 797 MODULE_DEVICE_TABLE(mbus, id_table);
 798 MODULE_AUTHOR("Intel Corporation");
 799 MODULE_DESCRIPTION("Intel(R) Virtio Over PCIe (VOP) driver");
 800 MODULE_LICENSE("GPL v2");