drivers/pci/xen-pcifront.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Xen PCI Frontend
   4  *
   5  * Author: Ryan Wilson <[email protected]>
   6  */
   7 #include <linux/module.h>
   8 #include <linux/init.h>
   9 #include <linux/mm.h>
  10 #include <xen/xenbus.h>
  11 #include <xen/events.h>
  12 #include <xen/grant_table.h>
  13 #include <xen/page.h>
  14 #include <linux/spinlock.h>
  15 #include <linux/pci.h>
  16 #include <linux/msi.h>
  17 #include <xen/interface/io/pciif.h>
  18 #include <asm/xen/pci.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/atomic.h>
  21 #include <linux/workqueue.h>
  22 #include <linux/bitops.h>
  23 #include <linux/time.h>
  24 #include <linux/ktime.h>
  25 #include <xen/platform_pci.h>
  26
  27 #include <asm/xen/swiotlb-xen.h>
  28 #define INVALID_GRANT_REF (0)
  29 #define INVALID_EVTCHN    (-1)
  30
  31 struct pci_bus_entry {
  32         struct list_head list;
  33         struct pci_bus *bus;
  34 };
  35
  36 #define _PDEVB_op_active                (0)
  37 #define PDEVB_op_active                 (1 << (_PDEVB_op_active))
  38
  39 struct pcifront_device {
  40         struct xenbus_device *xdev;
  41         struct list_head root_buses;
  42
  43         int evtchn;
  44         int gnt_ref;
  45
  46         int irq;
  47
  48         /* Lock this when doing any operations in sh_info */
  49         spinlock_t sh_info_lock;
  50         struct xen_pci_sharedinfo *sh_info;
  51         struct work_struct op_work;
  52         unsigned long flags;
  53
  54 };
  55
  56 struct pcifront_sd {
  57         struct pci_sysdata sd;
  58         struct pcifront_device *pdev;
  59 };
  60
  61 static inline struct pcifront_device *
  62 pcifront_get_pdev(struct pcifront_sd *sd)
  63 {
  64         return sd->pdev;
  65 }
  66
  67 static inline void pcifront_init_sd(struct pcifront_sd *sd,
  68                                     unsigned int domain, unsigned int bus,
  69                                     struct pcifront_device *pdev)
  70 {
  71         /* Because we do not expose that information via XenBus. */
  72         sd->sd.node = first_online_node;
  73         sd->sd.domain = domain;
  74         sd->pdev = pdev;
  75 }
  76
  77 static DEFINE_SPINLOCK(pcifront_dev_lock);
  78 static struct pcifront_device *pcifront_dev;
  79
  80 static int errno_to_pcibios_err(int errno)
  81 {
  82         switch (errno) {
  83         case XEN_PCI_ERR_success:
  84                 return PCIBIOS_SUCCESSFUL;
  85
  86         case XEN_PCI_ERR_dev_not_found:
  87                 return PCIBIOS_DEVICE_NOT_FOUND;
  88
  89         case XEN_PCI_ERR_invalid_offset:
  90         case XEN_PCI_ERR_op_failed:
  91                 return PCIBIOS_BAD_REGISTER_NUMBER;
  92
  93         case XEN_PCI_ERR_not_implemented:
  94                 return PCIBIOS_FUNC_NOT_SUPPORTED;
  95
  96         case XEN_PCI_ERR_access_denied:
  97                 return PCIBIOS_SET_FAILED;
  98         }
  99         return errno;
 100 }
 101
 102 static inline void schedule_pcifront_aer_op(struct pcifront_device *pdev)
 103 {
 104         if (test_bit(_XEN_PCIB_active, (unsigned long *)&pdev->sh_info->flags)
 105                 && !test_and_set_bit(_PDEVB_op_active, &pdev->flags)) {
 106                 dev_dbg(&pdev->xdev->dev, "schedule aer frontend job\n");
 107                 schedule_work(&pdev->op_work);
 108         }
 109 }
 110
 111 static int do_pci_op(struct pcifront_device *pdev, struct xen_pci_op *op)
 112 {
 113         int err = 0;
 114         struct xen_pci_op *active_op = &pdev->sh_info->op;
 115         unsigned long irq_flags;
 116         evtchn_port_t port = pdev->evtchn;
 117         unsigned irq = pdev->irq;
 118         s64 ns, ns_timeout;
 119
 120         spin_lock_irqsave(&pdev->sh_info_lock, irq_flags);
 121
 122         memcpy(active_op, op, sizeof(struct xen_pci_op));
 123
 124         /* Go */
 125         wmb();
 126         set_bit(_XEN_PCIF_active, (unsigned long *)&pdev->sh_info->flags);
 127         notify_remote_via_evtchn(port);
 128
 129         /*
 130          * We set a poll timeout of 3 seconds but give up on return after
 131          * 2 seconds. It is better to time out too late rather than too early
 132          * (in the latter case we end up continually re-executing poll() with a
 133          * timeout in the past). 1s difference gives plenty of slack for error.
 134          */
 135         ns_timeout = ktime_get_ns() + 2 * (s64)NSEC_PER_SEC;
 136
 137         xen_clear_irq_pending(irq);
 138
 139         while (test_bit(_XEN_PCIF_active,
 140                         (unsigned long *)&pdev->sh_info->flags)) {
 141                 xen_poll_irq_timeout(irq, jiffies + 3*HZ);
 142                 xen_clear_irq_pending(irq);
 143                 ns = ktime_get_ns();
 144                 if (ns > ns_timeout) {
 145                         dev_err(&pdev->xdev->dev,
 146                                 "pciback not responding!!!\n");
 147                         clear_bit(_XEN_PCIF_active,
 148                                   (unsigned long *)&pdev->sh_info->flags);
 149                         err = XEN_PCI_ERR_dev_not_found;
 150                         goto out;
 151                 }
 152         }
 153
 154         /*
 155         * We might lose backend service request since we
 156         * reuse same evtchn with pci_conf backend response. So re-schedule
 157         * aer pcifront service.
 158         */
 159         if (test_bit(_XEN_PCIB_active,
 160                         (unsigned long *)&pdev->sh_info->flags)) {
 161                 dev_err(&pdev->xdev->dev,
 162                         "schedule aer pcifront service\n");
 163                 schedule_pcifront_aer_op(pdev);
 164         }
 165
 166         memcpy(op, active_op, sizeof(struct xen_pci_op));
 167
 168         err = op->err;
 169 out:
 170         spin_unlock_irqrestore(&pdev->sh_info_lock, irq_flags);
 171         return err;
 172 }
 173
 174 /* Access to this function is spinlocked in drivers/pci/access.c */
 175 static int pcifront_bus_read(struct pci_bus *bus, unsigned int devfn,
 176                              int where, int size, u32 *val)
 177 {
 178         int err = 0;
 179         struct xen_pci_op op = {
 180                 .cmd    = XEN_PCI_OP_conf_read,
 181                 .domain = pci_domain_nr(bus),
 182                 .bus    = bus->number,
 183                 .devfn  = devfn,
 184                 .offset = where,
 185                 .size   = size,
 186         };
 187         struct pcifront_sd *sd = bus->sysdata;
 188         struct pcifront_device *pdev = pcifront_get_pdev(sd);
 189
 190         dev_dbg(&pdev->xdev->dev,
 191                 "read dev=%04x:%02x:%02x.%d - offset %x size %d\n",
 192                 pci_domain_nr(bus), bus->number, PCI_SLOT(devfn),
 193                 PCI_FUNC(devfn), where, size);
 194
 195         err = do_pci_op(pdev, &op);
 196
 197         if (likely(!err)) {
 198                 dev_dbg(&pdev->xdev->dev, "read got back value %x\n",
 199                         op.value);
 200
 201                 *val = op.value;
 202         } else if (err == -ENODEV) {
 203                 /* No device here, pretend that it just returned 0 */
 204                 err = 0;
 205                 *val = 0;
 206         }
 207
 208         return errno_to_pcibios_err(err);
 209 }
 210
 211 /* Access to this function is spinlocked in drivers/pci/access.c */
 212 static int pcifront_bus_write(struct pci_bus *bus, unsigned int devfn,
 213                               int where, int size, u32 val)
 214 {
 215         struct xen_pci_op op = {
 216                 .cmd    = XEN_PCI_OP_conf_write,
 217                 .domain = pci_domain_nr(bus),
 218                 .bus    = bus->number,
 219                 .devfn  = devfn,
 220                 .offset = where,
 221                 .size   = size,
 222                 .value  = val,
 223         };
 224         struct pcifront_sd *sd = bus->sysdata;
 225         struct pcifront_device *pdev = pcifront_get_pdev(sd);
 226
 227         dev_dbg(&pdev->xdev->dev,
 228                 "write dev=%04x:%02x:%02x.%d - offset %x size %d val %x\n",
 229                 pci_domain_nr(bus), bus->number,
 230                 PCI_SLOT(devfn), PCI_FUNC(devfn), where, size, val);
 231
 232         return errno_to_pcibios_err(do_pci_op(pdev, &op));
 233 }
 234
 235 static struct pci_ops pcifront_bus_ops = {
 236         .read = pcifront_bus_read,
 237         .write = pcifront_bus_write,
 238 };
 239
 240 #ifdef CONFIG_PCI_MSI
 241 static int pci_frontend_enable_msix(struct pci_dev *dev,
 242                                     int vector[], int nvec)
 243 {
 244         int err;
 245         int i;
 246         struct xen_pci_op op = {
 247                 .cmd    = XEN_PCI_OP_enable_msix,
 248                 .domain = pci_domain_nr(dev->bus),
 249                 .bus = dev->bus->number,
 250                 .devfn = dev->devfn,
 251                 .value = nvec,
 252         };
 253         struct pcifront_sd *sd = dev->bus->sysdata;
 254         struct pcifront_device *pdev = pcifront_get_pdev(sd);
 255         struct msi_desc *entry;
 256
 257         if (nvec > SH_INFO_MAX_VEC) {
 258                 pci_err(dev, "too many vectors (0x%x) for PCI frontend:"
 259                                    " Increase SH_INFO_MAX_VEC\n", nvec);
 260                 return -EINVAL;
 261         }
 262
 263         i = 0;
 264         for_each_pci_msi_entry(entry, dev) {
 265                 op.msix_entries[i].entry = entry->msi_attrib.entry_nr;
 266                 /* Vector is useless at this point. */
 267                 op.msix_entries[i].vector = -1;
 268                 i++;
 269         }
 270
 271         err = do_pci_op(pdev, &op);
 272
 273         if (likely(!err)) {
 274                 if (likely(!op.value)) {
 275                         /* we get the result */
 276                         for (i = 0; i < nvec; i++) {
 277                                 if (op.msix_entries[i].vector <= 0) {
 278                                         pci_warn(dev, "MSI-X entry %d is invalid: %d!\n",
 279                                                 i, op.msix_entries[i].vector);
 280                                         err = -EINVAL;
 281                                         vector[i] = -1;
 282                                         continue;
 283                                 }
 284                                 vector[i] = op.msix_entries[i].vector;
 285                         }
 286                 } else {
 287                         pr_info("enable msix get value %x\n", op.value);
 288                         err = op.value;
 289                 }
 290         } else {
 291                 pci_err(dev, "enable msix get err %x\n", err);
 292         }
 293         return err;
 294 }
 295
 296 static void pci_frontend_disable_msix(struct pci_dev *dev)
 297 {
 298         int err;
 299         struct xen_pci_op op = {
 300                 .cmd    = XEN_PCI_OP_disable_msix,
 301                 .domain = pci_domain_nr(dev->bus),
 302                 .bus = dev->bus->number,
 303                 .devfn = dev->devfn,
 304         };
 305         struct pcifront_sd *sd = dev->bus->sysdata;
 306         struct pcifront_device *pdev = pcifront_get_pdev(sd);
 307
 308         err = do_pci_op(pdev, &op);
 309
 310         /* What should do for error ? */
 311         if (err)
 312                 pci_err(dev, "pci_disable_msix get err %x\n", err);
 313 }
 314
 315 static int pci_frontend_enable_msi(struct pci_dev *dev, int vector[])
 316 {
 317         int err;
 318         struct xen_pci_op op = {
 319                 .cmd    = XEN_PCI_OP_enable_msi,
 320                 .domain = pci_domain_nr(dev->bus),
 321                 .bus = dev->bus->number,
 322                 .devfn = dev->devfn,
 323         };
 324         struct pcifront_sd *sd = dev->bus->sysdata;
 325         struct pcifront_device *pdev = pcifront_get_pdev(sd);
 326
 327         err = do_pci_op(pdev, &op);
 328         if (likely(!err)) {
 329                 vector[0] = op.value;
 330                 if (op.value <= 0) {
 331                         pci_warn(dev, "MSI entry is invalid: %d!\n",
 332                                 op.value);
 333                         err = -EINVAL;
 334                         vector[0] = -1;
 335                 }
 336         } else {
 337                 pci_err(dev, "pci frontend enable msi failed for dev "
 338                                     "%x:%x\n", op.bus, op.devfn);
 339                 err = -EINVAL;
 340         }
 341         return err;
 342 }
 343
 344 static void pci_frontend_disable_msi(struct pci_dev *dev)
 345 {
 346         int err;
 347         struct xen_pci_op op = {
 348                 .cmd    = XEN_PCI_OP_disable_msi,
 349                 .domain = pci_domain_nr(dev->bus),
 350                 .bus = dev->bus->number,
 351                 .devfn = dev->devfn,
 352         };
 353         struct pcifront_sd *sd = dev->bus->sysdata;
 354         struct pcifront_device *pdev = pcifront_get_pdev(sd);
 355
 356         err = do_pci_op(pdev, &op);
 357         if (err == XEN_PCI_ERR_dev_not_found) {
 358                 /* XXX No response from backend, what shall we do? */
 359                 pr_info("get no response from backend for disable MSI\n");
 360                 return;
 361         }
 362         if (err)
 363                 /* how can pciback notify us fail? */
 364                 pr_info("get fake response from backend\n");
 365 }
 366
 367 static struct xen_pci_frontend_ops pci_frontend_ops = {
 368         .enable_msi = pci_frontend_enable_msi,
 369         .disable_msi = pci_frontend_disable_msi,
 370         .enable_msix = pci_frontend_enable_msix,
 371         .disable_msix = pci_frontend_disable_msix,
 372 };
 373
 374 static void pci_frontend_registrar(int enable)
 375 {
 376         if (enable)
 377                 xen_pci_frontend = &pci_frontend_ops;
 378         else
 379                 xen_pci_frontend = NULL;
 380 };
 381 #else
 382 static inline void pci_frontend_registrar(int enable) { };
 383 #endif /* CONFIG_PCI_MSI */
 384
 385 /* Claim resources for the PCI frontend as-is, backend won't allow changes */
 386 static int pcifront_claim_resource(struct pci_dev *dev, void *data)
 387 {
 388         struct pcifront_device *pdev = data;
 389         int i;
 390         struct resource *r;
 391
 392         for (i = 0; i < PCI_NUM_RESOURCES; i++) {
 393                 r = &dev->resource[i];
 394
 395                 if (!r->parent && r->start && r->flags) {
 396                         dev_info(&pdev->xdev->dev, "claiming resource %s/%d\n",
 397                                 pci_name(dev), i);
 398                         if (pci_claim_resource(dev, i)) {
 399                                 dev_err(&pdev->xdev->dev, "Could not claim resource %s/%d! "
 400                                         "Device offline. Try using e820_host=1 in the guest config.\n",
 401                                         pci_name(dev), i);
 402                         }
 403                 }
 404         }
 405
 406         return 0;
 407 }
 408
 409 static int pcifront_scan_bus(struct pcifront_device *pdev,
 410                                 unsigned int domain, unsigned int bus,
 411                                 struct pci_bus *b)
 412 {
 413         struct pci_dev *d;
 414         unsigned int devfn;
 415
 416         /* Scan the bus for functions and add.
 417          * We omit handling of PCI bridge attachment because pciback prevents
 418          * bridges from being exported.
 419          */
 420         for (devfn = 0; devfn < 0x100; devfn++) {
 421                 d = pci_get_slot(b, devfn);
 422                 if (d) {
 423                         /* Device is already known. */
 424                         pci_dev_put(d);
 425                         continue;
 426                 }
 427
 428                 d = pci_scan_single_device(b, devfn);
 429                 if (d)
 430                         dev_info(&pdev->xdev->dev, "New device on "
 431                                  "%04x:%02x:%02x.%d found.\n", domain, bus,
 432                                  PCI_SLOT(devfn), PCI_FUNC(devfn));
 433         }
 434
 435         return 0;
 436 }
 437
 438 static int pcifront_scan_root(struct pcifront_device *pdev,
 439                                  unsigned int domain, unsigned int bus)
 440 {
 441         struct pci_bus *b;
 442         LIST_HEAD(resources);
 443         struct pcifront_sd *sd = NULL;
 444         struct pci_bus_entry *bus_entry = NULL;
 445         int err = 0;
 446         static struct resource busn_res = {
 447                 .start = 0,
 448                 .end = 255,
 449                 .flags = IORESOURCE_BUS,
 450         };
 451
 452 #ifndef CONFIG_PCI_DOMAINS
 453         if (domain != 0) {
 454                 dev_err(&pdev->xdev->dev,
 455                         "PCI Root in non-zero PCI Domain! domain=%d\n", domain);
 456                 dev_err(&pdev->xdev->dev,
 457                         "Please compile with CONFIG_PCI_DOMAINS\n");
 458                 err = -EINVAL;
 459                 goto err_out;
 460         }
 461 #endif
 462
 463         dev_info(&pdev->xdev->dev, "Creating PCI Frontend Bus %04x:%02x\n",
 464                  domain, bus);
 465
 466         bus_entry = kzalloc(sizeof(*bus_entry), GFP_KERNEL);
 467         sd = kzalloc(sizeof(*sd), GFP_KERNEL);
 468         if (!bus_entry || !sd) {
 469                 err = -ENOMEM;
 470                 goto err_out;
 471         }
 472         pci_add_resource(&resources, &ioport_resource);
 473         pci_add_resource(&resources, &iomem_resource);
 474         pci_add_resource(&resources, &busn_res);
 475         pcifront_init_sd(sd, domain, bus, pdev);
 476
 477         pci_lock_rescan_remove();
 478
 479         b = pci_scan_root_bus(&pdev->xdev->dev, bus,
 480                                   &pcifront_bus_ops, sd, &resources);
 481         if (!b) {
 482                 dev_err(&pdev->xdev->dev,
 483                         "Error creating PCI Frontend Bus!\n");
 484                 err = -ENOMEM;
 485                 pci_unlock_rescan_remove();
 486                 pci_free_resource_list(&resources);
 487                 goto err_out;
 488         }
 489
 490         bus_entry->bus = b;
 491
 492         list_add(&bus_entry->list, &pdev->root_buses);
 493
 494         /* pci_scan_root_bus skips devices which do not have a
 495         * devfn==0. The pcifront_scan_bus enumerates all devfn. */
 496         err = pcifront_scan_bus(pdev, domain, bus, b);
 497
 498         /* Claim resources before going "live" with our devices */
 499         pci_walk_bus(b, pcifront_claim_resource, pdev);
 500
 501         /* Create SysFS and notify udev of the devices. Aka: "going live" */
 502         pci_bus_add_devices(b);
 503
 504         pci_unlock_rescan_remove();
 505         return err;
 506
 507 err_out:
 508         kfree(bus_entry);
 509         kfree(sd);
 510
 511         return err;
 512 }
 513
 514 static int pcifront_rescan_root(struct pcifront_device *pdev,
 515                                    unsigned int domain, unsigned int bus)
 516 {
 517         int err;
 518         struct pci_bus *b;
 519
 520 #ifndef CONFIG_PCI_DOMAINS
 521         if (domain != 0) {
 522                 dev_err(&pdev->xdev->dev,
 523                         "PCI Root in non-zero PCI Domain! domain=%d\n", domain);
 524                 dev_err(&pdev->xdev->dev,
 525                         "Please compile with CONFIG_PCI_DOMAINS\n");
 526                 return -EINVAL;
 527         }
 528 #endif
 529
 530         dev_info(&pdev->xdev->dev, "Rescanning PCI Frontend Bus %04x:%02x\n",
 531                  domain, bus);
 532
 533         b = pci_find_bus(domain, bus);
 534         if (!b)
 535                 /* If the bus is unknown, create it. */
 536                 return pcifront_scan_root(pdev, domain, bus);
 537
 538         err = pcifront_scan_bus(pdev, domain, bus, b);
 539
 540         /* Claim resources before going "live" with our devices */
 541         pci_walk_bus(b, pcifront_claim_resource, pdev);
 542
 543         /* Create SysFS and notify udev of the devices. Aka: "going live" */
 544         pci_bus_add_devices(b);
 545
 546         return err;
 547 }
 548
 549 static void free_root_bus_devs(struct pci_bus *bus)
 550 {
 551         struct pci_dev *dev;
 552
 553         while (!list_empty(&bus->devices)) {
 554                 dev = container_of(bus->devices.next, struct pci_dev,
 555                                    bus_list);
 556                 pci_dbg(dev, "removing device\n");
 557                 pci_stop_and_remove_bus_device(dev);
 558         }
 559 }
 560
 561 static void pcifront_free_roots(struct pcifront_device *pdev)
 562 {
 563         struct pci_bus_entry *bus_entry, *t;
 564
 565         dev_dbg(&pdev->xdev->dev, "cleaning up root buses\n");
 566
 567         pci_lock_rescan_remove();
 568         list_for_each_entry_safe(bus_entry, t, &pdev->root_buses, list) {
 569                 list_del(&bus_entry->list);
 570
 571                 free_root_bus_devs(bus_entry->bus);
 572
 573                 kfree(bus_entry->bus->sysdata);
 574
 575                 device_unregister(bus_entry->bus->bridge);
 576                 pci_remove_bus(bus_entry->bus);
 577
 578                 kfree(bus_entry);
 579         }
 580         pci_unlock_rescan_remove();
 581 }
 582
 583 static pci_ers_result_t pcifront_common_process(int cmd,
 584                                                 struct pcifront_device *pdev,
 585                                                 pci_channel_state_t state)
 586 {
 587         pci_ers_result_t result;
 588         struct pci_driver *pdrv;
 589         int bus = pdev->sh_info->aer_op.bus;
 590         int devfn = pdev->sh_info->aer_op.devfn;
 591         int domain = pdev->sh_info->aer_op.domain;
 592         struct pci_dev *pcidev;
 593         int flag = 0;
 594
 595         dev_dbg(&pdev->xdev->dev,
 596                 "pcifront AER process: cmd %x (bus:%x, devfn%x)",
 597                 cmd, bus, devfn);
 598         result = PCI_ERS_RESULT_NONE;
 599
 600         pcidev = pci_get_domain_bus_and_slot(domain, bus, devfn);
 601         if (!pcidev || !pcidev->driver) {
 602                 dev_err(&pdev->xdev->dev, "device or AER driver is NULL\n");
 603                 pci_dev_put(pcidev);
 604                 return result;
 605         }
 606         pdrv = pcidev->driver;
 607
 608         if (pdrv) {
 609                 if (pdrv->err_handler && pdrv->err_handler->error_detected) {
 610                         pci_dbg(pcidev, "trying to call AER service\n");
 611                         if (pcidev) {
 612                                 flag = 1;
 613                                 switch (cmd) {
 614                                 case XEN_PCI_OP_aer_detected:
 615                                         result = pdrv->err_handler->
 616                                                  error_detected(pcidev, state);
 617                                         break;
 618                                 case XEN_PCI_OP_aer_mmio:
 619                                         result = pdrv->err_handler->
 620                                                  mmio_enabled(pcidev);
 621                                         break;
 622                                 case XEN_PCI_OP_aer_slotreset:
 623                                         result = pdrv->err_handler->
 624                                                  slot_reset(pcidev);
 625                                         break;
 626                                 case XEN_PCI_OP_aer_resume:
 627                                         pdrv->err_handler->resume(pcidev);
 628                                         break;
 629                                 default:
 630                                         dev_err(&pdev->xdev->dev,
 631                                                 "bad request in aer recovery "
 632                                                 "operation!\n");
 633
 634                                 }
 635                         }
 636                 }
 637         }
 638         if (!flag)
 639                 result = PCI_ERS_RESULT_NONE;
 640
 641         return result;
 642 }
 643
 644
 645 static void pcifront_do_aer(struct work_struct *data)
 646 {
 647         struct pcifront_device *pdev =
 648                 container_of(data, struct pcifront_device, op_work);
 649         int cmd = pdev->sh_info->aer_op.cmd;
 650         pci_channel_state_t state =
 651                 (pci_channel_state_t)pdev->sh_info->aer_op.err;
 652
 653         /*If a pci_conf op is in progress,
 654                 we have to wait until it is done before service aer op*/
 655         dev_dbg(&pdev->xdev->dev,
 656                 "pcifront service aer bus %x devfn %x\n",
 657                 pdev->sh_info->aer_op.bus, pdev->sh_info->aer_op.devfn);
 658
 659         pdev->sh_info->aer_op.err = pcifront_common_process(cmd, pdev, state);
 660
 661         /* Post the operation to the guest. */
 662         wmb();
 663         clear_bit(_XEN_PCIB_active, (unsigned long *)&pdev->sh_info->flags);
 664         notify_remote_via_evtchn(pdev->evtchn);
 665
 666         /*in case of we lost an aer request in four lines time_window*/
 667         smp_mb__before_atomic();
 668         clear_bit(_PDEVB_op_active, &pdev->flags);
 669         smp_mb__after_atomic();
 670
 671         schedule_pcifront_aer_op(pdev);
 672
 673 }
 674
 675 static irqreturn_t pcifront_handler_aer(int irq, void *dev)
 676 {
 677         struct pcifront_device *pdev = dev;
 678         schedule_pcifront_aer_op(pdev);
 679         return IRQ_HANDLED;
 680 }
 681 static int pcifront_connect_and_init_dma(struct pcifront_device *pdev)
 682 {
 683         int err = 0;
 684
 685         spin_lock(&pcifront_dev_lock);
 686
 687         if (!pcifront_dev) {
 688                 dev_info(&pdev->xdev->dev, "Installing PCI frontend\n");
 689                 pcifront_dev = pdev;
 690         } else
 691                 err = -EEXIST;
 692
 693         spin_unlock(&pcifront_dev_lock);
 694
 695         if (!err && !swiotlb_nr_tbl()) {
 696                 err = pci_xen_swiotlb_init_late();
 697                 if (err)
 698                         dev_err(&pdev->xdev->dev, "Could not setup SWIOTLB!\n");
 699         }
 700         return err;
 701 }
 702
 703 static void pcifront_disconnect(struct pcifront_device *pdev)
 704 {
 705         spin_lock(&pcifront_dev_lock);
 706
 707         if (pdev == pcifront_dev) {
 708                 dev_info(&pdev->xdev->dev,
 709                          "Disconnecting PCI Frontend Buses\n");
 710                 pcifront_dev = NULL;
 711         }
 712
 713         spin_unlock(&pcifront_dev_lock);
 714 }
 715 static struct pcifront_device *alloc_pdev(struct xenbus_device *xdev)
 716 {
 717         struct pcifront_device *pdev;
 718
 719         pdev = kzalloc(sizeof(struct pcifront_device), GFP_KERNEL);
 720         if (pdev == NULL)
 721                 goto out;
 722
 723         pdev->sh_info =
 724             (struct xen_pci_sharedinfo *)__get_free_page(GFP_KERNEL);
 725         if (pdev->sh_info == NULL) {
 726                 kfree(pdev);
 727                 pdev = NULL;
 728                 goto out;
 729         }
 730         pdev->sh_info->flags = 0;
 731
 732         /*Flag for registering PV AER handler*/
 733         set_bit(_XEN_PCIB_AERHANDLER, (void *)&pdev->sh_info->flags);
 734
 735         dev_set_drvdata(&xdev->dev, pdev);
 736         pdev->xdev = xdev;
 737
 738         INIT_LIST_HEAD(&pdev->root_buses);
 739
 740         spin_lock_init(&pdev->sh_info_lock);
 741
 742         pdev->evtchn = INVALID_EVTCHN;
 743         pdev->gnt_ref = INVALID_GRANT_REF;
 744         pdev->irq = -1;
 745
 746         INIT_WORK(&pdev->op_work, pcifront_do_aer);
 747
 748         dev_dbg(&xdev->dev, "Allocated pdev @ 0x%p pdev->sh_info @ 0x%p\n",
 749                 pdev, pdev->sh_info);
 750 out:
 751         return pdev;
 752 }
 753
 754 static void free_pdev(struct pcifront_device *pdev)
 755 {
 756         dev_dbg(&pdev->xdev->dev, "freeing pdev @ 0x%p\n", pdev);
 757
 758         pcifront_free_roots(pdev);
 759
 760         cancel_work_sync(&pdev->op_work);
 761
 762         if (pdev->irq >= 0)
 763                 unbind_from_irqhandler(pdev->irq, pdev);
 764
 765         if (pdev->evtchn != INVALID_EVTCHN)
 766                 xenbus_free_evtchn(pdev->xdev, pdev->evtchn);
 767
 768         if (pdev->gnt_ref != INVALID_GRANT_REF)
 769                 gnttab_end_foreign_access(pdev->gnt_ref, 0 /* r/w page */,
 770                                           (unsigned long)pdev->sh_info);
 771         else
 772                 free_page((unsigned long)pdev->sh_info);
 773
 774         dev_set_drvdata(&pdev->xdev->dev, NULL);
 775
 776         kfree(pdev);
 777 }
 778
 779 static int pcifront_publish_info(struct pcifront_device *pdev)
 780 {
 781         int err = 0;
 782         struct xenbus_transaction trans;
 783         grant_ref_t gref;
 784
 785         err = xenbus_grant_ring(pdev->xdev, pdev->sh_info, 1, &gref);
 786         if (err < 0)
 787                 goto out;
 788
 789         pdev->gnt_ref = gref;
 790
 791         err = xenbus_alloc_evtchn(pdev->xdev, &pdev->evtchn);
 792         if (err)
 793                 goto out;
 794
 795         err = bind_evtchn_to_irqhandler(pdev->evtchn, pcifront_handler_aer,
 796                 0, "pcifront", pdev);
 797
 798         if (err < 0)
 799                 return err;
 800
 801         pdev->irq = err;
 802
 803 do_publish:
 804         err = xenbus_transaction_start(&trans);
 805         if (err) {
 806                 xenbus_dev_fatal(pdev->xdev, err,
 807                                  "Error writing configuration for backend "
 808                                  "(start transaction)");
 809                 goto out;
 810         }
 811
 812         err = xenbus_printf(trans, pdev->xdev->nodename,
 813                             "pci-op-ref", "%u", pdev->gnt_ref);
 814         if (!err)
 815                 err = xenbus_printf(trans, pdev->xdev->nodename,
 816                                     "event-channel", "%u", pdev->evtchn);
 817         if (!err)
 818                 err = xenbus_printf(trans, pdev->xdev->nodename,
 819                                     "magic", XEN_PCI_MAGIC);
 820
 821         if (err) {
 822                 xenbus_transaction_end(trans, 1);
 823                 xenbus_dev_fatal(pdev->xdev, err,
 824                                  "Error writing configuration for backend");
 825                 goto out;
 826         } else {
 827                 err = xenbus_transaction_end(trans, 0);
 828                 if (err == -EAGAIN)
 829                         goto do_publish;
 830                 else if (err) {
 831                         xenbus_dev_fatal(pdev->xdev, err,
 832                                          "Error completing transaction "
 833                                          "for backend");
 834                         goto out;
 835                 }
 836         }
 837
 838         xenbus_switch_state(pdev->xdev, XenbusStateInitialised);
 839
 840         dev_dbg(&pdev->xdev->dev, "publishing successful!\n");
 841
 842 out:
 843         return err;
 844 }
 845
 846 static int pcifront_try_connect(struct pcifront_device *pdev)
 847 {
 848         int err = -EFAULT;
 849         int i, num_roots, len;
 850         char str[64];
 851         unsigned int domain, bus;
 852
 853
 854         /* Only connect once */
 855         if (xenbus_read_driver_state(pdev->xdev->nodename) !=
 856             XenbusStateInitialised)
 857                 goto out;
 858
 859         err = pcifront_connect_and_init_dma(pdev);
 860         if (err && err != -EEXIST) {
 861                 xenbus_dev_fatal(pdev->xdev, err,
 862                                  "Error setting up PCI Frontend");
 863                 goto out;
 864         }
 865
 866         err = xenbus_scanf(XBT_NIL, pdev->xdev->otherend,
 867                            "root_num", "%d", &num_roots);
 868         if (err == -ENOENT) {
 869                 xenbus_dev_error(pdev->xdev, err,
 870                                  "No PCI Roots found, trying 0000:00");
 871                 err = pcifront_scan_root(pdev, 0, 0);
 872                 if (err) {
 873                         xenbus_dev_fatal(pdev->xdev, err,
 874                                          "Error scanning PCI root 0000:00");
 875                         goto out;
 876                 }
 877                 num_roots = 0;
 878         } else if (err != 1) {
 879                 if (err == 0)
 880                         err = -EINVAL;
 881                 xenbus_dev_fatal(pdev->xdev, err,
 882                                  "Error reading number of PCI roots");
 883                 goto out;
 884         }
 885
 886         for (i = 0; i < num_roots; i++) {
 887                 len = snprintf(str, sizeof(str), "root-%d", i);
 888                 if (unlikely(len >= (sizeof(str) - 1))) {
 889                         err = -ENOMEM;
 890                         goto out;
 891                 }
 892
 893                 err = xenbus_scanf(XBT_NIL, pdev->xdev->otherend, str,
 894                                    "%x:%x", &domain, &bus);
 895                 if (err != 2) {
 896                         if (err >= 0)
 897                                 err = -EINVAL;
 898                         xenbus_dev_fatal(pdev->xdev, err,
 899                                          "Error reading PCI root %d", i);
 900                         goto out;
 901                 }
 902
 903                 err = pcifront_scan_root(pdev, domain, bus);
 904                 if (err) {
 905                         xenbus_dev_fatal(pdev->xdev, err,
 906                                          "Error scanning PCI root %04x:%02x",
 907                                          domain, bus);
 908                         goto out;
 909                 }
 910         }
 911
 912         err = xenbus_switch_state(pdev->xdev, XenbusStateConnected);
 913
 914 out:
 915         return err;
 916 }
 917
 918 static int pcifront_try_disconnect(struct pcifront_device *pdev)
 919 {
 920         int err = 0;
 921         enum xenbus_state prev_state;
 922
 923
 924         prev_state = xenbus_read_driver_state(pdev->xdev->nodename);
 925
 926         if (prev_state >= XenbusStateClosing)
 927                 goto out;
 928
 929         if (prev_state == XenbusStateConnected) {
 930                 pcifront_free_roots(pdev);
 931                 pcifront_disconnect(pdev);
 932         }
 933
 934         err = xenbus_switch_state(pdev->xdev, XenbusStateClosed);
 935
 936 out:
 937
 938         return err;
 939 }
 940
 941 static int pcifront_attach_devices(struct pcifront_device *pdev)
 942 {
 943         int err = -EFAULT;
 944         int i, num_roots, len;
 945         unsigned int domain, bus;
 946         char str[64];
 947
 948         if (xenbus_read_driver_state(pdev->xdev->nodename) !=
 949             XenbusStateReconfiguring)
 950                 goto out;
 951
 952         err = xenbus_scanf(XBT_NIL, pdev->xdev->otherend,
 953                            "root_num", "%d", &num_roots);
 954         if (err == -ENOENT) {
 955                 xenbus_dev_error(pdev->xdev, err,
 956                                  "No PCI Roots found, trying 0000:00");
 957                 err = pcifront_rescan_root(pdev, 0, 0);
 958                 if (err) {
 959                         xenbus_dev_fatal(pdev->xdev, err,
 960                                          "Error scanning PCI root 0000:00");
 961                         goto out;
 962                 }
 963                 num_roots = 0;
 964         } else if (err != 1) {
 965                 if (err == 0)
 966                         err = -EINVAL;
 967                 xenbus_dev_fatal(pdev->xdev, err,
 968                                  "Error reading number of PCI roots");
 969                 goto out;
 970         }
 971
 972         for (i = 0; i < num_roots; i++) {
 973                 len = snprintf(str, sizeof(str), "root-%d", i);
 974                 if (unlikely(len >= (sizeof(str) - 1))) {
 975                         err = -ENOMEM;
 976                         goto out;
 977                 }
 978
 979                 err = xenbus_scanf(XBT_NIL, pdev->xdev->otherend, str,
 980                                    "%x:%x", &domain, &bus);
 981                 if (err != 2) {
 982                         if (err >= 0)
 983                                 err = -EINVAL;
 984                         xenbus_dev_fatal(pdev->xdev, err,
 985                                          "Error reading PCI root %d", i);
 986                         goto out;
 987                 }
 988
 989                 err = pcifront_rescan_root(pdev, domain, bus);
 990                 if (err) {
 991                         xenbus_dev_fatal(pdev->xdev, err,
 992                                          "Error scanning PCI root %04x:%02x",
 993                                          domain, bus);
 994                         goto out;
 995                 }
 996         }
 997
 998         xenbus_switch_state(pdev->xdev, XenbusStateConnected);
 999
1000 out:
1001         return err;
1002 }
1003
1004 static int pcifront_detach_devices(struct pcifront_device *pdev)
1005 {
1006         int err = 0;
1007         int i, num_devs;
1008         unsigned int domain, bus, slot, func;
1009         struct pci_dev *pci_dev;
1010         char str[64];
1011
1012         if (xenbus_read_driver_state(pdev->xdev->nodename) !=
1013             XenbusStateConnected)
1014                 goto out;
1015
1016         err = xenbus_scanf(XBT_NIL, pdev->xdev->otherend, "num_devs", "%d",
1017                            &num_devs);
1018         if (err != 1) {
1019                 if (err >= 0)
1020                         err = -EINVAL;
1021                 xenbus_dev_fatal(pdev->xdev, err,
1022                                  "Error reading number of PCI devices");
1023                 goto out;
1024         }
1025
1026         /* Find devices being detached and remove them. */
1027         for (i = 0; i < num_devs; i++) {
1028                 int l, state;
1029                 l = snprintf(str, sizeof(str), "state-%d", i);
1030                 if (unlikely(l >= (sizeof(str) - 1))) {
1031                         err = -ENOMEM;
1032                         goto out;
1033                 }
1034                 state = xenbus_read_unsigned(pdev->xdev->otherend, str,
1035                                              XenbusStateUnknown);
1036
1037                 if (state != XenbusStateClosing)
1038                         continue;
1039
1040                 /* Remove device. */
1041                 l = snprintf(str, sizeof(str), "vdev-%d", i);
1042                 if (unlikely(l >= (sizeof(str) - 1))) {
1043                         err = -ENOMEM;
1044                         goto out;
1045                 }
1046                 err = xenbus_scanf(XBT_NIL, pdev->xdev->otherend, str,
1047                                    "%x:%x:%x.%x", &domain, &bus, &slot, &func);
1048                 if (err != 4) {
1049                         if (err >= 0)
1050                                 err = -EINVAL;
1051                         xenbus_dev_fatal(pdev->xdev, err,
1052                                          "Error reading PCI device %d", i);
1053                         goto out;
1054                 }
1055
1056                 pci_dev = pci_get_domain_bus_and_slot(domain, bus,
1057                                 PCI_DEVFN(slot, func));
1058                 if (!pci_dev) {
1059                         dev_dbg(&pdev->xdev->dev,
1060                                 "Cannot get PCI device %04x:%02x:%02x.%d\n",
1061                                 domain, bus, slot, func);
1062                         continue;
1063                 }
1064                 pci_lock_rescan_remove();
1065                 pci_stop_and_remove_bus_device(pci_dev);
1066                 pci_dev_put(pci_dev);
1067                 pci_unlock_rescan_remove();
1068
1069                 dev_dbg(&pdev->xdev->dev,
1070                         "PCI device %04x:%02x:%02x.%d removed.\n",
1071                         domain, bus, slot, func);
1072         }
1073
1074         err = xenbus_switch_state(pdev->xdev, XenbusStateReconfiguring);
1075
1076 out:
1077         return err;
1078 }
1079
1080 static void __ref pcifront_backend_changed(struct xenbus_device *xdev,
1081                                                   enum xenbus_state be_state)
1082 {
1083         struct pcifront_device *pdev = dev_get_drvdata(&xdev->dev);
1084
1085         switch (be_state) {
1086         case XenbusStateUnknown:
1087         case XenbusStateInitialising:
1088         case XenbusStateInitWait:
1089         case XenbusStateInitialised:
1090                 break;
1091
1092         case XenbusStateConnected:
1093                 pcifront_try_connect(pdev);
1094                 break;
1095
1096         case XenbusStateClosed:
1097                 if (xdev->state == XenbusStateClosed)
1098                         break;
1099                 fallthrough;    /* Missed the backend's CLOSING state */
1100         case XenbusStateClosing:
1101                 dev_warn(&xdev->dev, "backend going away!\n");
1102                 pcifront_try_disconnect(pdev);
1103                 break;
1104
1105         case XenbusStateReconfiguring:
1106                 pcifront_detach_devices(pdev);
1107                 break;
1108
1109         case XenbusStateReconfigured:
1110                 pcifront_attach_devices(pdev);
1111                 break;
1112         }
1113 }
1114
1115 static int pcifront_xenbus_probe(struct xenbus_device *xdev,
1116                                  const struct xenbus_device_id *id)
1117 {
1118         int err = 0;
1119         struct pcifront_device *pdev = alloc_pdev(xdev);
1120
1121         if (pdev == NULL) {
1122                 err = -ENOMEM;
1123                 xenbus_dev_fatal(xdev, err,
1124                                  "Error allocating pcifront_device struct");
1125                 goto out;
1126         }
1127
1128         err = pcifront_publish_info(pdev);
1129         if (err)
1130                 free_pdev(pdev);
1131
1132 out:
1133         return err;
1134 }
1135
1136 static int pcifront_xenbus_remove(struct xenbus_device *xdev)
1137 {
1138         struct pcifront_device *pdev = dev_get_drvdata(&xdev->dev);
1139         if (pdev)
1140                 free_pdev(pdev);
1141
1142         return 0;
1143 }
1144
1145 static const struct xenbus_device_id xenpci_ids[] = {
1146         {"pci"},
1147         {""},
1148 };
1149
1150 static struct xenbus_driver xenpci_driver = {
1151         .name                   = "pcifront",
1152         .ids                    = xenpci_ids,
1153         .probe                  = pcifront_xenbus_probe,
1154         .remove                 = pcifront_xenbus_remove,
1155         .otherend_changed       = pcifront_backend_changed,
1156 };
1157
1158 static int __init pcifront_init(void)
1159 {
1160         if (!xen_pv_domain() || xen_initial_domain())
1161                 return -ENODEV;
1162
1163         if (!xen_has_pv_devices())
1164                 return -ENODEV;
1165
1166         pci_frontend_registrar(1 /* enable */);
1167
1168         return xenbus_register_frontend(&xenpci_driver);
1169 }
1170
1171 static void __exit pcifront_cleanup(void)
1172 {
1173         xenbus_unregister_driver(&xenpci_driver);
1174         pci_frontend_registrar(0 /* disable */);
1175 }
1176 module_init(pcifront_init);
1177 module_exit(pcifront_cleanup);
1178
1179 MODULE_DESCRIPTION("Xen PCI passthrough frontend.");
1180 MODULE_LICENSE("GPL");
1181 MODULE_ALIAS("xen:pci");