drivers/iommu/intel/dmar.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2006, Intel Corporation.
   4  *
   5  * Copyright (C) 2006-2008 Intel Corporation
   6  * Author: Ashok Raj <[email protected]>
   7  * Author: Shaohua Li <[email protected]>
   8  * Author: Anil S Keshavamurthy <[email protected]>
   9  *
  10  * This file implements early detection/parsing of Remapping Devices
  11  * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
  12  * tables.
  13  *
  14  * These routines are used by both DMA-remapping and Interrupt-remapping
  15  */
  16
  17 #define pr_fmt(fmt)     "DMAR: " fmt
  18
  19 #include <linux/pci.h>
  20 #include <linux/dmar.h>
  21 #include <linux/iova.h>
  22 #include <linux/timer.h>
  23 #include <linux/irq.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/tboot.h>
  26 #include <linux/dmi.h>
  27 #include <linux/slab.h>
  28 #include <linux/iommu.h>
  29 #include <linux/numa.h>
  30 #include <linux/limits.h>
  31 #include <asm/irq_remapping.h>
  32
  33 #include "iommu.h"
  34 #include "../irq_remapping.h"
  35 #include "../iommu-pages.h"
  36 #include "perf.h"
  37 #include "trace.h"
  38 #include "perfmon.h"
  39
  40 typedef int (*dmar_res_handler_t)(struct acpi_dmar_header *, void *);
  41 struct dmar_res_callback {
  42         dmar_res_handler_t      cb[ACPI_DMAR_TYPE_RESERVED];
  43         void                    *arg[ACPI_DMAR_TYPE_RESERVED];
  44         bool                    ignore_unhandled;
  45         bool                    print_entry;
  46 };
  47
  48 /*
  49  * Assumptions:
  50  * 1) The hotplug framework guarentees that DMAR unit will be hot-added
  51  *    before IO devices managed by that unit.
  52  * 2) The hotplug framework guarantees that DMAR unit will be hot-removed
  53  *    after IO devices managed by that unit.
  54  * 3) Hotplug events are rare.
  55  *
  56  * Locking rules for DMA and interrupt remapping related global data structures:
  57  * 1) Use dmar_global_lock in process context
  58  * 2) Use RCU in interrupt context
  59  */
  60 DECLARE_RWSEM(dmar_global_lock);
  61 LIST_HEAD(dmar_drhd_units);
  62
  63 struct acpi_table_header * __initdata dmar_tbl;
  64 static int dmar_dev_scope_status = 1;
  65 static DEFINE_IDA(dmar_seq_ids);
  66
  67 static int alloc_iommu(struct dmar_drhd_unit *drhd);
  68 static void free_iommu(struct intel_iommu *iommu);
  69
  70 static void dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
  71 {
  72         /*
  73          * add INCLUDE_ALL at the tail, so scan the list will find it at
  74          * the very end.
  75          */
  76         if (drhd->include_all)
  77                 list_add_tail_rcu(&drhd->list, &dmar_drhd_units);
  78         else
  79                 list_add_rcu(&drhd->list, &dmar_drhd_units);
  80 }
  81
  82 void *dmar_alloc_dev_scope(void *start, void *end, int *cnt)
  83 {
  84         struct acpi_dmar_device_scope *scope;
  85
  86         *cnt = 0;
  87         while (start < end) {
  88                 scope = start;
  89                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_NAMESPACE ||
  90                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
  91                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
  92                         (*cnt)++;
  93                 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
  94                         scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
  95                         pr_warn("Unsupported device scope\n");
  96                 }
  97                 start += scope->length;
  98         }
  99         if (*cnt == 0)
 100                 return NULL;
 101
 102         return kcalloc(*cnt, sizeof(struct dmar_dev_scope), GFP_KERNEL);
 103 }
 104
 105 void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt)
 106 {
 107         int i;
 108         struct device *tmp_dev;
 109
 110         if (*devices && *cnt) {
 111                 for_each_active_dev_scope(*devices, *cnt, i, tmp_dev)
 112                         put_device(tmp_dev);
 113                 kfree(*devices);
 114         }
 115
 116         *devices = NULL;
 117         *cnt = 0;
 118 }
 119
 120 /* Optimize out kzalloc()/kfree() for normal cases */
 121 static char dmar_pci_notify_info_buf[64];
 122
 123 static struct dmar_pci_notify_info *
 124 dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event)
 125 {
 126         int level = 0;
 127         size_t size;
 128         struct pci_dev *tmp;
 129         struct dmar_pci_notify_info *info;
 130
 131         /*
 132          * Ignore devices that have a domain number higher than what can
 133          * be looked up in DMAR, e.g. VMD subdevices with domain 0x10000
 134          */
 135         if (pci_domain_nr(dev->bus) > U16_MAX)
 136                 return NULL;
 137
 138         /* Only generate path[] for device addition event */
 139         if (event == BUS_NOTIFY_ADD_DEVICE)
 140                 for (tmp = dev; tmp; tmp = tmp->bus->self)
 141                         level++;
 142
 143         size = struct_size(info, path, level);
 144         if (size <= sizeof(dmar_pci_notify_info_buf)) {
 145                 info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf;
 146         } else {
 147                 info = kzalloc(size, GFP_KERNEL);
 148                 if (!info) {
 149                         if (dmar_dev_scope_status == 0)
 150                                 dmar_dev_scope_status = -ENOMEM;
 151                         return NULL;
 152                 }
 153         }
 154
 155         info->event = event;
 156         info->dev = dev;
 157         info->seg = pci_domain_nr(dev->bus);
 158         info->level = level;
 159         if (event == BUS_NOTIFY_ADD_DEVICE) {
 160                 for (tmp = dev; tmp; tmp = tmp->bus->self) {
 161                         level--;
 162                         info->path[level].bus = tmp->bus->number;
 163                         info->path[level].device = PCI_SLOT(tmp->devfn);
 164                         info->path[level].function = PCI_FUNC(tmp->devfn);
 165                         if (pci_is_root_bus(tmp->bus))
 166                                 info->bus = tmp->bus->number;
 167                 }
 168         }
 169
 170         return info;
 171 }
 172
 173 static inline void dmar_free_pci_notify_info(struct dmar_pci_notify_info *info)
 174 {
 175         if ((void *)info != dmar_pci_notify_info_buf)
 176                 kfree(info);
 177 }
 178
 179 static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus,
 180                                 struct acpi_dmar_pci_path *path, int count)
 181 {
 182         int i;
 183
 184         if (info->bus != bus)
 185                 goto fallback;
 186         if (info->level != count)
 187                 goto fallback;
 188
 189         for (i = 0; i < count; i++) {
 190                 if (path[i].device != info->path[i].device ||
 191                     path[i].function != info->path[i].function)
 192                         goto fallback;
 193         }
 194
 195         return true;
 196
 197 fallback:
 198
 199         if (count != 1)
 200                 return false;
 201
 202         i = info->level - 1;
 203         if (bus              == info->path[i].bus &&
 204             path[0].device   == info->path[i].device &&
 205             path[0].function == info->path[i].function) {
 206                 pr_info(FW_BUG "RMRR entry for device %02x:%02x.%x is broken - applying workaround\n",
 207                         bus, path[0].device, path[0].function);
 208                 return true;
 209         }
 210
 211         return false;
 212 }
 213
 214 /* Return: > 0 if match found, 0 if no match found, < 0 if error happens */
 215 int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
 216                           void *start, void*end, u16 segment,
 217                           struct dmar_dev_scope *devices,
 218                           int devices_cnt)
 219 {
 220         int i, level;
 221         struct device *tmp, *dev = &info->dev->dev;
 222         struct acpi_dmar_device_scope *scope;
 223         struct acpi_dmar_pci_path *path;
 224
 225         if (segment != info->seg)
 226                 return 0;
 227
 228         for (; start < end; start += scope->length) {
 229                 scope = start;
 230                 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
 231                     scope->entry_type != ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 232                         continue;
 233
 234                 path = (struct acpi_dmar_pci_path *)(scope + 1);
 235                 level = (scope->length - sizeof(*scope)) / sizeof(*path);
 236                 if (!dmar_match_pci_path(info, scope->bus, path, level))
 237                         continue;
 238
 239                 /*
 240                  * We expect devices with endpoint scope to have normal PCI
 241                  * headers, and devices with bridge scope to have bridge PCI
 242                  * headers.  However PCI NTB devices may be listed in the
 243                  * DMAR table with bridge scope, even though they have a
 244                  * normal PCI header.  NTB devices are identified by class
 245                  * "BRIDGE_OTHER" (0680h) - we don't declare a socpe mismatch
 246                  * for this special case.
 247                  */
 248                 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
 249                      info->dev->hdr_type != PCI_HEADER_TYPE_NORMAL) ||
 250                     (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE &&
 251                      (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL &&
 252                       info->dev->class >> 16 != PCI_BASE_CLASS_BRIDGE))) {
 253                         pr_warn("Device scope type does not match for %s\n",
 254                                 pci_name(info->dev));
 255                         return -EINVAL;
 256                 }
 257
 258                 for_each_dev_scope(devices, devices_cnt, i, tmp)
 259                         if (tmp == NULL) {
 260                                 devices[i].bus = info->dev->bus->number;
 261                                 devices[i].devfn = info->dev->devfn;
 262                                 rcu_assign_pointer(devices[i].dev,
 263                                                    get_device(dev));
 264                                 return 1;
 265                         }
 266                 if (WARN_ON(i >= devices_cnt))
 267                         return -EINVAL;
 268         }
 269
 270         return 0;
 271 }
 272
 273 int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment,
 274                           struct dmar_dev_scope *devices, int count)
 275 {
 276         int index;
 277         struct device *tmp;
 278
 279         if (info->seg != segment)
 280                 return 0;
 281
 282         for_each_active_dev_scope(devices, count, index, tmp)
 283                 if (tmp == &info->dev->dev) {
 284                         RCU_INIT_POINTER(devices[index].dev, NULL);
 285                         synchronize_rcu();
 286                         put_device(tmp);
 287                         return 1;
 288                 }
 289
 290         return 0;
 291 }
 292
 293 static int dmar_pci_bus_add_dev(struct dmar_pci_notify_info *info)
 294 {
 295         int ret = 0;
 296         struct dmar_drhd_unit *dmaru;
 297         struct acpi_dmar_hardware_unit *drhd;
 298
 299         for_each_drhd_unit(dmaru) {
 300                 if (dmaru->include_all)
 301                         continue;
 302
 303                 drhd = container_of(dmaru->hdr,
 304                                     struct acpi_dmar_hardware_unit, header);
 305                 ret = dmar_insert_dev_scope(info, (void *)(drhd + 1),
 306                                 ((void *)drhd) + drhd->header.length,
 307                                 dmaru->segment,
 308                                 dmaru->devices, dmaru->devices_cnt);
 309                 if (ret)
 310                         break;
 311         }
 312         if (ret >= 0)
 313                 ret = dmar_iommu_notify_scope_dev(info);
 314         if (ret < 0 && dmar_dev_scope_status == 0)
 315                 dmar_dev_scope_status = ret;
 316
 317         if (ret >= 0)
 318                 intel_irq_remap_add_device(info);
 319
 320         return ret;
 321 }
 322
 323 static void  dmar_pci_bus_del_dev(struct dmar_pci_notify_info *info)
 324 {
 325         struct dmar_drhd_unit *dmaru;
 326
 327         for_each_drhd_unit(dmaru)
 328                 if (dmar_remove_dev_scope(info, dmaru->segment,
 329                         dmaru->devices, dmaru->devices_cnt))
 330                         break;
 331         dmar_iommu_notify_scope_dev(info);
 332 }
 333
 334 static inline void vf_inherit_msi_domain(struct pci_dev *pdev)
 335 {
 336         struct pci_dev *physfn = pci_physfn(pdev);
 337
 338         dev_set_msi_domain(&pdev->dev, dev_get_msi_domain(&physfn->dev));
 339 }
 340
 341 static int dmar_pci_bus_notifier(struct notifier_block *nb,
 342                                  unsigned long action, void *data)
 343 {
 344         struct pci_dev *pdev = to_pci_dev(data);
 345         struct dmar_pci_notify_info *info;
 346
 347         /* Only care about add/remove events for physical functions.
 348          * For VFs we actually do the lookup based on the corresponding
 349          * PF in device_to_iommu() anyway. */
 350         if (pdev->is_virtfn) {
 351                 /*
 352                  * Ensure that the VF device inherits the irq domain of the
 353                  * PF device. Ideally the device would inherit the domain
 354                  * from the bus, but DMAR can have multiple units per bus
 355                  * which makes this impossible. The VF 'bus' could inherit
 356                  * from the PF device, but that's yet another x86'sism to
 357                  * inflict on everybody else.
 358                  */
 359                 if (action == BUS_NOTIFY_ADD_DEVICE)
 360                         vf_inherit_msi_domain(pdev);
 361                 return NOTIFY_DONE;
 362         }
 363
 364         if (action != BUS_NOTIFY_ADD_DEVICE &&
 365             action != BUS_NOTIFY_REMOVED_DEVICE)
 366                 return NOTIFY_DONE;
 367
 368         info = dmar_alloc_pci_notify_info(pdev, action);
 369         if (!info)
 370                 return NOTIFY_DONE;
 371
 372         down_write(&dmar_global_lock);
 373         if (action == BUS_NOTIFY_ADD_DEVICE)
 374                 dmar_pci_bus_add_dev(info);
 375         else if (action == BUS_NOTIFY_REMOVED_DEVICE)
 376                 dmar_pci_bus_del_dev(info);
 377         up_write(&dmar_global_lock);
 378
 379         dmar_free_pci_notify_info(info);
 380
 381         return NOTIFY_OK;
 382 }
 383
 384 static struct notifier_block dmar_pci_bus_nb = {
 385         .notifier_call = dmar_pci_bus_notifier,
 386         .priority = 1,
 387 };
 388
 389 static struct dmar_drhd_unit *
 390 dmar_find_dmaru(struct acpi_dmar_hardware_unit *drhd)
 391 {
 392         struct dmar_drhd_unit *dmaru;
 393
 394         list_for_each_entry_rcu(dmaru, &dmar_drhd_units, list,
 395                                 dmar_rcu_check())
 396                 if (dmaru->segment == drhd->segment &&
 397                     dmaru->reg_base_addr == drhd->address)
 398                         return dmaru;
 399
 400         return NULL;
 401 }
 402
 403 /*
 404  * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
 405  * structure which uniquely represent one DMA remapping hardware unit
 406  * present in the platform
 407  */
 408 static int dmar_parse_one_drhd(struct acpi_dmar_header *header, void *arg)
 409 {
 410         struct acpi_dmar_hardware_unit *drhd;
 411         struct dmar_drhd_unit *dmaru;
 412         int ret;
 413
 414         drhd = (struct acpi_dmar_hardware_unit *)header;
 415         dmaru = dmar_find_dmaru(drhd);
 416         if (dmaru)
 417                 goto out;
 418
 419         dmaru = kzalloc(sizeof(*dmaru) + header->length, GFP_KERNEL);
 420         if (!dmaru)
 421                 return -ENOMEM;
 422
 423         /*
 424          * If header is allocated from slab by ACPI _DSM method, we need to
 425          * copy the content because the memory buffer will be freed on return.
 426          */
 427         dmaru->hdr = (void *)(dmaru + 1);
 428         memcpy(dmaru->hdr, header, header->length);
 429         dmaru->reg_base_addr = drhd->address;
 430         dmaru->segment = drhd->segment;
 431         /* The size of the register set is 2 ^ N 4 KB pages. */
 432         dmaru->reg_size = 1UL << (drhd->size + 12);
 433         dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
 434         dmaru->devices = dmar_alloc_dev_scope((void *)(drhd + 1),
 435                                               ((void *)drhd) + drhd->header.length,
 436                                               &dmaru->devices_cnt);
 437         if (dmaru->devices_cnt && dmaru->devices == NULL) {
 438                 kfree(dmaru);
 439                 return -ENOMEM;
 440         }
 441
 442         ret = alloc_iommu(dmaru);
 443         if (ret) {
 444                 dmar_free_dev_scope(&dmaru->devices,
 445                                     &dmaru->devices_cnt);
 446                 kfree(dmaru);
 447                 return ret;
 448         }
 449         dmar_register_drhd_unit(dmaru);
 450
 451 out:
 452         if (arg)
 453                 (*(int *)arg)++;
 454
 455         return 0;
 456 }
 457
 458 static void dmar_free_drhd(struct dmar_drhd_unit *dmaru)
 459 {
 460         if (dmaru->devices && dmaru->devices_cnt)
 461                 dmar_free_dev_scope(&dmaru->devices, &dmaru->devices_cnt);
 462         if (dmaru->iommu)
 463                 free_iommu(dmaru->iommu);
 464         kfree(dmaru);
 465 }
 466
 467 static int __init dmar_parse_one_andd(struct acpi_dmar_header *header,
 468                                       void *arg)
 469 {
 470         struct acpi_dmar_andd *andd = (void *)header;
 471
 472         /* Check for NUL termination within the designated length */
 473         if (strnlen(andd->device_name, header->length - 8) == header->length - 8) {
 474                 pr_warn(FW_BUG
 475                            "Your BIOS is broken; ANDD object name is not NUL-terminated\n"
 476                            "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 477                            dmi_get_system_info(DMI_BIOS_VENDOR),
 478                            dmi_get_system_info(DMI_BIOS_VERSION),
 479                            dmi_get_system_info(DMI_PRODUCT_VERSION));
 480                 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
 481                 return -EINVAL;
 482         }
 483         pr_info("ANDD device: %x name: %s\n", andd->device_number,
 484                 andd->device_name);
 485
 486         return 0;
 487 }
 488
 489 #ifdef CONFIG_ACPI_NUMA
 490 static int dmar_parse_one_rhsa(struct acpi_dmar_header *header, void *arg)
 491 {
 492         struct acpi_dmar_rhsa *rhsa;
 493         struct dmar_drhd_unit *drhd;
 494
 495         rhsa = (struct acpi_dmar_rhsa *)header;
 496         for_each_drhd_unit(drhd) {
 497                 if (drhd->reg_base_addr == rhsa->base_address) {
 498                         int node = pxm_to_node(rhsa->proximity_domain);
 499
 500                         if (node != NUMA_NO_NODE && !node_online(node))
 501                                 node = NUMA_NO_NODE;
 502                         drhd->iommu->node = node;
 503                         return 0;
 504                 }
 505         }
 506         pr_warn(FW_BUG
 507                 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
 508                 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 509                 rhsa->base_address,
 510                 dmi_get_system_info(DMI_BIOS_VENDOR),
 511                 dmi_get_system_info(DMI_BIOS_VERSION),
 512                 dmi_get_system_info(DMI_PRODUCT_VERSION));
 513         add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
 514
 515         return 0;
 516 }
 517 #else
 518 #define dmar_parse_one_rhsa             dmar_res_noop
 519 #endif
 520
 521 static void
 522 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
 523 {
 524         struct acpi_dmar_hardware_unit *drhd;
 525         struct acpi_dmar_reserved_memory *rmrr;
 526         struct acpi_dmar_atsr *atsr;
 527         struct acpi_dmar_rhsa *rhsa;
 528         struct acpi_dmar_satc *satc;
 529
 530         switch (header->type) {
 531         case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 532                 drhd = container_of(header, struct acpi_dmar_hardware_unit,
 533                                     header);
 534                 pr_info("DRHD base: %#016Lx flags: %#x\n",
 535                         (unsigned long long)drhd->address, drhd->flags);
 536                 break;
 537         case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 538                 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
 539                                     header);
 540                 pr_info("RMRR base: %#016Lx end: %#016Lx\n",
 541                         (unsigned long long)rmrr->base_address,
 542                         (unsigned long long)rmrr->end_address);
 543                 break;
 544         case ACPI_DMAR_TYPE_ROOT_ATS:
 545                 atsr = container_of(header, struct acpi_dmar_atsr, header);
 546                 pr_info("ATSR flags: %#x\n", atsr->flags);
 547                 break;
 548         case ACPI_DMAR_TYPE_HARDWARE_AFFINITY:
 549                 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
 550                 pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
 551                        (unsigned long long)rhsa->base_address,
 552                        rhsa->proximity_domain);
 553                 break;
 554         case ACPI_DMAR_TYPE_NAMESPACE:
 555                 /* We don't print this here because we need to sanity-check
 556                    it first. So print it in dmar_parse_one_andd() instead. */
 557                 break;
 558         case ACPI_DMAR_TYPE_SATC:
 559                 satc = container_of(header, struct acpi_dmar_satc, header);
 560                 pr_info("SATC flags: 0x%x\n", satc->flags);
 561                 break;
 562         }
 563 }
 564
 565 /**
 566  * dmar_table_detect - checks to see if the platform supports DMAR devices
 567  */
 568 static int __init dmar_table_detect(void)
 569 {
 570         acpi_status status = AE_OK;
 571
 572         /* if we could find DMAR table, then there are DMAR devices */
 573         status = acpi_get_table(ACPI_SIG_DMAR, 0, &dmar_tbl);
 574
 575         if (ACPI_SUCCESS(status) && !dmar_tbl) {
 576                 pr_warn("Unable to map DMAR\n");
 577                 status = AE_NOT_FOUND;
 578         }
 579
 580         return ACPI_SUCCESS(status) ? 0 : -ENOENT;
 581 }
 582
 583 static int dmar_walk_remapping_entries(struct acpi_dmar_header *start,
 584                                        size_t len, struct dmar_res_callback *cb)
 585 {
 586         struct acpi_dmar_header *iter, *next;
 587         struct acpi_dmar_header *end = ((void *)start) + len;
 588
 589         for (iter = start; iter < end; iter = next) {
 590                 next = (void *)iter + iter->length;
 591                 if (iter->length == 0) {
 592                         /* Avoid looping forever on bad ACPI tables */
 593                         pr_debug(FW_BUG "Invalid 0-length structure\n");
 594                         break;
 595                 } else if (next > end) {
 596                         /* Avoid passing table end */
 597                         pr_warn(FW_BUG "Record passes table end\n");
 598                         return -EINVAL;
 599                 }
 600
 601                 if (cb->print_entry)
 602                         dmar_table_print_dmar_entry(iter);
 603
 604                 if (iter->type >= ACPI_DMAR_TYPE_RESERVED) {
 605                         /* continue for forward compatibility */
 606                         pr_debug("Unknown DMAR structure type %d\n",
 607                                  iter->type);
 608                 } else if (cb->cb[iter->type]) {
 609                         int ret;
 610
 611                         ret = cb->cb[iter->type](iter, cb->arg[iter->type]);
 612                         if (ret)
 613                                 return ret;
 614                 } else if (!cb->ignore_unhandled) {
 615                         pr_warn("No handler for DMAR structure type %d\n",
 616                                 iter->type);
 617                         return -EINVAL;
 618                 }
 619         }
 620
 621         return 0;
 622 }
 623
 624 static inline int dmar_walk_dmar_table(struct acpi_table_dmar *dmar,
 625                                        struct dmar_res_callback *cb)
 626 {
 627         return dmar_walk_remapping_entries((void *)(dmar + 1),
 628                         dmar->header.length - sizeof(*dmar), cb);
 629 }
 630
 631 /**
 632  * parse_dmar_table - parses the DMA reporting table
 633  */
 634 static int __init
 635 parse_dmar_table(void)
 636 {
 637         struct acpi_table_dmar *dmar;
 638         int drhd_count = 0;
 639         int ret;
 640         struct dmar_res_callback cb = {
 641                 .print_entry = true,
 642                 .ignore_unhandled = true,
 643                 .arg[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &drhd_count,
 644                 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_parse_one_drhd,
 645                 .cb[ACPI_DMAR_TYPE_RESERVED_MEMORY] = &dmar_parse_one_rmrr,
 646                 .cb[ACPI_DMAR_TYPE_ROOT_ATS] = &dmar_parse_one_atsr,
 647                 .cb[ACPI_DMAR_TYPE_HARDWARE_AFFINITY] = &dmar_parse_one_rhsa,
 648                 .cb[ACPI_DMAR_TYPE_NAMESPACE] = &dmar_parse_one_andd,
 649                 .cb[ACPI_DMAR_TYPE_SATC] = &dmar_parse_one_satc,
 650         };
 651
 652         /*
 653          * Do it again, earlier dmar_tbl mapping could be mapped with
 654          * fixed map.
 655          */
 656         dmar_table_detect();
 657
 658         /*
 659          * ACPI tables may not be DMA protected by tboot, so use DMAR copy
 660          * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
 661          */
 662         dmar_tbl = tboot_get_dmar_table(dmar_tbl);
 663
 664         dmar = (struct acpi_table_dmar *)dmar_tbl;
 665         if (!dmar)
 666                 return -ENODEV;
 667
 668         if (dmar->width < PAGE_SHIFT - 1) {
 669                 pr_warn("Invalid DMAR haw\n");
 670                 return -EINVAL;
 671         }
 672
 673         pr_info("Host address width %d\n", dmar->width + 1);
 674         ret = dmar_walk_dmar_table(dmar, &cb);
 675         if (ret == 0 && drhd_count == 0)
 676                 pr_warn(FW_BUG "No DRHD structure found in DMAR table\n");
 677
 678         return ret;
 679 }
 680
 681 static int dmar_pci_device_match(struct dmar_dev_scope devices[],
 682                                  int cnt, struct pci_dev *dev)
 683 {
 684         int index;
 685         struct device *tmp;
 686
 687         while (dev) {
 688                 for_each_active_dev_scope(devices, cnt, index, tmp)
 689                         if (dev_is_pci(tmp) && dev == to_pci_dev(tmp))
 690                                 return 1;
 691
 692                 /* Check our parent */
 693                 dev = dev->bus->self;
 694         }
 695
 696         return 0;
 697 }
 698
 699 struct dmar_drhd_unit *
 700 dmar_find_matched_drhd_unit(struct pci_dev *dev)
 701 {
 702         struct dmar_drhd_unit *dmaru;
 703         struct acpi_dmar_hardware_unit *drhd;
 704
 705         dev = pci_physfn(dev);
 706
 707         rcu_read_lock();
 708         for_each_drhd_unit(dmaru) {
 709                 drhd = container_of(dmaru->hdr,
 710                                     struct acpi_dmar_hardware_unit,
 711                                     header);
 712
 713                 if (dmaru->include_all &&
 714                     drhd->segment == pci_domain_nr(dev->bus))
 715                         goto out;
 716
 717                 if (dmar_pci_device_match(dmaru->devices,
 718                                           dmaru->devices_cnt, dev))
 719                         goto out;
 720         }
 721         dmaru = NULL;
 722 out:
 723         rcu_read_unlock();
 724
 725         return dmaru;
 726 }
 727
 728 static void __init dmar_acpi_insert_dev_scope(u8 device_number,
 729                                               struct acpi_device *adev)
 730 {
 731         struct dmar_drhd_unit *dmaru;
 732         struct acpi_dmar_hardware_unit *drhd;
 733         struct acpi_dmar_device_scope *scope;
 734         struct device *tmp;
 735         int i;
 736         struct acpi_dmar_pci_path *path;
 737
 738         for_each_drhd_unit(dmaru) {
 739                 drhd = container_of(dmaru->hdr,
 740                                     struct acpi_dmar_hardware_unit,
 741                                     header);
 742
 743                 for (scope = (void *)(drhd + 1);
 744                      (unsigned long)scope < ((unsigned long)drhd) + drhd->header.length;
 745                      scope = ((void *)scope) + scope->length) {
 746                         if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_NAMESPACE)
 747                                 continue;
 748                         if (scope->enumeration_id != device_number)
 749                                 continue;
 750
 751                         path = (void *)(scope + 1);
 752                         pr_info("ACPI device \"%s\" under DMAR at %llx as %02x:%02x.%d\n",
 753                                 dev_name(&adev->dev), dmaru->reg_base_addr,
 754                                 scope->bus, path->device, path->function);
 755                         for_each_dev_scope(dmaru->devices, dmaru->devices_cnt, i, tmp)
 756                                 if (tmp == NULL) {
 757                                         dmaru->devices[i].bus = scope->bus;
 758                                         dmaru->devices[i].devfn = PCI_DEVFN(path->device,
 759                                                                             path->function);
 760                                         rcu_assign_pointer(dmaru->devices[i].dev,
 761                                                            get_device(&adev->dev));
 762                                         return;
 763                                 }
 764                         BUG_ON(i >= dmaru->devices_cnt);
 765                 }
 766         }
 767         pr_warn("No IOMMU scope found for ANDD enumeration ID %d (%s)\n",
 768                 device_number, dev_name(&adev->dev));
 769 }
 770
 771 static int __init dmar_acpi_dev_scope_init(void)
 772 {
 773         struct acpi_dmar_andd *andd;
 774
 775         if (dmar_tbl == NULL)
 776                 return -ENODEV;
 777
 778         for (andd = (void *)dmar_tbl + sizeof(struct acpi_table_dmar);
 779              ((unsigned long)andd) < ((unsigned long)dmar_tbl) + dmar_tbl->length;
 780              andd = ((void *)andd) + andd->header.length) {
 781                 if (andd->header.type == ACPI_DMAR_TYPE_NAMESPACE) {
 782                         acpi_handle h;
 783                         struct acpi_device *adev;
 784
 785                         if (!ACPI_SUCCESS(acpi_get_handle(ACPI_ROOT_OBJECT,
 786                                                           andd->device_name,
 787                                                           &h))) {
 788                                 pr_err("Failed to find handle for ACPI object %s\n",
 789                                        andd->device_name);
 790                                 continue;
 791                         }
 792                         adev = acpi_fetch_acpi_dev(h);
 793                         if (!adev) {
 794                                 pr_err("Failed to get device for ACPI object %s\n",
 795                                        andd->device_name);
 796                                 continue;
 797                         }
 798                         dmar_acpi_insert_dev_scope(andd->device_number, adev);
 799                 }
 800         }
 801         return 0;
 802 }
 803
 804 int __init dmar_dev_scope_init(void)
 805 {
 806         struct pci_dev *dev = NULL;
 807         struct dmar_pci_notify_info *info;
 808
 809         if (dmar_dev_scope_status != 1)
 810                 return dmar_dev_scope_status;
 811
 812         if (list_empty(&dmar_drhd_units)) {
 813                 dmar_dev_scope_status = -ENODEV;
 814         } else {
 815                 dmar_dev_scope_status = 0;
 816
 817                 dmar_acpi_dev_scope_init();
 818
 819                 for_each_pci_dev(dev) {
 820                         if (dev->is_virtfn)
 821                                 continue;
 822
 823                         info = dmar_alloc_pci_notify_info(dev,
 824                                         BUS_NOTIFY_ADD_DEVICE);
 825                         if (!info) {
 826                                 pci_dev_put(dev);
 827                                 return dmar_dev_scope_status;
 828                         } else {
 829                                 dmar_pci_bus_add_dev(info);
 830                                 dmar_free_pci_notify_info(info);
 831                         }
 832                 }
 833         }
 834
 835         return dmar_dev_scope_status;
 836 }
 837
 838 void __init dmar_register_bus_notifier(void)
 839 {
 840         bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb);
 841 }
 842
 843
 844 int __init dmar_table_init(void)
 845 {
 846         static int dmar_table_initialized;
 847         int ret;
 848
 849         if (dmar_table_initialized == 0) {
 850                 ret = parse_dmar_table();
 851                 if (ret < 0) {
 852                         if (ret != -ENODEV)
 853                                 pr_info("Parse DMAR table failure.\n");
 854                 } else  if (list_empty(&dmar_drhd_units)) {
 855                         pr_info("No DMAR devices found\n");
 856                         ret = -ENODEV;
 857                 }
 858
 859                 if (ret < 0)
 860                         dmar_table_initialized = ret;
 861                 else
 862                         dmar_table_initialized = 1;
 863         }
 864
 865         return dmar_table_initialized < 0 ? dmar_table_initialized : 0;
 866 }
 867
 868 static void warn_invalid_dmar(u64 addr, const char *message)
 869 {
 870         pr_warn_once(FW_BUG
 871                 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
 872                 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 873                 addr, message,
 874                 dmi_get_system_info(DMI_BIOS_VENDOR),
 875                 dmi_get_system_info(DMI_BIOS_VERSION),
 876                 dmi_get_system_info(DMI_PRODUCT_VERSION));
 877         add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
 878 }
 879
 880 static int __ref
 881 dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg)
 882 {
 883         struct acpi_dmar_hardware_unit *drhd;
 884         void __iomem *addr;
 885         u64 cap, ecap;
 886
 887         drhd = (void *)entry;
 888         if (!drhd->address) {
 889                 warn_invalid_dmar(0, "");
 890                 return -EINVAL;
 891         }
 892
 893         if (arg)
 894                 addr = ioremap(drhd->address, VTD_PAGE_SIZE);
 895         else
 896                 addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
 897         if (!addr) {
 898                 pr_warn("Can't validate DRHD address: %llx\n", drhd->address);
 899                 return -EINVAL;
 900         }
 901
 902         cap = dmar_readq(addr + DMAR_CAP_REG);
 903         ecap = dmar_readq(addr + DMAR_ECAP_REG);
 904
 905         if (arg)
 906                 iounmap(addr);
 907         else
 908                 early_iounmap(addr, VTD_PAGE_SIZE);
 909
 910         if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
 911                 warn_invalid_dmar(drhd->address, " returns all ones");
 912                 return -EINVAL;
 913         }
 914
 915         return 0;
 916 }
 917
 918 void __init detect_intel_iommu(void)
 919 {
 920         int ret;
 921         struct dmar_res_callback validate_drhd_cb = {
 922                 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_validate_one_drhd,
 923                 .ignore_unhandled = true,
 924         };
 925
 926         down_write(&dmar_global_lock);
 927         ret = dmar_table_detect();
 928         if (!ret)
 929                 ret = dmar_walk_dmar_table((struct acpi_table_dmar *)dmar_tbl,
 930                                            &validate_drhd_cb);
 931         if (!ret && !no_iommu && !iommu_detected &&
 932             (!dmar_disabled || dmar_platform_optin())) {
 933                 iommu_detected = 1;
 934                 /* Make sure ACS will be enabled */
 935                 pci_request_acs();
 936         }
 937
 938 #ifdef CONFIG_X86
 939         if (!ret) {
 940                 x86_init.iommu.iommu_init = intel_iommu_init;
 941                 x86_platform.iommu_shutdown = intel_iommu_shutdown;
 942         }
 943
 944 #endif
 945
 946         if (dmar_tbl) {
 947                 acpi_put_table(dmar_tbl);
 948                 dmar_tbl = NULL;
 949         }
 950         up_write(&dmar_global_lock);
 951 }
 952
 953 static void unmap_iommu(struct intel_iommu *iommu)
 954 {
 955         iounmap(iommu->reg);
 956         release_mem_region(iommu->reg_phys, iommu->reg_size);
 957 }
 958
 959 /**
 960  * map_iommu: map the iommu's registers
 961  * @iommu: the iommu to map
 962  * @drhd: DMA remapping hardware definition structure
 963  *
 964  * Memory map the iommu's registers.  Start w/ a single page, and
 965  * possibly expand if that turns out to be insufficent.
 966  */
 967 static int map_iommu(struct intel_iommu *iommu, struct dmar_drhd_unit *drhd)
 968 {
 969         u64 phys_addr = drhd->reg_base_addr;
 970         int map_size, err=0;
 971
 972         iommu->reg_phys = phys_addr;
 973         iommu->reg_size = drhd->reg_size;
 974
 975         if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
 976                 pr_err("Can't reserve memory\n");
 977                 err = -EBUSY;
 978                 goto out;
 979         }
 980
 981         iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
 982         if (!iommu->reg) {
 983                 pr_err("Can't map the region\n");
 984                 err = -ENOMEM;
 985                 goto release;
 986         }
 987
 988         iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 989         iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 990
 991         if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
 992                 err = -EINVAL;
 993                 warn_invalid_dmar(phys_addr, " returns all ones");
 994                 goto unmap;
 995         }
 996
 997         /* the registers might be more than one page */
 998         map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 999                          cap_max_fault_reg_offset(iommu->cap));
1000         map_size = VTD_PAGE_ALIGN(map_size);
1001         if (map_size > iommu->reg_size) {
1002                 iounmap(iommu->reg);
1003                 release_mem_region(iommu->reg_phys, iommu->reg_size);
1004                 iommu->reg_size = map_size;
1005                 if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
1006                                         iommu->name)) {
1007                         pr_err("Can't reserve memory\n");
1008                         err = -EBUSY;
1009                         goto out;
1010                 }
1011                 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
1012                 if (!iommu->reg) {
1013                         pr_err("Can't map the region\n");
1014                         err = -ENOMEM;
1015                         goto release;
1016                 }
1017         }
1018
1019         if (cap_ecmds(iommu->cap)) {
1020                 int i;
1021
1022                 for (i = 0; i < DMA_MAX_NUM_ECMDCAP; i++) {
1023                         iommu->ecmdcap[i] = dmar_readq(iommu->reg + DMAR_ECCAP_REG +
1024                                                        i * DMA_ECMD_REG_STEP);
1025                 }
1026         }
1027
1028         err = 0;
1029         goto out;
1030
1031 unmap:
1032         iounmap(iommu->reg);
1033 release:
1034         release_mem_region(iommu->reg_phys, iommu->reg_size);
1035 out:
1036         return err;
1037 }
1038
1039 static int alloc_iommu(struct dmar_drhd_unit *drhd)
1040 {
1041         struct intel_iommu *iommu;
1042         u32 ver, sts;
1043         int agaw = -1;
1044         int msagaw = -1;
1045         int err;
1046
1047         if (!drhd->reg_base_addr) {
1048                 warn_invalid_dmar(0, "");
1049                 return -EINVAL;
1050         }
1051
1052         iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
1053         if (!iommu)
1054                 return -ENOMEM;
1055
1056         iommu->seq_id = ida_alloc_range(&dmar_seq_ids, 0,
1057                                         DMAR_UNITS_SUPPORTED - 1, GFP_KERNEL);
1058         if (iommu->seq_id < 0) {
1059                 pr_err("Failed to allocate seq_id\n");
1060                 err = iommu->seq_id;
1061                 goto error;
1062         }
1063         sprintf(iommu->name, "dmar%d", iommu->seq_id);
1064
1065         err = map_iommu(iommu, drhd);
1066         if (err) {
1067                 pr_err("Failed to map %s\n", iommu->name);
1068                 goto error_free_seq_id;
1069         }
1070
1071         if (!cap_sagaw(iommu->cap) &&
1072             (!ecap_smts(iommu->ecap) || ecap_slts(iommu->ecap))) {
1073                 pr_info("%s: No supported address widths. Not attempting DMA translation.\n",
1074                         iommu->name);
1075                 drhd->ignored = 1;
1076         }
1077
1078         if (!drhd->ignored) {
1079                 agaw = iommu_calculate_agaw(iommu);
1080                 if (agaw < 0) {
1081                         pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
1082                                iommu->seq_id);
1083                         drhd->ignored = 1;
1084                 }
1085         }
1086         if (!drhd->ignored) {
1087                 msagaw = iommu_calculate_max_sagaw(iommu);
1088                 if (msagaw < 0) {
1089                         pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
1090                                iommu->seq_id);
1091                         drhd->ignored = 1;
1092                         agaw = -1;
1093                 }
1094         }
1095         iommu->agaw = agaw;
1096         iommu->msagaw = msagaw;
1097         iommu->segment = drhd->segment;
1098         iommu->device_rbtree = RB_ROOT;
1099         spin_lock_init(&iommu->device_rbtree_lock);
1100         mutex_init(&iommu->iopf_lock);
1101         iommu->node = NUMA_NO_NODE;
1102
1103         ver = readl(iommu->reg + DMAR_VER_REG);
1104         pr_info("%s: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
1105                 iommu->name,
1106                 (unsigned long long)drhd->reg_base_addr,
1107                 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
1108                 (unsigned long long)iommu->cap,
1109                 (unsigned long long)iommu->ecap);
1110
1111         /* Reflect status in gcmd */
1112         sts = readl(iommu->reg + DMAR_GSTS_REG);
1113         if (sts & DMA_GSTS_IRES)
1114                 iommu->gcmd |= DMA_GCMD_IRE;
1115         if (sts & DMA_GSTS_TES)
1116                 iommu->gcmd |= DMA_GCMD_TE;
1117         if (sts & DMA_GSTS_QIES)
1118                 iommu->gcmd |= DMA_GCMD_QIE;
1119
1120         if (alloc_iommu_pmu(iommu))
1121                 pr_debug("Cannot alloc PMU for iommu (seq_id = %d)\n", iommu->seq_id);
1122
1123         raw_spin_lock_init(&iommu->register_lock);
1124
1125         /*
1126          * A value of N in PSS field of eCap register indicates hardware
1127          * supports PASID field of N+1 bits.
1128          */
1129         if (pasid_supported(iommu))
1130                 iommu->iommu.max_pasids = 2UL << ecap_pss(iommu->ecap);
1131
1132         /*
1133          * This is only for hotplug; at boot time intel_iommu_enabled won't
1134          * be set yet. When intel_iommu_init() runs, it registers the units
1135          * present at boot time, then sets intel_iommu_enabled.
1136          */
1137         if (intel_iommu_enabled && !drhd->ignored) {
1138                 err = iommu_device_sysfs_add(&iommu->iommu, NULL,
1139                                              intel_iommu_groups,
1140                                              "%s", iommu->name);
1141                 if (err)
1142                         goto err_unmap;
1143
1144                 err = iommu_device_register(&iommu->iommu, &intel_iommu_ops, NULL);
1145                 if (err)
1146                         goto err_sysfs;
1147
1148                 iommu_pmu_register(iommu);
1149         }
1150
1151         drhd->iommu = iommu;
1152         iommu->drhd = drhd;
1153
1154         return 0;
1155
1156 err_sysfs:
1157         iommu_device_sysfs_remove(&iommu->iommu);
1158 err_unmap:
1159         free_iommu_pmu(iommu);
1160         unmap_iommu(iommu);
1161 error_free_seq_id:
1162         ida_free(&dmar_seq_ids, iommu->seq_id);
1163 error:
1164         kfree(iommu);
1165         return err;
1166 }
1167
1168 static void free_iommu(struct intel_iommu *iommu)
1169 {
1170         if (intel_iommu_enabled && !iommu->drhd->ignored) {
1171                 iommu_pmu_unregister(iommu);
1172                 iommu_device_unregister(&iommu->iommu);
1173                 iommu_device_sysfs_remove(&iommu->iommu);
1174         }
1175
1176         free_iommu_pmu(iommu);
1177
1178         if (iommu->irq) {
1179                 if (iommu->pr_irq) {
1180                         free_irq(iommu->pr_irq, iommu);
1181                         dmar_free_hwirq(iommu->pr_irq);
1182                         iommu->pr_irq = 0;
1183                 }
1184                 free_irq(iommu->irq, iommu);
1185                 dmar_free_hwirq(iommu->irq);
1186                 iommu->irq = 0;
1187         }
1188
1189         if (iommu->qi) {
1190                 iommu_free_page(iommu->qi->desc);
1191                 kfree(iommu->qi->desc_status);
1192                 kfree(iommu->qi);
1193         }
1194
1195         if (iommu->reg)
1196                 unmap_iommu(iommu);
1197
1198         ida_free(&dmar_seq_ids, iommu->seq_id);
1199         kfree(iommu);
1200 }
1201
1202 /*
1203  * Reclaim all the submitted descriptors which have completed its work.
1204  */
1205 static inline void reclaim_free_desc(struct q_inval *qi)
1206 {
1207         while (qi->desc_status[qi->free_tail] == QI_DONE ||
1208                qi->desc_status[qi->free_tail] == QI_ABORT) {
1209                 qi->desc_status[qi->free_tail] = QI_FREE;
1210                 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
1211                 qi->free_cnt++;
1212         }
1213 }
1214
1215 static const char *qi_type_string(u8 type)
1216 {
1217         switch (type) {
1218         case QI_CC_TYPE:
1219                 return "Context-cache Invalidation";
1220         case QI_IOTLB_TYPE:
1221                 return "IOTLB Invalidation";
1222         case QI_DIOTLB_TYPE:
1223                 return "Device-TLB Invalidation";
1224         case QI_IEC_TYPE:
1225                 return "Interrupt Entry Cache Invalidation";
1226         case QI_IWD_TYPE:
1227                 return "Invalidation Wait";
1228         case QI_EIOTLB_TYPE:
1229                 return "PASID-based IOTLB Invalidation";
1230         case QI_PC_TYPE:
1231                 return "PASID-cache Invalidation";
1232         case QI_DEIOTLB_TYPE:
1233                 return "PASID-based Device-TLB Invalidation";
1234         case QI_PGRP_RESP_TYPE:
1235                 return "Page Group Response";
1236         default:
1237                 return "UNKNOWN";
1238         }
1239 }
1240
1241 static void qi_dump_fault(struct intel_iommu *iommu, u32 fault)
1242 {
1243         unsigned int head = dmar_readl(iommu->reg + DMAR_IQH_REG);
1244         u64 iqe_err = dmar_readq(iommu->reg + DMAR_IQER_REG);
1245         struct qi_desc *desc = iommu->qi->desc + head;
1246
1247         if (fault & DMA_FSTS_IQE)
1248                 pr_err("VT-d detected Invalidation Queue Error: Reason %llx",
1249                        DMAR_IQER_REG_IQEI(iqe_err));
1250         if (fault & DMA_FSTS_ITE)
1251                 pr_err("VT-d detected Invalidation Time-out Error: SID %llx",
1252                        DMAR_IQER_REG_ITESID(iqe_err));
1253         if (fault & DMA_FSTS_ICE)
1254                 pr_err("VT-d detected Invalidation Completion Error: SID %llx",
1255                        DMAR_IQER_REG_ICESID(iqe_err));
1256
1257         pr_err("QI HEAD: %s qw0 = 0x%llx, qw1 = 0x%llx\n",
1258                qi_type_string(desc->qw0 & 0xf),
1259                (unsigned long long)desc->qw0,
1260                (unsigned long long)desc->qw1);
1261
1262         head = ((head >> qi_shift(iommu)) + QI_LENGTH - 1) % QI_LENGTH;
1263         head <<= qi_shift(iommu);
1264         desc = iommu->qi->desc + head;
1265
1266         pr_err("QI PRIOR: %s qw0 = 0x%llx, qw1 = 0x%llx\n",
1267                qi_type_string(desc->qw0 & 0xf),
1268                (unsigned long long)desc->qw0,
1269                (unsigned long long)desc->qw1);
1270 }
1271
1272 static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index)
1273 {
1274         u32 fault;
1275         int head, tail;
1276         struct device *dev;
1277         u64 iqe_err, ite_sid;
1278         struct q_inval *qi = iommu->qi;
1279         int shift = qi_shift(iommu);
1280
1281         if (qi->desc_status[wait_index] == QI_ABORT)
1282                 return -EAGAIN;
1283
1284         fault = readl(iommu->reg + DMAR_FSTS_REG);
1285         if (fault & (DMA_FSTS_IQE | DMA_FSTS_ITE | DMA_FSTS_ICE))
1286                 qi_dump_fault(iommu, fault);
1287
1288         /*
1289          * If IQE happens, the head points to the descriptor associated
1290          * with the error. No new descriptors are fetched until the IQE
1291          * is cleared.
1292          */
1293         if (fault & DMA_FSTS_IQE) {
1294                 head = readl(iommu->reg + DMAR_IQH_REG);
1295                 if ((head >> shift) == index) {
1296                         struct qi_desc *desc = qi->desc + head;
1297
1298                         /*
1299                          * desc->qw2 and desc->qw3 are either reserved or
1300                          * used by software as private data. We won't print
1301                          * out these two qw's for security consideration.
1302                          */
1303                         memcpy(desc, qi->desc + (wait_index << shift),
1304                                1 << shift);
1305                         writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
1306                         pr_info("Invalidation Queue Error (IQE) cleared\n");
1307                         return -EINVAL;
1308                 }
1309         }
1310
1311         /*
1312          * If ITE happens, all pending wait_desc commands are aborted.
1313          * No new descriptors are fetched until the ITE is cleared.
1314          */
1315         if (fault & DMA_FSTS_ITE) {
1316                 head = readl(iommu->reg + DMAR_IQH_REG);
1317                 head = ((head >> shift) - 1 + QI_LENGTH) % QI_LENGTH;
1318                 head |= 1;
1319                 tail = readl(iommu->reg + DMAR_IQT_REG);
1320                 tail = ((tail >> shift) - 1 + QI_LENGTH) % QI_LENGTH;
1321
1322                 /*
1323                  * SID field is valid only when the ITE field is Set in FSTS_REG
1324                  * see Intel VT-d spec r4.1, section 11.4.9.9
1325                  */
1326                 iqe_err = dmar_readq(iommu->reg + DMAR_IQER_REG);
1327                 ite_sid = DMAR_IQER_REG_ITESID(iqe_err);
1328
1329                 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
1330                 pr_info("Invalidation Time-out Error (ITE) cleared\n");
1331
1332                 do {
1333                         if (qi->desc_status[head] == QI_IN_USE)
1334                                 qi->desc_status[head] = QI_ABORT;
1335                         head = (head - 2 + QI_LENGTH) % QI_LENGTH;
1336                 } while (head != tail);
1337
1338                 /*
1339                  * If device was released or isn't present, no need to retry
1340                  * the ATS invalidate request anymore.
1341                  *
1342                  * 0 value of ite_sid means old VT-d device, no ite_sid value.
1343                  * see Intel VT-d spec r4.1, section 11.4.9.9
1344                  */
1345                 if (ite_sid) {
1346                         dev = device_rbtree_find(iommu, ite_sid);
1347                         if (!dev || !dev_is_pci(dev) ||
1348                             !pci_device_is_present(to_pci_dev(dev)))
1349                                 return -ETIMEDOUT;
1350                 }
1351                 if (qi->desc_status[wait_index] == QI_ABORT)
1352                         return -EAGAIN;
1353         }
1354
1355         if (fault & DMA_FSTS_ICE) {
1356                 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
1357                 pr_info("Invalidation Completion Error (ICE) cleared\n");
1358         }
1359
1360         return 0;
1361 }
1362
1363 /*
1364  * Function to submit invalidation descriptors of all types to the queued
1365  * invalidation interface(QI). Multiple descriptors can be submitted at a
1366  * time, a wait descriptor will be appended to each submission to ensure
1367  * hardware has completed the invalidation before return. Wait descriptors
1368  * can be part of the submission but it will not be polled for completion.
1369  */
1370 int qi_submit_sync(struct intel_iommu *iommu, struct qi_desc *desc,
1371                    unsigned int count, unsigned long options)
1372 {
1373         struct q_inval *qi = iommu->qi;
1374         s64 devtlb_start_ktime = 0;
1375         s64 iotlb_start_ktime = 0;
1376         s64 iec_start_ktime = 0;
1377         struct qi_desc wait_desc;
1378         int wait_index, index;
1379         unsigned long flags;
1380         int offset, shift;
1381         int rc, i;
1382         u64 type;
1383
1384         if (!qi)
1385                 return 0;
1386
1387         type = desc->qw0 & GENMASK_ULL(3, 0);
1388
1389         if ((type == QI_IOTLB_TYPE || type == QI_EIOTLB_TYPE) &&
1390             dmar_latency_enabled(iommu, DMAR_LATENCY_INV_IOTLB))
1391                 iotlb_start_ktime = ktime_to_ns(ktime_get());
1392
1393         if ((type == QI_DIOTLB_TYPE || type == QI_DEIOTLB_TYPE) &&
1394             dmar_latency_enabled(iommu, DMAR_LATENCY_INV_DEVTLB))
1395                 devtlb_start_ktime = ktime_to_ns(ktime_get());
1396
1397         if (type == QI_IEC_TYPE &&
1398             dmar_latency_enabled(iommu, DMAR_LATENCY_INV_IEC))
1399                 iec_start_ktime = ktime_to_ns(ktime_get());
1400
1401 restart:
1402         rc = 0;
1403
1404         raw_spin_lock_irqsave(&qi->q_lock, flags);
1405         /*
1406          * Check if we have enough empty slots in the queue to submit,
1407          * the calculation is based on:
1408          * # of desc + 1 wait desc + 1 space between head and tail
1409          */
1410         while (qi->free_cnt < count + 2) {
1411                 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1412                 cpu_relax();
1413                 raw_spin_lock_irqsave(&qi->q_lock, flags);
1414         }
1415
1416         index = qi->free_head;
1417         wait_index = (index + count) % QI_LENGTH;
1418         shift = qi_shift(iommu);
1419
1420         for (i = 0; i < count; i++) {
1421                 offset = ((index + i) % QI_LENGTH) << shift;
1422                 memcpy(qi->desc + offset, &desc[i], 1 << shift);
1423                 qi->desc_status[(index + i) % QI_LENGTH] = QI_IN_USE;
1424                 trace_qi_submit(iommu, desc[i].qw0, desc[i].qw1,
1425                                 desc[i].qw2, desc[i].qw3);
1426         }
1427         qi->desc_status[wait_index] = QI_IN_USE;
1428
1429         wait_desc.qw0 = QI_IWD_STATUS_DATA(QI_DONE) |
1430                         QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
1431         if (options & QI_OPT_WAIT_DRAIN)
1432                 wait_desc.qw0 |= QI_IWD_PRQ_DRAIN;
1433         wait_desc.qw1 = virt_to_phys(&qi->desc_status[wait_index]);
1434         wait_desc.qw2 = 0;
1435         wait_desc.qw3 = 0;
1436
1437         offset = wait_index << shift;
1438         memcpy(qi->desc + offset, &wait_desc, 1 << shift);
1439
1440         qi->free_head = (qi->free_head + count + 1) % QI_LENGTH;
1441         qi->free_cnt -= count + 1;
1442
1443         /*
1444          * update the HW tail register indicating the presence of
1445          * new descriptors.
1446          */
1447         writel(qi->free_head << shift, iommu->reg + DMAR_IQT_REG);
1448
1449         while (READ_ONCE(qi->desc_status[wait_index]) != QI_DONE) {
1450                 /*
1451                  * We will leave the interrupts disabled, to prevent interrupt
1452                  * context to queue another cmd while a cmd is already submitted
1453                  * and waiting for completion on this cpu. This is to avoid
1454                  * a deadlock where the interrupt context can wait indefinitely
1455                  * for free slots in the queue.
1456                  */
1457                 rc = qi_check_fault(iommu, index, wait_index);
1458                 if (rc)
1459                         break;
1460
1461                 raw_spin_unlock(&qi->q_lock);
1462                 cpu_relax();
1463                 raw_spin_lock(&qi->q_lock);
1464         }
1465
1466         for (i = 0; i < count; i++)
1467                 qi->desc_status[(index + i) % QI_LENGTH] = QI_DONE;
1468
1469         reclaim_free_desc(qi);
1470         raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1471
1472         if (rc == -EAGAIN)
1473                 goto restart;
1474
1475         if (iotlb_start_ktime)
1476                 dmar_latency_update(iommu, DMAR_LATENCY_INV_IOTLB,
1477                                 ktime_to_ns(ktime_get()) - iotlb_start_ktime);
1478
1479         if (devtlb_start_ktime)
1480                 dmar_latency_update(iommu, DMAR_LATENCY_INV_DEVTLB,
1481                                 ktime_to_ns(ktime_get()) - devtlb_start_ktime);
1482
1483         if (iec_start_ktime)
1484                 dmar_latency_update(iommu, DMAR_LATENCY_INV_IEC,
1485                                 ktime_to_ns(ktime_get()) - iec_start_ktime);
1486
1487         return rc;
1488 }
1489
1490 /*
1491  * Flush the global interrupt entry cache.
1492  */
1493 void qi_global_iec(struct intel_iommu *iommu)
1494 {
1495         struct qi_desc desc;
1496
1497         desc.qw0 = QI_IEC_TYPE;
1498         desc.qw1 = 0;
1499         desc.qw2 = 0;
1500         desc.qw3 = 0;
1501
1502         /* should never fail */
1503         qi_submit_sync(iommu, &desc, 1, 0);
1504 }
1505
1506 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
1507                       u64 type)
1508 {
1509         struct qi_desc desc;
1510
1511         desc.qw0 = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1512                         | QI_CC_GRAN(type) | QI_CC_TYPE;
1513         desc.qw1 = 0;
1514         desc.qw2 = 0;
1515         desc.qw3 = 0;
1516
1517         qi_submit_sync(iommu, &desc, 1, 0);
1518 }
1519
1520 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1521                     unsigned int size_order, u64 type)
1522 {
1523         u8 dw = 0, dr = 0;
1524
1525         struct qi_desc desc;
1526         int ih = 0;
1527
1528         if (cap_write_drain(iommu->cap))
1529                 dw = 1;
1530
1531         if (cap_read_drain(iommu->cap))
1532                 dr = 1;
1533
1534         desc.qw0 = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1535                 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1536         desc.qw1 = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1537                 | QI_IOTLB_AM(size_order);
1538         desc.qw2 = 0;
1539         desc.qw3 = 0;
1540
1541         qi_submit_sync(iommu, &desc, 1, 0);
1542 }
1543
1544 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
1545                         u16 qdep, u64 addr, unsigned mask)
1546 {
1547         struct qi_desc desc;
1548
1549         /*
1550          * VT-d spec, section 4.3:
1551          *
1552          * Software is recommended to not submit any Device-TLB invalidation
1553          * requests while address remapping hardware is disabled.
1554          */
1555         if (!(iommu->gcmd & DMA_GCMD_TE))
1556                 return;
1557
1558         if (mask) {
1559                 addr |= (1ULL << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1560                 desc.qw1 = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1561         } else
1562                 desc.qw1 = QI_DEV_IOTLB_ADDR(addr);
1563
1564         if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1565                 qdep = 0;
1566
1567         desc.qw0 = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1568                    QI_DIOTLB_TYPE | QI_DEV_IOTLB_PFSID(pfsid);
1569         desc.qw2 = 0;
1570         desc.qw3 = 0;
1571
1572         qi_submit_sync(iommu, &desc, 1, 0);
1573 }
1574
1575 /* PASID-based IOTLB invalidation */
1576 void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,
1577                      unsigned long npages, bool ih)
1578 {
1579         struct qi_desc desc = {.qw2 = 0, .qw3 = 0};
1580
1581         /*
1582          * npages == -1 means a PASID-selective invalidation, otherwise,
1583          * a positive value for Page-selective-within-PASID invalidation.
1584          * 0 is not a valid input.
1585          */
1586         if (WARN_ON(!npages)) {
1587                 pr_err("Invalid input npages = %ld\n", npages);
1588                 return;
1589         }
1590
1591         if (npages == -1) {
1592                 desc.qw0 = QI_EIOTLB_PASID(pasid) |
1593                                 QI_EIOTLB_DID(did) |
1594                                 QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) |
1595                                 QI_EIOTLB_TYPE;
1596                 desc.qw1 = 0;
1597         } else {
1598                 int mask = ilog2(__roundup_pow_of_two(npages));
1599                 unsigned long align = (1ULL << (VTD_PAGE_SHIFT + mask));
1600
1601                 if (WARN_ON_ONCE(!IS_ALIGNED(addr, align)))
1602                         addr = ALIGN_DOWN(addr, align);
1603
1604                 desc.qw0 = QI_EIOTLB_PASID(pasid) |
1605                                 QI_EIOTLB_DID(did) |
1606                                 QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) |
1607                                 QI_EIOTLB_TYPE;
1608                 desc.qw1 = QI_EIOTLB_ADDR(addr) |
1609                                 QI_EIOTLB_IH(ih) |
1610                                 QI_EIOTLB_AM(mask);
1611         }
1612
1613         qi_submit_sync(iommu, &desc, 1, 0);
1614 }
1615
1616 /* PASID-based device IOTLB Invalidate */
1617 void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid,
1618                               u32 pasid,  u16 qdep, u64 addr, unsigned int size_order)
1619 {
1620         unsigned long mask = 1UL << (VTD_PAGE_SHIFT + size_order - 1);
1621         struct qi_desc desc = {.qw1 = 0, .qw2 = 0, .qw3 = 0};
1622
1623         /*
1624          * VT-d spec, section 4.3:
1625          *
1626          * Software is recommended to not submit any Device-TLB invalidation
1627          * requests while address remapping hardware is disabled.
1628          */
1629         if (!(iommu->gcmd & DMA_GCMD_TE))
1630                 return;
1631
1632         desc.qw0 = QI_DEV_EIOTLB_PASID(pasid) | QI_DEV_EIOTLB_SID(sid) |
1633                 QI_DEV_EIOTLB_QDEP(qdep) | QI_DEIOTLB_TYPE |
1634                 QI_DEV_IOTLB_PFSID(pfsid);
1635
1636         /*
1637          * If S bit is 0, we only flush a single page. If S bit is set,
1638          * The least significant zero bit indicates the invalidation address
1639          * range. VT-d spec 6.5.2.6.
1640          * e.g. address bit 12[0] indicates 8KB, 13[0] indicates 16KB.
1641          * size order = 0 is PAGE_SIZE 4KB
1642          * Max Invs Pending (MIP) is set to 0 for now until we have DIT in
1643          * ECAP.
1644          */
1645         if (!IS_ALIGNED(addr, VTD_PAGE_SIZE << size_order))
1646                 pr_warn_ratelimited("Invalidate non-aligned address %llx, order %d\n",
1647                                     addr, size_order);
1648
1649         /* Take page address */
1650         desc.qw1 = QI_DEV_EIOTLB_ADDR(addr);
1651
1652         if (size_order) {
1653                 /*
1654                  * Existing 0s in address below size_order may be the least
1655                  * significant bit, we must set them to 1s to avoid having
1656                  * smaller size than desired.
1657                  */
1658                 desc.qw1 |= GENMASK_ULL(size_order + VTD_PAGE_SHIFT - 1,
1659                                         VTD_PAGE_SHIFT);
1660                 /* Clear size_order bit to indicate size */
1661                 desc.qw1 &= ~mask;
1662                 /* Set the S bit to indicate flushing more than 1 page */
1663                 desc.qw1 |= QI_DEV_EIOTLB_SIZE;
1664         }
1665
1666         qi_submit_sync(iommu, &desc, 1, 0);
1667 }
1668
1669 void qi_flush_pasid_cache(struct intel_iommu *iommu, u16 did,
1670                           u64 granu, u32 pasid)
1671 {
1672         struct qi_desc desc = {.qw1 = 0, .qw2 = 0, .qw3 = 0};
1673
1674         desc.qw0 = QI_PC_PASID(pasid) | QI_PC_DID(did) |
1675                         QI_PC_GRAN(granu) | QI_PC_TYPE;
1676         qi_submit_sync(iommu, &desc, 1, 0);
1677 }
1678
1679 /*
1680  * Disable Queued Invalidation interface.
1681  */
1682 void dmar_disable_qi(struct intel_iommu *iommu)
1683 {
1684         unsigned long flags;
1685         u32 sts;
1686         cycles_t start_time = get_cycles();
1687
1688         if (!ecap_qis(iommu->ecap))
1689                 return;
1690
1691         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1692
1693         sts =  readl(iommu->reg + DMAR_GSTS_REG);
1694         if (!(sts & DMA_GSTS_QIES))
1695                 goto end;
1696
1697         /*
1698          * Give a chance to HW to complete the pending invalidation requests.
1699          */
1700         while ((readl(iommu->reg + DMAR_IQT_REG) !=
1701                 readl(iommu->reg + DMAR_IQH_REG)) &&
1702                 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1703                 cpu_relax();
1704
1705         iommu->gcmd &= ~DMA_GCMD_QIE;
1706         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1707
1708         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1709                       !(sts & DMA_GSTS_QIES), sts);
1710 end:
1711         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1712 }
1713
1714 /*
1715  * Enable queued invalidation.
1716  */
1717 static void __dmar_enable_qi(struct intel_iommu *iommu)
1718 {
1719         u32 sts;
1720         unsigned long flags;
1721         struct q_inval *qi = iommu->qi;
1722         u64 val = virt_to_phys(qi->desc);
1723
1724         qi->free_head = qi->free_tail = 0;
1725         qi->free_cnt = QI_LENGTH;
1726
1727         /*
1728          * Set DW=1 and QS=1 in IQA_REG when Scalable Mode capability
1729          * is present.
1730          */
1731         if (ecap_smts(iommu->ecap))
1732                 val |= BIT_ULL(11) | BIT_ULL(0);
1733
1734         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1735
1736         /* write zero to the tail reg */
1737         writel(0, iommu->reg + DMAR_IQT_REG);
1738
1739         dmar_writeq(iommu->reg + DMAR_IQA_REG, val);
1740
1741         iommu->gcmd |= DMA_GCMD_QIE;
1742         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1743
1744         /* Make sure hardware complete it */
1745         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1746
1747         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1748 }
1749
1750 /*
1751  * Enable Queued Invalidation interface. This is a must to support
1752  * interrupt-remapping. Also used by DMA-remapping, which replaces
1753  * register based IOTLB invalidation.
1754  */
1755 int dmar_enable_qi(struct intel_iommu *iommu)
1756 {
1757         struct q_inval *qi;
1758         void *desc;
1759         int order;
1760
1761         if (!ecap_qis(iommu->ecap))
1762                 return -ENOENT;
1763
1764         /*
1765          * queued invalidation is already setup and enabled.
1766          */
1767         if (iommu->qi)
1768                 return 0;
1769
1770         iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1771         if (!iommu->qi)
1772                 return -ENOMEM;
1773
1774         qi = iommu->qi;
1775
1776         /*
1777          * Need two pages to accommodate 256 descriptors of 256 bits each
1778          * if the remapping hardware supports scalable mode translation.
1779          */
1780         order = ecap_smts(iommu->ecap) ? 1 : 0;
1781         desc = iommu_alloc_pages_node(iommu->node, GFP_ATOMIC, order);
1782         if (!desc) {
1783                 kfree(qi);
1784                 iommu->qi = NULL;
1785                 return -ENOMEM;
1786         }
1787
1788         qi->desc = desc;
1789
1790         qi->desc_status = kcalloc(QI_LENGTH, sizeof(int), GFP_ATOMIC);
1791         if (!qi->desc_status) {
1792                 iommu_free_page(qi->desc);
1793                 kfree(qi);
1794                 iommu->qi = NULL;
1795                 return -ENOMEM;
1796         }
1797
1798         raw_spin_lock_init(&qi->q_lock);
1799
1800         __dmar_enable_qi(iommu);
1801
1802         return 0;
1803 }
1804
1805 /* iommu interrupt handling. Most stuff are MSI-like. */
1806
1807 enum faulttype {
1808         DMA_REMAP,
1809         INTR_REMAP,
1810         UNKNOWN,
1811 };
1812
1813 static const char *dma_remap_fault_reasons[] =
1814 {
1815         "Software",
1816         "Present bit in root entry is clear",
1817         "Present bit in context entry is clear",
1818         "Invalid context entry",
1819         "Access beyond MGAW",
1820         "PTE Write access is not set",
1821         "PTE Read access is not set",
1822         "Next page table ptr is invalid",
1823         "Root table address invalid",
1824         "Context table ptr is invalid",
1825         "non-zero reserved fields in RTP",
1826         "non-zero reserved fields in CTP",
1827         "non-zero reserved fields in PTE",
1828         "PCE for translation request specifies blocking",
1829 };
1830
1831 static const char * const dma_remap_sm_fault_reasons[] = {
1832         "SM: Invalid Root Table Address",
1833         "SM: TTM 0 for request with PASID",
1834         "SM: TTM 0 for page group request",
1835         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x33-0x37 */
1836         "SM: Error attempting to access Root Entry",
1837         "SM: Present bit in Root Entry is clear",
1838         "SM: Non-zero reserved field set in Root Entry",
1839         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x3B-0x3F */
1840         "SM: Error attempting to access Context Entry",
1841         "SM: Present bit in Context Entry is clear",
1842         "SM: Non-zero reserved field set in the Context Entry",
1843         "SM: Invalid Context Entry",
1844         "SM: DTE field in Context Entry is clear",
1845         "SM: PASID Enable field in Context Entry is clear",
1846         "SM: PASID is larger than the max in Context Entry",
1847         "SM: PRE field in Context-Entry is clear",
1848         "SM: RID_PASID field error in Context-Entry",
1849         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x49-0x4F */
1850         "SM: Error attempting to access the PASID Directory Entry",
1851         "SM: Present bit in Directory Entry is clear",
1852         "SM: Non-zero reserved field set in PASID Directory Entry",
1853         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x53-0x57 */
1854         "SM: Error attempting to access PASID Table Entry",
1855         "SM: Present bit in PASID Table Entry is clear",
1856         "SM: Non-zero reserved field set in PASID Table Entry",
1857         "SM: Invalid Scalable-Mode PASID Table Entry",
1858         "SM: ERE field is clear in PASID Table Entry",
1859         "SM: SRE field is clear in PASID Table Entry",
1860         "Unknown", "Unknown",/* 0x5E-0x5F */
1861         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x60-0x67 */
1862         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x68-0x6F */
1863         "SM: Error attempting to access first-level paging entry",
1864         "SM: Present bit in first-level paging entry is clear",
1865         "SM: Non-zero reserved field set in first-level paging entry",
1866         "SM: Error attempting to access FL-PML4 entry",
1867         "SM: First-level entry address beyond MGAW in Nested translation",
1868         "SM: Read permission error in FL-PML4 entry in Nested translation",
1869         "SM: Read permission error in first-level paging entry in Nested translation",
1870         "SM: Write permission error in first-level paging entry in Nested translation",
1871         "SM: Error attempting to access second-level paging entry",
1872         "SM: Read/Write permission error in second-level paging entry",
1873         "SM: Non-zero reserved field set in second-level paging entry",
1874         "SM: Invalid second-level page table pointer",
1875         "SM: A/D bit update needed in second-level entry when set up in no snoop",
1876         "Unknown", "Unknown", "Unknown", /* 0x7D-0x7F */
1877         "SM: Address in first-level translation is not canonical",
1878         "SM: U/S set 0 for first-level translation with user privilege",
1879         "SM: No execute permission for request with PASID and ER=1",
1880         "SM: Address beyond the DMA hardware max",
1881         "SM: Second-level entry address beyond the max",
1882         "SM: No write permission for Write/AtomicOp request",
1883         "SM: No read permission for Read/AtomicOp request",
1884         "SM: Invalid address-interrupt address",
1885         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x88-0x8F */
1886         "SM: A/D bit update needed in first-level entry when set up in no snoop",
1887 };
1888
1889 static const char *irq_remap_fault_reasons[] =
1890 {
1891         "Detected reserved fields in the decoded interrupt-remapped request",
1892         "Interrupt index exceeded the interrupt-remapping table size",
1893         "Present field in the IRTE entry is clear",
1894         "Error accessing interrupt-remapping table pointed by IRTA_REG",
1895         "Detected reserved fields in the IRTE entry",
1896         "Blocked a compatibility format interrupt request",
1897         "Blocked an interrupt request due to source-id verification failure",
1898 };
1899
1900 static const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1901 {
1902         if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1903                                         ARRAY_SIZE(irq_remap_fault_reasons))) {
1904                 *fault_type = INTR_REMAP;
1905                 return irq_remap_fault_reasons[fault_reason - 0x20];
1906         } else if (fault_reason >= 0x30 && (fault_reason - 0x30 <
1907                         ARRAY_SIZE(dma_remap_sm_fault_reasons))) {
1908                 *fault_type = DMA_REMAP;
1909                 return dma_remap_sm_fault_reasons[fault_reason - 0x30];
1910         } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1911                 *fault_type = DMA_REMAP;
1912                 return dma_remap_fault_reasons[fault_reason];
1913         } else {
1914                 *fault_type = UNKNOWN;
1915                 return "Unknown";
1916         }
1917 }
1918
1919
1920 static inline int dmar_msi_reg(struct intel_iommu *iommu, int irq)
1921 {
1922         if (iommu->irq == irq)
1923                 return DMAR_FECTL_REG;
1924         else if (iommu->pr_irq == irq)
1925                 return DMAR_PECTL_REG;
1926         else if (iommu->perf_irq == irq)
1927                 return DMAR_PERFINTRCTL_REG;
1928         else
1929                 BUG();
1930 }
1931
1932 void dmar_msi_unmask(struct irq_data *data)
1933 {
1934         struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1935         int reg = dmar_msi_reg(iommu, data->irq);
1936         unsigned long flag;
1937
1938         /* unmask it */
1939         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1940         writel(0, iommu->reg + reg);
1941         /* Read a reg to force flush the post write */
1942         readl(iommu->reg + reg);
1943         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1944 }
1945
1946 void dmar_msi_mask(struct irq_data *data)
1947 {
1948         struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1949         int reg = dmar_msi_reg(iommu, data->irq);
1950         unsigned long flag;
1951
1952         /* mask it */
1953         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1954         writel(DMA_FECTL_IM, iommu->reg + reg);
1955         /* Read a reg to force flush the post write */
1956         readl(iommu->reg + reg);
1957         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1958 }
1959
1960 void dmar_msi_write(int irq, struct msi_msg *msg)
1961 {
1962         struct intel_iommu *iommu = irq_get_handler_data(irq);
1963         int reg = dmar_msi_reg(iommu, irq);
1964         unsigned long flag;
1965
1966         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1967         writel(msg->data, iommu->reg + reg + 4);
1968         writel(msg->address_lo, iommu->reg + reg + 8);
1969         writel(msg->address_hi, iommu->reg + reg + 12);
1970         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1971 }
1972
1973 void dmar_msi_read(int irq, struct msi_msg *msg)
1974 {
1975         struct intel_iommu *iommu = irq_get_handler_data(irq);
1976         int reg = dmar_msi_reg(iommu, irq);
1977         unsigned long flag;
1978
1979         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1980         msg->data = readl(iommu->reg + reg + 4);
1981         msg->address_lo = readl(iommu->reg + reg + 8);
1982         msg->address_hi = readl(iommu->reg + reg + 12);
1983         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1984 }
1985
1986 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1987                 u8 fault_reason, u32 pasid, u16 source_id,
1988                 unsigned long long addr)
1989 {
1990         const char *reason;
1991         int fault_type;
1992
1993         reason = dmar_get_fault_reason(fault_reason, &fault_type);
1994
1995         if (fault_type == INTR_REMAP) {
1996                 pr_err("[INTR-REMAP] Request device [%02x:%02x.%d] fault index 0x%llx [fault reason 0x%02x] %s\n",
1997                        source_id >> 8, PCI_SLOT(source_id & 0xFF),
1998                        PCI_FUNC(source_id & 0xFF), addr >> 48,
1999                        fault_reason, reason);
2000
2001                 return 0;
2002         }
2003
2004         if (pasid == IOMMU_PASID_INVALID)
2005                 pr_err("[%s NO_PASID] Request device [%02x:%02x.%d] fault addr 0x%llx [fault reason 0x%02x] %s\n",
2006                        type ? "DMA Read" : "DMA Write",
2007                        source_id >> 8, PCI_SLOT(source_id & 0xFF),
2008                        PCI_FUNC(source_id & 0xFF), addr,
2009                        fault_reason, reason);
2010         else
2011                 pr_err("[%s PASID 0x%x] Request device [%02x:%02x.%d] fault addr 0x%llx [fault reason 0x%02x] %s\n",
2012                        type ? "DMA Read" : "DMA Write", pasid,
2013                        source_id >> 8, PCI_SLOT(source_id & 0xFF),
2014                        PCI_FUNC(source_id & 0xFF), addr,
2015                        fault_reason, reason);
2016
2017         dmar_fault_dump_ptes(iommu, source_id, addr, pasid);
2018
2019         return 0;
2020 }
2021
2022 #define PRIMARY_FAULT_REG_LEN (16)
2023 irqreturn_t dmar_fault(int irq, void *dev_id)
2024 {
2025         struct intel_iommu *iommu = dev_id;
2026         int reg, fault_index;
2027         u32 fault_status;
2028         unsigned long flag;
2029         static DEFINE_RATELIMIT_STATE(rs,
2030                                       DEFAULT_RATELIMIT_INTERVAL,
2031                                       DEFAULT_RATELIMIT_BURST);
2032
2033         raw_spin_lock_irqsave(&iommu->register_lock, flag);
2034         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
2035         if (fault_status && __ratelimit(&rs))
2036                 pr_err("DRHD: handling fault status reg %x\n", fault_status);
2037
2038         /* TBD: ignore advanced fault log currently */
2039         if (!(fault_status & DMA_FSTS_PPF))
2040                 goto unlock_exit;
2041
2042         fault_index = dma_fsts_fault_record_index(fault_status);
2043         reg = cap_fault_reg_offset(iommu->cap);
2044         while (1) {
2045                 /* Disable printing, simply clear the fault when ratelimited */
2046                 bool ratelimited = !__ratelimit(&rs);
2047                 u8 fault_reason;
2048                 u16 source_id;
2049                 u64 guest_addr;
2050                 u32 pasid;
2051                 int type;
2052                 u32 data;
2053                 bool pasid_present;
2054
2055                 /* highest 32 bits */
2056                 data = readl(iommu->reg + reg +
2057                                 fault_index * PRIMARY_FAULT_REG_LEN + 12);
2058                 if (!(data & DMA_FRCD_F))
2059                         break;
2060
2061                 if (!ratelimited) {
2062                         fault_reason = dma_frcd_fault_reason(data);
2063                         type = dma_frcd_type(data);
2064
2065                         pasid = dma_frcd_pasid_value(data);
2066                         data = readl(iommu->reg + reg +
2067                                      fault_index * PRIMARY_FAULT_REG_LEN + 8);
2068                         source_id = dma_frcd_source_id(data);
2069
2070                         pasid_present = dma_frcd_pasid_present(data);
2071                         guest_addr = dmar_readq(iommu->reg + reg +
2072                                         fault_index * PRIMARY_FAULT_REG_LEN);
2073                         guest_addr = dma_frcd_page_addr(guest_addr);
2074                 }
2075
2076                 /* clear the fault */
2077                 writel(DMA_FRCD_F, iommu->reg + reg +
2078                         fault_index * PRIMARY_FAULT_REG_LEN + 12);
2079
2080                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
2081
2082                 if (!ratelimited)
2083                         /* Using pasid -1 if pasid is not present */
2084                         dmar_fault_do_one(iommu, type, fault_reason,
2085                                           pasid_present ? pasid : IOMMU_PASID_INVALID,
2086                                           source_id, guest_addr);
2087
2088                 fault_index++;
2089                 if (fault_index >= cap_num_fault_regs(iommu->cap))
2090                         fault_index = 0;
2091                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
2092         }
2093
2094         writel(DMA_FSTS_PFO | DMA_FSTS_PPF | DMA_FSTS_PRO,
2095                iommu->reg + DMAR_FSTS_REG);
2096
2097 unlock_exit:
2098         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
2099         return IRQ_HANDLED;
2100 }
2101
2102 int dmar_set_interrupt(struct intel_iommu *iommu)
2103 {
2104         int irq, ret;
2105
2106         /*
2107          * Check if the fault interrupt is already initialized.
2108          */
2109         if (iommu->irq)
2110                 return 0;
2111
2112         irq = dmar_alloc_hwirq(iommu->seq_id, iommu->node, iommu);
2113         if (irq > 0) {
2114                 iommu->irq = irq;
2115         } else {
2116                 pr_err("No free IRQ vectors\n");
2117                 return -EINVAL;
2118         }
2119
2120         ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
2121         if (ret)
2122                 pr_err("Can't request irq\n");
2123         return ret;
2124 }
2125
2126 int enable_drhd_fault_handling(unsigned int cpu)
2127 {
2128         struct dmar_drhd_unit *drhd;
2129         struct intel_iommu *iommu;
2130
2131         /*
2132          * Enable fault control interrupt.
2133          */
2134         for_each_iommu(iommu, drhd) {
2135                 u32 fault_status;
2136                 int ret;
2137
2138                 if (iommu->irq || iommu->node != cpu_to_node(cpu))
2139                         continue;
2140
2141                 ret = dmar_set_interrupt(iommu);
2142
2143                 if (ret) {
2144                         pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
2145                                (unsigned long long)drhd->reg_base_addr, ret);
2146                         return -1;
2147                 }
2148
2149                 /*
2150                  * Clear any previous faults.
2151                  */
2152                 dmar_fault(iommu->irq, iommu);
2153                 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
2154                 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
2155         }
2156
2157         return 0;
2158 }
2159
2160 /*
2161  * Re-enable Queued Invalidation interface.
2162  */
2163 int dmar_reenable_qi(struct intel_iommu *iommu)
2164 {
2165         if (!ecap_qis(iommu->ecap))
2166                 return -ENOENT;
2167
2168         if (!iommu->qi)
2169                 return -ENOENT;
2170
2171         /*
2172          * First disable queued invalidation.
2173          */
2174         dmar_disable_qi(iommu);
2175         /*
2176          * Then enable queued invalidation again. Since there is no pending
2177          * invalidation requests now, it's safe to re-enable queued
2178          * invalidation.
2179          */
2180         __dmar_enable_qi(iommu);
2181
2182         return 0;
2183 }
2184
2185 /*
2186  * Check interrupt remapping support in DMAR table description.
2187  */
2188 int __init dmar_ir_support(void)
2189 {
2190         struct acpi_table_dmar *dmar;
2191         dmar = (struct acpi_table_dmar *)dmar_tbl;
2192         if (!dmar)
2193                 return 0;
2194         return dmar->flags & 0x1;
2195 }
2196
2197 /* Check whether DMAR units are in use */
2198 static inline bool dmar_in_use(void)
2199 {
2200         return irq_remapping_enabled || intel_iommu_enabled;
2201 }
2202
2203 static int __init dmar_free_unused_resources(void)
2204 {
2205         struct dmar_drhd_unit *dmaru, *dmaru_n;
2206
2207         if (dmar_in_use())
2208                 return 0;
2209
2210         if (dmar_dev_scope_status != 1 && !list_empty(&dmar_drhd_units))
2211                 bus_unregister_notifier(&pci_bus_type, &dmar_pci_bus_nb);
2212
2213         down_write(&dmar_global_lock);
2214         list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) {
2215                 list_del(&dmaru->list);
2216                 dmar_free_drhd(dmaru);
2217         }
2218         up_write(&dmar_global_lock);
2219
2220         return 0;
2221 }
2222
2223 late_initcall(dmar_free_unused_resources);
2224
2225 /*
2226  * DMAR Hotplug Support
2227  * For more details, please refer to Intel(R) Virtualization Technology
2228  * for Directed-IO Architecture Specifiction, Rev 2.2, Section 8.8
2229  * "Remapping Hardware Unit Hot Plug".
2230  */
2231 static guid_t dmar_hp_guid =
2232         GUID_INIT(0xD8C1A3A6, 0xBE9B, 0x4C9B,
2233                   0x91, 0xBF, 0xC3, 0xCB, 0x81, 0xFC, 0x5D, 0xAF);
2234
2235 /*
2236  * Currently there's only one revision and BIOS will not check the revision id,
2237  * so use 0 for safety.
2238  */
2239 #define DMAR_DSM_REV_ID                 0
2240 #define DMAR_DSM_FUNC_DRHD              1
2241 #define DMAR_DSM_FUNC_ATSR              2
2242 #define DMAR_DSM_FUNC_RHSA              3
2243 #define DMAR_DSM_FUNC_SATC              4
2244
2245 static inline bool dmar_detect_dsm(acpi_handle handle, int func)
2246 {
2247         return acpi_check_dsm(handle, &dmar_hp_guid, DMAR_DSM_REV_ID, 1 << func);
2248 }
2249
2250 static int dmar_walk_dsm_resource(acpi_handle handle, int func,
2251                                   dmar_res_handler_t handler, void *arg)
2252 {
2253         int ret = -ENODEV;
2254         union acpi_object *obj;
2255         struct acpi_dmar_header *start;
2256         struct dmar_res_callback callback;
2257         static int res_type[] = {
2258                 [DMAR_DSM_FUNC_DRHD] = ACPI_DMAR_TYPE_HARDWARE_UNIT,
2259                 [DMAR_DSM_FUNC_ATSR] = ACPI_DMAR_TYPE_ROOT_ATS,
2260                 [DMAR_DSM_FUNC_RHSA] = ACPI_DMAR_TYPE_HARDWARE_AFFINITY,
2261                 [DMAR_DSM_FUNC_SATC] = ACPI_DMAR_TYPE_SATC,
2262         };
2263
2264         if (!dmar_detect_dsm(handle, func))
2265                 return 0;
2266
2267         obj = acpi_evaluate_dsm_typed(handle, &dmar_hp_guid, DMAR_DSM_REV_ID,
2268                                       func, NULL, ACPI_TYPE_BUFFER);
2269         if (!obj)
2270                 return -ENODEV;
2271
2272         memset(&callback, 0, sizeof(callback));
2273         callback.cb[res_type[func]] = handler;
2274         callback.arg[res_type[func]] = arg;
2275         start = (struct acpi_dmar_header *)obj->buffer.pointer;
2276         ret = dmar_walk_remapping_entries(start, obj->buffer.length, &callback);
2277
2278         ACPI_FREE(obj);
2279
2280         return ret;
2281 }
2282
2283 static int dmar_hp_add_drhd(struct acpi_dmar_header *header, void *arg)
2284 {
2285         int ret;
2286         struct dmar_drhd_unit *dmaru;
2287
2288         dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
2289         if (!dmaru)
2290                 return -ENODEV;
2291
2292         ret = dmar_ir_hotplug(dmaru, true);
2293         if (ret == 0)
2294                 ret = dmar_iommu_hotplug(dmaru, true);
2295
2296         return ret;
2297 }
2298
2299 static int dmar_hp_remove_drhd(struct acpi_dmar_header *header, void *arg)
2300 {
2301         int i, ret;
2302         struct device *dev;
2303         struct dmar_drhd_unit *dmaru;
2304
2305         dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
2306         if (!dmaru)
2307                 return 0;
2308
2309         /*
2310          * All PCI devices managed by this unit should have been destroyed.
2311          */
2312         if (!dmaru->include_all && dmaru->devices && dmaru->devices_cnt) {
2313                 for_each_active_dev_scope(dmaru->devices,
2314                                           dmaru->devices_cnt, i, dev)
2315                         return -EBUSY;
2316         }
2317
2318         ret = dmar_ir_hotplug(dmaru, false);
2319         if (ret == 0)
2320                 ret = dmar_iommu_hotplug(dmaru, false);
2321
2322         return ret;
2323 }
2324
2325 static int dmar_hp_release_drhd(struct acpi_dmar_header *header, void *arg)
2326 {
2327         struct dmar_drhd_unit *dmaru;
2328
2329         dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
2330         if (dmaru) {
2331                 list_del_rcu(&dmaru->list);
2332                 synchronize_rcu();
2333                 dmar_free_drhd(dmaru);
2334         }
2335
2336         return 0;
2337 }
2338
2339 static int dmar_hotplug_insert(acpi_handle handle)
2340 {
2341         int ret;
2342         int drhd_count = 0;
2343
2344         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2345                                      &dmar_validate_one_drhd, (void *)1);
2346         if (ret)
2347                 goto out;
2348
2349         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2350                                      &dmar_parse_one_drhd, (void *)&drhd_count);
2351         if (ret == 0 && drhd_count == 0) {
2352                 pr_warn(FW_BUG "No DRHD structures in buffer returned by _DSM method\n");
2353                 goto out;
2354         } else if (ret) {
2355                 goto release_drhd;
2356         }
2357
2358         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_RHSA,
2359                                      &dmar_parse_one_rhsa, NULL);
2360         if (ret)
2361                 goto release_drhd;
2362
2363         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2364                                      &dmar_parse_one_atsr, NULL);
2365         if (ret)
2366                 goto release_atsr;
2367
2368         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2369                                      &dmar_hp_add_drhd, NULL);
2370         if (!ret)
2371                 return 0;
2372
2373         dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2374                                &dmar_hp_remove_drhd, NULL);
2375 release_atsr:
2376         dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2377                                &dmar_release_one_atsr, NULL);
2378 release_drhd:
2379         dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2380                                &dmar_hp_release_drhd, NULL);
2381 out:
2382         return ret;
2383 }
2384
2385 static int dmar_hotplug_remove(acpi_handle handle)
2386 {
2387         int ret;
2388
2389         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2390                                      &dmar_check_one_atsr, NULL);
2391         if (ret)
2392                 return ret;
2393
2394         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2395                                      &dmar_hp_remove_drhd, NULL);
2396         if (ret == 0) {
2397                 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2398                                                &dmar_release_one_atsr, NULL));
2399                 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2400                                                &dmar_hp_release_drhd, NULL));
2401         } else {
2402                 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2403                                        &dmar_hp_add_drhd, NULL);
2404         }
2405
2406         return ret;
2407 }
2408
2409 static acpi_status dmar_get_dsm_handle(acpi_handle handle, u32 lvl,
2410                                        void *context, void **retval)
2411 {
2412         acpi_handle *phdl = retval;
2413
2414         if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
2415                 *phdl = handle;
2416                 return AE_CTRL_TERMINATE;
2417         }
2418
2419         return AE_OK;
2420 }
2421
2422 static int dmar_device_hotplug(acpi_handle handle, bool insert)
2423 {
2424         int ret;
2425         acpi_handle tmp = NULL;
2426         acpi_status status;
2427
2428         if (!dmar_in_use())
2429                 return 0;
2430
2431         if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
2432                 tmp = handle;
2433         } else {
2434                 status = acpi_walk_namespace(ACPI_TYPE_DEVICE, handle,
2435                                              ACPI_UINT32_MAX,
2436                                              dmar_get_dsm_handle,
2437                                              NULL, NULL, &tmp);
2438                 if (ACPI_FAILURE(status)) {
2439                         pr_warn("Failed to locate _DSM method.\n");
2440                         return -ENXIO;
2441                 }
2442         }
2443         if (tmp == NULL)
2444                 return 0;
2445
2446         down_write(&dmar_global_lock);
2447         if (insert)
2448                 ret = dmar_hotplug_insert(tmp);
2449         else
2450                 ret = dmar_hotplug_remove(tmp);
2451         up_write(&dmar_global_lock);
2452
2453         return ret;
2454 }
2455
2456 int dmar_device_add(acpi_handle handle)
2457 {
2458         return dmar_device_hotplug(handle, true);
2459 }
2460
2461 int dmar_device_remove(acpi_handle handle)
2462 {
2463         return dmar_device_hotplug(handle, false);
2464 }
2465
2466 /*
2467  * dmar_platform_optin - Is %DMA_CTRL_PLATFORM_OPT_IN_FLAG set in DMAR table
2468  *
2469  * Returns true if the platform has %DMA_CTRL_PLATFORM_OPT_IN_FLAG set in
2470  * the ACPI DMAR table. This means that the platform boot firmware has made
2471  * sure no device can issue DMA outside of RMRR regions.
2472  */
2473 bool dmar_platform_optin(void)
2474 {
2475         struct acpi_table_dmar *dmar;
2476         acpi_status status;
2477         bool ret;
2478
2479         status = acpi_get_table(ACPI_SIG_DMAR, 0,
2480                                 (struct acpi_table_header **)&dmar);
2481         if (ACPI_FAILURE(status))
2482                 return false;
2483
2484         ret = !!(dmar->flags & DMAR_PLATFORM_OPT_IN);
2485         acpi_put_table((struct acpi_table_header *)dmar);
2486
2487         return ret;
2488 }
2489 EXPORT_SYMBOL_GPL(dmar_platform_optin);