drivers/iommu/intel-iommu.c

   1 /*
   2  * Copyright © 2006-2014 Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * Authors: David Woodhouse <[email protected]>,
  14  *          Ashok Raj <[email protected]>,
  15  *          Shaohua Li <[email protected]>,
  16  *          Anil S Keshavamurthy <[email protected]>,
  17  *          Fenghua Yu <[email protected]>
  18  *          Joerg Roedel <[email protected]>
  19  */
  20
  21 #define pr_fmt(fmt)     "DMAR: " fmt
  22
  23 #include <linux/init.h>
  24 #include <linux/bitmap.h>
  25 #include <linux/debugfs.h>
  26 #include <linux/export.h>
  27 #include <linux/slab.h>
  28 #include <linux/irq.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/spinlock.h>
  31 #include <linux/pci.h>
  32 #include <linux/dmar.h>
  33 #include <linux/dma-mapping.h>
  34 #include <linux/mempool.h>
  35 #include <linux/memory.h>
  36 #include <linux/cpu.h>
  37 #include <linux/timer.h>
  38 #include <linux/io.h>
  39 #include <linux/iova.h>
  40 #include <linux/iommu.h>
  41 #include <linux/intel-iommu.h>
  42 #include <linux/syscore_ops.h>
  43 #include <linux/tboot.h>
  44 #include <linux/dmi.h>
  45 #include <linux/pci-ats.h>
  46 #include <linux/memblock.h>
  47 #include <linux/dma-contiguous.h>
  48 #include <linux/crash_dump.h>
  49 #include <asm/irq_remapping.h>
  50 #include <asm/cacheflush.h>
  51 #include <asm/iommu.h>
  52
  53 #include "irq_remapping.h"
  54
  55 #define ROOT_SIZE               VTD_PAGE_SIZE
  56 #define CONTEXT_SIZE            VTD_PAGE_SIZE
  57
  58 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  59 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
  60 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  61 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
  62
  63 #define IOAPIC_RANGE_START      (0xfee00000)
  64 #define IOAPIC_RANGE_END        (0xfeefffff)
  65 #define IOVA_START_ADDR         (0x1000)
  66
  67 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
  68
  69 #define MAX_AGAW_WIDTH 64
  70 #define MAX_AGAW_PFN_WIDTH      (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
  71
  72 #define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
  73 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
  74
  75 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
  76    to match. That way, we can use 'unsigned long' for PFNs with impunity. */
  77 #define DOMAIN_MAX_PFN(gaw)     ((unsigned long) min_t(uint64_t, \
  78                                 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
  79 #define DOMAIN_MAX_ADDR(gaw)    (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
  80
  81 /* IO virtual address start page frame number */
  82 #define IOVA_START_PFN          (1)
  83
  84 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
  85 #define DMA_32BIT_PFN           IOVA_PFN(DMA_BIT_MASK(32))
  86 #define DMA_64BIT_PFN           IOVA_PFN(DMA_BIT_MASK(64))
  87
  88 /* page table handling */
  89 #define LEVEL_STRIDE            (9)
  90 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
  91
  92 /*
  93  * This bitmap is used to advertise the page sizes our hardware support
  94  * to the IOMMU core, which will then use this information to split
  95  * physically contiguous memory regions it is mapping into page sizes
  96  * that we support.
  97  *
  98  * Traditionally the IOMMU core just handed us the mappings directly,
  99  * after making sure the size is an order of a 4KiB page and that the
 100  * mapping has natural alignment.
 101  *
 102  * To retain this behavior, we currently advertise that we support
 103  * all page sizes that are an order of 4KiB.
 104  *
 105  * If at some point we'd like to utilize the IOMMU core's new behavior,
 106  * we could change this to advertise the real page sizes we support.
 107  */
 108 #define INTEL_IOMMU_PGSIZES     (~0xFFFUL)
 109
 110 static inline int agaw_to_level(int agaw)
 111 {
 112         return agaw + 2;
 113 }
 114
 115 static inline int agaw_to_width(int agaw)
 116 {
 117         return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
 118 }
 119
 120 static inline int width_to_agaw(int width)
 121 {
 122         return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
 123 }
 124
 125 static inline unsigned int level_to_offset_bits(int level)
 126 {
 127         return (level - 1) * LEVEL_STRIDE;
 128 }
 129
 130 static inline int pfn_level_offset(unsigned long pfn, int level)
 131 {
 132         return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
 133 }
 134
 135 static inline unsigned long level_mask(int level)
 136 {
 137         return -1UL << level_to_offset_bits(level);
 138 }
 139
 140 static inline unsigned long level_size(int level)
 141 {
 142         return 1UL << level_to_offset_bits(level);
 143 }
 144
 145 static inline unsigned long align_to_level(unsigned long pfn, int level)
 146 {
 147         return (pfn + level_size(level) - 1) & level_mask(level);
 148 }
 149
 150 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
 151 {
 152         return  1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
 153 }
 154
 155 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
 156    are never going to work. */
 157 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
 158 {
 159         return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
 160 }
 161
 162 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
 163 {
 164         return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
 165 }
 166 static inline unsigned long page_to_dma_pfn(struct page *pg)
 167 {
 168         return mm_to_dma_pfn(page_to_pfn(pg));
 169 }
 170 static inline unsigned long virt_to_dma_pfn(void *p)
 171 {
 172         return page_to_dma_pfn(virt_to_page(p));
 173 }
 174
 175 /* global iommu list, set NULL for ignored DMAR units */
 176 static struct intel_iommu **g_iommus;
 177
 178 static void __init check_tylersburg_isoch(void);
 179 static int rwbf_quirk;
 180
 181 /*
 182  * set to 1 to panic kernel if can't successfully enable VT-d
 183  * (used when kernel is launched w/ TXT)
 184  */
 185 static int force_on = 0;
 186
 187 /*
 188  * 0: Present
 189  * 1-11: Reserved
 190  * 12-63: Context Ptr (12 - (haw-1))
 191  * 64-127: Reserved
 192  */
 193 struct root_entry {
 194         u64     lo;
 195         u64     hi;
 196 };
 197 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
 198
 199 /*
 200  * Take a root_entry and return the Lower Context Table Pointer (LCTP)
 201  * if marked present.
 202  */
 203 static phys_addr_t root_entry_lctp(struct root_entry *re)
 204 {
 205         if (!(re->lo & 1))
 206                 return 0;
 207
 208         return re->lo & VTD_PAGE_MASK;
 209 }
 210
 211 /*
 212  * Take a root_entry and return the Upper Context Table Pointer (UCTP)
 213  * if marked present.
 214  */
 215 static phys_addr_t root_entry_uctp(struct root_entry *re)
 216 {
 217         if (!(re->hi & 1))
 218                 return 0;
 219
 220         return re->hi & VTD_PAGE_MASK;
 221 }
 222 /*
 223  * low 64 bits:
 224  * 0: present
 225  * 1: fault processing disable
 226  * 2-3: translation type
 227  * 12-63: address space root
 228  * high 64 bits:
 229  * 0-2: address width
 230  * 3-6: aval
 231  * 8-23: domain id
 232  */
 233 struct context_entry {
 234         u64 lo;
 235         u64 hi;
 236 };
 237
 238 static inline void context_clear_pasid_enable(struct context_entry *context)
 239 {
 240         context->lo &= ~(1ULL << 11);
 241 }
 242
 243 static inline bool context_pasid_enabled(struct context_entry *context)
 244 {
 245         return !!(context->lo & (1ULL << 11));
 246 }
 247
 248 static inline void context_set_copied(struct context_entry *context)
 249 {
 250         context->hi |= (1ull << 3);
 251 }
 252
 253 static inline bool context_copied(struct context_entry *context)
 254 {
 255         return !!(context->hi & (1ULL << 3));
 256 }
 257
 258 static inline bool __context_present(struct context_entry *context)
 259 {
 260         return (context->lo & 1);
 261 }
 262
 263 static inline bool context_present(struct context_entry *context)
 264 {
 265         return context_pasid_enabled(context) ?
 266              __context_present(context) :
 267              __context_present(context) && !context_copied(context);
 268 }
 269
 270 static inline void context_set_present(struct context_entry *context)
 271 {
 272         context->lo |= 1;
 273 }
 274
 275 static inline void context_set_fault_enable(struct context_entry *context)
 276 {
 277         context->lo &= (((u64)-1) << 2) | 1;
 278 }
 279
 280 static inline void context_set_translation_type(struct context_entry *context,
 281                                                 unsigned long value)
 282 {
 283         context->lo &= (((u64)-1) << 4) | 3;
 284         context->lo |= (value & 3) << 2;
 285 }
 286
 287 static inline void context_set_address_root(struct context_entry *context,
 288                                             unsigned long value)
 289 {
 290         context->lo &= ~VTD_PAGE_MASK;
 291         context->lo |= value & VTD_PAGE_MASK;
 292 }
 293
 294 static inline void context_set_address_width(struct context_entry *context,
 295                                              unsigned long value)
 296 {
 297         context->hi |= value & 7;
 298 }
 299
 300 static inline void context_set_domain_id(struct context_entry *context,
 301                                          unsigned long value)
 302 {
 303         context->hi |= (value & ((1 << 16) - 1)) << 8;
 304 }
 305
 306 static inline int context_domain_id(struct context_entry *c)
 307 {
 308         return((c->hi >> 8) & 0xffff);
 309 }
 310
 311 static inline void context_clear_entry(struct context_entry *context)
 312 {
 313         context->lo = 0;
 314         context->hi = 0;
 315 }
 316
 317 /*
 318  * 0: readable
 319  * 1: writable
 320  * 2-6: reserved
 321  * 7: super page
 322  * 8-10: available
 323  * 11: snoop behavior
 324  * 12-63: Host physcial address
 325  */
 326 struct dma_pte {
 327         u64 val;
 328 };
 329
 330 static inline void dma_clear_pte(struct dma_pte *pte)
 331 {
 332         pte->val = 0;
 333 }
 334
 335 static inline u64 dma_pte_addr(struct dma_pte *pte)
 336 {
 337 #ifdef CONFIG_64BIT
 338         return pte->val & VTD_PAGE_MASK;
 339 #else
 340         /* Must have a full atomic 64-bit read */
 341         return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
 342 #endif
 343 }
 344
 345 static inline bool dma_pte_present(struct dma_pte *pte)
 346 {
 347         return (pte->val & 3) != 0;
 348 }
 349
 350 static inline bool dma_pte_superpage(struct dma_pte *pte)
 351 {
 352         return (pte->val & DMA_PTE_LARGE_PAGE);
 353 }
 354
 355 static inline int first_pte_in_page(struct dma_pte *pte)
 356 {
 357         return !((unsigned long)pte & ~VTD_PAGE_MASK);
 358 }
 359
 360 /*
 361  * This domain is a statically identity mapping domain.
 362  *      1. This domain creats a static 1:1 mapping to all usable memory.
 363  *      2. It maps to each iommu if successful.
 364  *      3. Each iommu mapps to this domain if successful.
 365  */
 366 static struct dmar_domain *si_domain;
 367 static int hw_pass_through = 1;
 368
 369 /*
 370  * Domain represents a virtual machine, more than one devices
 371  * across iommus may be owned in one domain, e.g. kvm guest.
 372  */
 373 #define DOMAIN_FLAG_VIRTUAL_MACHINE     (1 << 0)
 374
 375 /* si_domain contains mulitple devices */
 376 #define DOMAIN_FLAG_STATIC_IDENTITY     (1 << 1)
 377
 378 #define for_each_domain_iommu(idx, domain)                      \
 379         for (idx = 0; idx < g_num_of_iommus; idx++)             \
 380                 if (domain->iommu_refcnt[idx])
 381
 382 struct dmar_domain {
 383         int     nid;                    /* node id */
 384
 385         unsigned        iommu_refcnt[DMAR_UNITS_SUPPORTED];
 386                                         /* Refcount of devices per iommu */
 387
 388
 389         u16             iommu_did[DMAR_UNITS_SUPPORTED];
 390                                         /* Domain ids per IOMMU. Use u16 since
 391                                          * domain ids are 16 bit wide according
 392                                          * to VT-d spec, section 9.3 */
 393
 394         bool has_iotlb_device;
 395         struct list_head devices;       /* all devices' list */
 396         struct iova_domain iovad;       /* iova's that belong to this domain */
 397
 398         struct dma_pte  *pgd;           /* virtual address */
 399         int             gaw;            /* max guest address width */
 400
 401         /* adjusted guest address width, 0 is level 2 30-bit */
 402         int             agaw;
 403
 404         int             flags;          /* flags to find out type of domain */
 405
 406         int             iommu_coherency;/* indicate coherency of iommu access */
 407         int             iommu_snooping; /* indicate snooping control feature*/
 408         int             iommu_count;    /* reference count of iommu */
 409         int             iommu_superpage;/* Level of superpages supported:
 410                                            0 == 4KiB (no superpages), 1 == 2MiB,
 411                                            2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
 412         u64             max_addr;       /* maximum mapped address */
 413
 414         struct iommu_domain domain;     /* generic domain data structure for
 415                                            iommu core */
 416 };
 417
 418 /* PCI domain-device relationship */
 419 struct device_domain_info {
 420         struct list_head link;  /* link to domain siblings */
 421         struct list_head global; /* link to global list */
 422         u8 bus;                 /* PCI bus number */
 423         u8 devfn;               /* PCI devfn number */
 424         u8 pasid_supported:3;
 425         u8 pasid_enabled:1;
 426         u8 pri_supported:1;
 427         u8 pri_enabled:1;
 428         u8 ats_supported:1;
 429         u8 ats_enabled:1;
 430         u8 ats_qdep;
 431         struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
 432         struct intel_iommu *iommu; /* IOMMU used by this device */
 433         struct dmar_domain *domain; /* pointer to domain */
 434 };
 435
 436 struct dmar_rmrr_unit {
 437         struct list_head list;          /* list of rmrr units   */
 438         struct acpi_dmar_header *hdr;   /* ACPI header          */
 439         u64     base_address;           /* reserved base address*/
 440         u64     end_address;            /* reserved end address */
 441         struct dmar_dev_scope *devices; /* target devices */
 442         int     devices_cnt;            /* target device count */
 443 };
 444
 445 struct dmar_atsr_unit {
 446         struct list_head list;          /* list of ATSR units */
 447         struct acpi_dmar_header *hdr;   /* ACPI header */
 448         struct dmar_dev_scope *devices; /* target devices */
 449         int devices_cnt;                /* target device count */
 450         u8 include_all:1;               /* include all ports */
 451 };
 452
 453 static LIST_HEAD(dmar_atsr_units);
 454 static LIST_HEAD(dmar_rmrr_units);
 455
 456 #define for_each_rmrr_units(rmrr) \
 457         list_for_each_entry(rmrr, &dmar_rmrr_units, list)
 458
 459 static void flush_unmaps_timeout(unsigned long data);
 460
 461 struct deferred_flush_entry {
 462         unsigned long iova_pfn;
 463         unsigned long nrpages;
 464         struct dmar_domain *domain;
 465         struct page *freelist;
 466 };
 467
 468 #define HIGH_WATER_MARK 250
 469 struct deferred_flush_table {
 470         int next;
 471         struct deferred_flush_entry entries[HIGH_WATER_MARK];
 472 };
 473
 474 struct deferred_flush_data {
 475         spinlock_t lock;
 476         int timer_on;
 477         struct timer_list timer;
 478         long size;
 479         struct deferred_flush_table *tables;
 480 };
 481
 482 DEFINE_PER_CPU(struct deferred_flush_data, deferred_flush);
 483
 484 /* bitmap for indexing intel_iommus */
 485 static int g_num_of_iommus;
 486
 487 static void domain_exit(struct dmar_domain *domain);
 488 static void domain_remove_dev_info(struct dmar_domain *domain);
 489 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
 490                                      struct device *dev);
 491 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
 492 static void domain_context_clear(struct intel_iommu *iommu,
 493                                  struct device *dev);
 494 static int domain_detach_iommu(struct dmar_domain *domain,
 495                                struct intel_iommu *iommu);
 496
 497 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
 498 int dmar_disabled = 0;
 499 #else
 500 int dmar_disabled = 1;
 501 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
 502
 503 int intel_iommu_enabled = 0;
 504 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
 505
 506 static int dmar_map_gfx = 1;
 507 static int dmar_forcedac;
 508 static int intel_iommu_strict;
 509 static int intel_iommu_superpage = 1;
 510 static int intel_iommu_ecs = 1;
 511 static int intel_iommu_pasid28;
 512 static int iommu_identity_mapping;
 513
 514 #define IDENTMAP_ALL            1
 515 #define IDENTMAP_GFX            2
 516 #define IDENTMAP_AZALIA         4
 517
 518 /* Broadwell and Skylake have broken ECS support — normal so-called "second
 519  * level" translation of DMA requests-without-PASID doesn't actually happen
 520  * unless you also set the NESTE bit in an extended context-entry. Which of
 521  * course means that SVM doesn't work because it's trying to do nested
 522  * translation of the physical addresses it finds in the process page tables,
 523  * through the IOVA->phys mapping found in the "second level" page tables.
 524  *
 525  * The VT-d specification was retroactively changed to change the definition
 526  * of the capability bits and pretend that Broadwell/Skylake never happened...
 527  * but unfortunately the wrong bit was changed. It's ECS which is broken, but
 528  * for some reason it was the PASID capability bit which was redefined (from
 529  * bit 28 on BDW/SKL to bit 40 in future).
 530  *
 531  * So our test for ECS needs to eschew those implementations which set the old
 532  * PASID capabiity bit 28, since those are the ones on which ECS is broken.
 533  * Unless we are working around the 'pasid28' limitations, that is, by putting
 534  * the device into passthrough mode for normal DMA and thus masking the bug.
 535  */
 536 #define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
 537                             (intel_iommu_pasid28 || !ecap_broken_pasid(iommu->ecap)))
 538 /* PASID support is thus enabled if ECS is enabled and *either* of the old
 539  * or new capability bits are set. */
 540 #define pasid_enabled(iommu) (ecs_enabled(iommu) &&                     \
 541                               (ecap_pasid(iommu->ecap) || ecap_broken_pasid(iommu->ecap)))
 542
 543 int intel_iommu_gfx_mapped;
 544 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
 545
 546 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 547 static DEFINE_SPINLOCK(device_domain_lock);
 548 static LIST_HEAD(device_domain_list);
 549
 550 static const struct iommu_ops intel_iommu_ops;
 551
 552 static bool translation_pre_enabled(struct intel_iommu *iommu)
 553 {
 554         return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
 555 }
 556
 557 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
 558 {
 559         iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
 560 }
 561
 562 static void init_translation_status(struct intel_iommu *iommu)
 563 {
 564         u32 gsts;
 565
 566         gsts = readl(iommu->reg + DMAR_GSTS_REG);
 567         if (gsts & DMA_GSTS_TES)
 568                 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
 569 }
 570
 571 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
 572 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
 573 {
 574         return container_of(dom, struct dmar_domain, domain);
 575 }
 576
 577 static int __init intel_iommu_setup(char *str)
 578 {
 579         if (!str)
 580                 return -EINVAL;
 581         while (*str) {
 582                 if (!strncmp(str, "on", 2)) {
 583                         dmar_disabled = 0;
 584                         pr_info("IOMMU enabled\n");
 585                 } else if (!strncmp(str, "off", 3)) {
 586                         dmar_disabled = 1;
 587                         pr_info("IOMMU disabled\n");
 588                 } else if (!strncmp(str, "igfx_off", 8)) {
 589                         dmar_map_gfx = 0;
 590                         pr_info("Disable GFX device mapping\n");
 591                 } else if (!strncmp(str, "forcedac", 8)) {
 592                         pr_info("Forcing DAC for PCI devices\n");
 593                         dmar_forcedac = 1;
 594                 } else if (!strncmp(str, "strict", 6)) {
 595                         pr_info("Disable batched IOTLB flush\n");
 596                         intel_iommu_strict = 1;
 597                 } else if (!strncmp(str, "sp_off", 6)) {
 598                         pr_info("Disable supported super page\n");
 599                         intel_iommu_superpage = 0;
 600                 } else if (!strncmp(str, "ecs_off", 7)) {
 601                         printk(KERN_INFO
 602                                 "Intel-IOMMU: disable extended context table support\n");
 603                         intel_iommu_ecs = 0;
 604                 } else if (!strncmp(str, "pasid28", 7)) {
 605                         printk(KERN_INFO
 606                                 "Intel-IOMMU: enable pre-production PASID support\n");
 607                         intel_iommu_pasid28 = 1;
 608                         iommu_identity_mapping |= IDENTMAP_GFX;
 609                 }
 610
 611                 str += strcspn(str, ",");
 612                 while (*str == ',')
 613                         str++;
 614         }
 615         return 0;
 616 }
 617 __setup("intel_iommu=", intel_iommu_setup);
 618
 619 static struct kmem_cache *iommu_domain_cache;
 620 static struct kmem_cache *iommu_devinfo_cache;
 621
 622 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
 623 {
 624         struct dmar_domain **domains;
 625         int idx = did >> 8;
 626
 627         domains = iommu->domains[idx];
 628         if (!domains)
 629                 return NULL;
 630
 631         return domains[did & 0xff];
 632 }
 633
 634 static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
 635                              struct dmar_domain *domain)
 636 {
 637         struct dmar_domain **domains;
 638         int idx = did >> 8;
 639
 640         if (!iommu->domains[idx]) {
 641                 size_t size = 256 * sizeof(struct dmar_domain *);
 642                 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
 643         }
 644
 645         domains = iommu->domains[idx];
 646         if (WARN_ON(!domains))
 647                 return;
 648         else
 649                 domains[did & 0xff] = domain;
 650 }
 651
 652 static inline void *alloc_pgtable_page(int node)
 653 {
 654         struct page *page;
 655         void *vaddr = NULL;
 656
 657         page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
 658         if (page)
 659                 vaddr = page_address(page);
 660         return vaddr;
 661 }
 662
 663 static inline void free_pgtable_page(void *vaddr)
 664 {
 665         free_page((unsigned long)vaddr);
 666 }
 667
 668 static inline void *alloc_domain_mem(void)
 669 {
 670         return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
 671 }
 672
 673 static void free_domain_mem(void *vaddr)
 674 {
 675         kmem_cache_free(iommu_domain_cache, vaddr);
 676 }
 677
 678 static inline void * alloc_devinfo_mem(void)
 679 {
 680         return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
 681 }
 682
 683 static inline void free_devinfo_mem(void *vaddr)
 684 {
 685         kmem_cache_free(iommu_devinfo_cache, vaddr);
 686 }
 687
 688 static inline int domain_type_is_vm(struct dmar_domain *domain)
 689 {
 690         return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
 691 }
 692
 693 static inline int domain_type_is_si(struct dmar_domain *domain)
 694 {
 695         return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
 696 }
 697
 698 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
 699 {
 700         return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
 701                                 DOMAIN_FLAG_STATIC_IDENTITY);
 702 }
 703
 704 static inline int domain_pfn_supported(struct dmar_domain *domain,
 705                                        unsigned long pfn)
 706 {
 707         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 708
 709         return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
 710 }
 711
 712 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
 713 {
 714         unsigned long sagaw;
 715         int agaw = -1;
 716
 717         sagaw = cap_sagaw(iommu->cap);
 718         for (agaw = width_to_agaw(max_gaw);
 719              agaw >= 0; agaw--) {
 720                 if (test_bit(agaw, &sagaw))
 721                         break;
 722         }
 723
 724         return agaw;
 725 }
 726
 727 /*
 728  * Calculate max SAGAW for each iommu.
 729  */
 730 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
 731 {
 732         return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
 733 }
 734
 735 /*
 736  * calculate agaw for each iommu.
 737  * "SAGAW" may be different across iommus, use a default agaw, and
 738  * get a supported less agaw for iommus that don't support the default agaw.
 739  */
 740 int iommu_calculate_agaw(struct intel_iommu *iommu)
 741 {
 742         return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
 743 }
 744
 745 /* This functionin only returns single iommu in a domain */
 746 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
 747 {
 748         int iommu_id;
 749
 750         /* si_domain and vm domain should not get here. */
 751         BUG_ON(domain_type_is_vm_or_si(domain));
 752         for_each_domain_iommu(iommu_id, domain)
 753                 break;
 754
 755         if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
 756                 return NULL;
 757
 758         return g_iommus[iommu_id];
 759 }
 760
 761 static void domain_update_iommu_coherency(struct dmar_domain *domain)
 762 {
 763         struct dmar_drhd_unit *drhd;
 764         struct intel_iommu *iommu;
 765         bool found = false;
 766         int i;
 767
 768         domain->iommu_coherency = 1;
 769
 770         for_each_domain_iommu(i, domain) {
 771                 found = true;
 772                 if (!ecap_coherent(g_iommus[i]->ecap)) {
 773                         domain->iommu_coherency = 0;
 774                         break;
 775                 }
 776         }
 777         if (found)
 778                 return;
 779
 780         /* No hardware attached; use lowest common denominator */
 781         rcu_read_lock();
 782         for_each_active_iommu(iommu, drhd) {
 783                 if (!ecap_coherent(iommu->ecap)) {
 784                         domain->iommu_coherency = 0;
 785                         break;
 786                 }
 787         }
 788         rcu_read_unlock();
 789 }
 790
 791 static int domain_update_iommu_snooping(struct intel_iommu *skip)
 792 {
 793         struct dmar_drhd_unit *drhd;
 794         struct intel_iommu *iommu;
 795         int ret = 1;
 796
 797         rcu_read_lock();
 798         for_each_active_iommu(iommu, drhd) {
 799                 if (iommu != skip) {
 800                         if (!ecap_sc_support(iommu->ecap)) {
 801                                 ret = 0;
 802                                 break;
 803                         }
 804                 }
 805         }
 806         rcu_read_unlock();
 807
 808         return ret;
 809 }
 810
 811 static int domain_update_iommu_superpage(struct intel_iommu *skip)
 812 {
 813         struct dmar_drhd_unit *drhd;
 814         struct intel_iommu *iommu;
 815         int mask = 0xf;
 816
 817         if (!intel_iommu_superpage) {
 818                 return 0;
 819         }
 820
 821         /* set iommu_superpage to the smallest common denominator */
 822         rcu_read_lock();
 823         for_each_active_iommu(iommu, drhd) {
 824                 if (iommu != skip) {
 825                         mask &= cap_super_page_val(iommu->cap);
 826                         if (!mask)
 827                                 break;
 828                 }
 829         }
 830         rcu_read_unlock();
 831
 832         return fls(mask);
 833 }
 834
 835 /* Some capabilities may be different across iommus */
 836 static void domain_update_iommu_cap(struct dmar_domain *domain)
 837 {
 838         domain_update_iommu_coherency(domain);
 839         domain->iommu_snooping = domain_update_iommu_snooping(NULL);
 840         domain->iommu_superpage = domain_update_iommu_superpage(NULL);
 841 }
 842
 843 static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
 844                                                        u8 bus, u8 devfn, int alloc)
 845 {
 846         struct root_entry *root = &iommu->root_entry[bus];
 847         struct context_entry *context;
 848         u64 *entry;
 849
 850         entry = &root->lo;
 851         if (ecs_enabled(iommu)) {
 852                 if (devfn >= 0x80) {
 853                         devfn -= 0x80;
 854                         entry = &root->hi;
 855                 }
 856                 devfn *= 2;
 857         }
 858         if (*entry & 1)
 859                 context = phys_to_virt(*entry & VTD_PAGE_MASK);
 860         else {
 861                 unsigned long phy_addr;
 862                 if (!alloc)
 863                         return NULL;
 864
 865                 context = alloc_pgtable_page(iommu->node);
 866                 if (!context)
 867                         return NULL;
 868
 869                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 870                 phy_addr = virt_to_phys((void *)context);
 871                 *entry = phy_addr | 1;
 872                 __iommu_flush_cache(iommu, entry, sizeof(*entry));
 873         }
 874         return &context[devfn];
 875 }
 876
 877 static int iommu_dummy(struct device *dev)
 878 {
 879         return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
 880 }
 881
 882 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
 883 {
 884         struct dmar_drhd_unit *drhd = NULL;
 885         struct intel_iommu *iommu;
 886         struct device *tmp;
 887         struct pci_dev *ptmp, *pdev = NULL;
 888         u16 segment = 0;
 889         int i;
 890
 891         if (iommu_dummy(dev))
 892                 return NULL;
 893
 894         if (dev_is_pci(dev)) {
 895                 pdev = to_pci_dev(dev);
 896                 segment = pci_domain_nr(pdev->bus);
 897         } else if (has_acpi_companion(dev))
 898                 dev = &ACPI_COMPANION(dev)->dev;
 899
 900         rcu_read_lock();
 901         for_each_active_iommu(iommu, drhd) {
 902                 if (pdev && segment != drhd->segment)
 903                         continue;
 904
 905                 for_each_active_dev_scope(drhd->devices,
 906                                           drhd->devices_cnt, i, tmp) {
 907                         if (tmp == dev) {
 908                                 *bus = drhd->devices[i].bus;
 909                                 *devfn = drhd->devices[i].devfn;
 910                                 goto out;
 911                         }
 912
 913                         if (!pdev || !dev_is_pci(tmp))
 914                                 continue;
 915
 916                         ptmp = to_pci_dev(tmp);
 917                         if (ptmp->subordinate &&
 918                             ptmp->subordinate->number <= pdev->bus->number &&
 919                             ptmp->subordinate->busn_res.end >= pdev->bus->number)
 920                                 goto got_pdev;
 921                 }
 922
 923                 if (pdev && drhd->include_all) {
 924                 got_pdev:
 925                         *bus = pdev->bus->number;
 926                         *devfn = pdev->devfn;
 927                         goto out;
 928                 }
 929         }
 930         iommu = NULL;
 931  out:
 932         rcu_read_unlock();
 933
 934         return iommu;
 935 }
 936
 937 static void domain_flush_cache(struct dmar_domain *domain,
 938                                void *addr, int size)
 939 {
 940         if (!domain->iommu_coherency)
 941                 clflush_cache_range(addr, size);
 942 }
 943
 944 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 945 {
 946         struct context_entry *context;
 947         int ret = 0;
 948         unsigned long flags;
 949
 950         spin_lock_irqsave(&iommu->lock, flags);
 951         context = iommu_context_addr(iommu, bus, devfn, 0);
 952         if (context)
 953                 ret = context_present(context);
 954         spin_unlock_irqrestore(&iommu->lock, flags);
 955         return ret;
 956 }
 957
 958 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
 959 {
 960         struct context_entry *context;
 961         unsigned long flags;
 962
 963         spin_lock_irqsave(&iommu->lock, flags);
 964         context = iommu_context_addr(iommu, bus, devfn, 0);
 965         if (context) {
 966                 context_clear_entry(context);
 967                 __iommu_flush_cache(iommu, context, sizeof(*context));
 968         }
 969         spin_unlock_irqrestore(&iommu->lock, flags);
 970 }
 971
 972 static void free_context_table(struct intel_iommu *iommu)
 973 {
 974         int i;
 975         unsigned long flags;
 976         struct context_entry *context;
 977
 978         spin_lock_irqsave(&iommu->lock, flags);
 979         if (!iommu->root_entry) {
 980                 goto out;
 981         }
 982         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 983                 context = iommu_context_addr(iommu, i, 0, 0);
 984                 if (context)
 985                         free_pgtable_page(context);
 986
 987                 if (!ecs_enabled(iommu))
 988                         continue;
 989
 990                 context = iommu_context_addr(iommu, i, 0x80, 0);
 991                 if (context)
 992                         free_pgtable_page(context);
 993
 994         }
 995         free_pgtable_page(iommu->root_entry);
 996         iommu->root_entry = NULL;
 997 out:
 998         spin_unlock_irqrestore(&iommu->lock, flags);
 999 }
1000
1001 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
1002                                       unsigned long pfn, int *target_level)
1003 {
1004         struct dma_pte *parent, *pte = NULL;
1005         int level = agaw_to_level(domain->agaw);
1006         int offset;
1007
1008         BUG_ON(!domain->pgd);
1009
1010         if (!domain_pfn_supported(domain, pfn))
1011                 /* Address beyond IOMMU's addressing capabilities. */
1012                 return NULL;
1013
1014         parent = domain->pgd;
1015
1016         while (1) {
1017                 void *tmp_page;
1018
1019                 offset = pfn_level_offset(pfn, level);
1020                 pte = &parent[offset];
1021                 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
1022                         break;
1023                 if (level == *target_level)
1024                         break;
1025
1026                 if (!dma_pte_present(pte)) {
1027                         uint64_t pteval;
1028
1029                         tmp_page = alloc_pgtable_page(domain->nid);
1030
1031                         if (!tmp_page)
1032                                 return NULL;
1033
1034                         domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
1035                         pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
1036                         if (cmpxchg64(&pte->val, 0ULL, pteval))
1037                                 /* Someone else set it while we were thinking; use theirs. */
1038                                 free_pgtable_page(tmp_page);
1039                         else
1040                                 domain_flush_cache(domain, pte, sizeof(*pte));
1041                 }
1042                 if (level == 1)
1043                         break;
1044
1045                 parent = phys_to_virt(dma_pte_addr(pte));
1046                 level--;
1047         }
1048
1049         if (!*target_level)
1050                 *target_level = level;
1051
1052         return pte;
1053 }
1054
1055
1056 /* return address's pte at specific level */
1057 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1058                                          unsigned long pfn,
1059                                          int level, int *large_page)
1060 {
1061         struct dma_pte *parent, *pte = NULL;
1062         int total = agaw_to_level(domain->agaw);
1063         int offset;
1064
1065         parent = domain->pgd;
1066         while (level <= total) {
1067                 offset = pfn_level_offset(pfn, total);
1068                 pte = &parent[offset];
1069                 if (level == total)
1070                         return pte;
1071
1072                 if (!dma_pte_present(pte)) {
1073                         *large_page = total;
1074                         break;
1075                 }
1076
1077                 if (dma_pte_superpage(pte)) {
1078                         *large_page = total;
1079                         return pte;
1080                 }
1081
1082                 parent = phys_to_virt(dma_pte_addr(pte));
1083                 total--;
1084         }
1085         return NULL;
1086 }
1087
1088 /* clear last level pte, a tlb flush should be followed */
1089 static void dma_pte_clear_range(struct dmar_domain *domain,
1090                                 unsigned long start_pfn,
1091                                 unsigned long last_pfn)
1092 {
1093         unsigned int large_page = 1;
1094         struct dma_pte *first_pte, *pte;
1095
1096         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1097         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1098         BUG_ON(start_pfn > last_pfn);
1099
1100         /* we don't need lock here; nobody else touches the iova range */
1101         do {
1102                 large_page = 1;
1103                 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
1104                 if (!pte) {
1105                         start_pfn = align_to_level(start_pfn + 1, large_page + 1);
1106                         continue;
1107                 }
1108                 do {
1109                         dma_clear_pte(pte);
1110                         start_pfn += lvl_to_nr_pages(large_page);
1111                         pte++;
1112                 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1113
1114                 domain_flush_cache(domain, first_pte,
1115                                    (void *)pte - (void *)first_pte);
1116
1117         } while (start_pfn && start_pfn <= last_pfn);
1118 }
1119
1120 static void dma_pte_free_level(struct dmar_domain *domain, int level,
1121                                struct dma_pte *pte, unsigned long pfn,
1122                                unsigned long start_pfn, unsigned long last_pfn)
1123 {
1124         pfn = max(start_pfn, pfn);
1125         pte = &pte[pfn_level_offset(pfn, level)];
1126
1127         do {
1128                 unsigned long level_pfn;
1129                 struct dma_pte *level_pte;
1130
1131                 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1132                         goto next;
1133
1134                 level_pfn = pfn & level_mask(level - 1);
1135                 level_pte = phys_to_virt(dma_pte_addr(pte));
1136
1137                 if (level > 2)
1138                         dma_pte_free_level(domain, level - 1, level_pte,
1139                                            level_pfn, start_pfn, last_pfn);
1140
1141                 /* If range covers entire pagetable, free it */
1142                 if (!(start_pfn > level_pfn ||
1143                       last_pfn < level_pfn + level_size(level) - 1)) {
1144                         dma_clear_pte(pte);
1145                         domain_flush_cache(domain, pte, sizeof(*pte));
1146                         free_pgtable_page(level_pte);
1147                 }
1148 next:
1149                 pfn += level_size(level);
1150         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1151 }
1152
1153 /* clear last level (leaf) ptes and free page table pages. */
1154 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1155                                    unsigned long start_pfn,
1156                                    unsigned long last_pfn)
1157 {
1158         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1159         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1160         BUG_ON(start_pfn > last_pfn);
1161
1162         dma_pte_clear_range(domain, start_pfn, last_pfn);
1163
1164         /* We don't need lock here; nobody else touches the iova range */
1165         dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1166                            domain->pgd, 0, start_pfn, last_pfn);
1167
1168         /* free pgd */
1169         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1170                 free_pgtable_page(domain->pgd);
1171                 domain->pgd = NULL;
1172         }
1173 }
1174
1175 /* When a page at a given level is being unlinked from its parent, we don't
1176    need to *modify* it at all. All we need to do is make a list of all the
1177    pages which can be freed just as soon as we've flushed the IOTLB and we
1178    know the hardware page-walk will no longer touch them.
1179    The 'pte' argument is the *parent* PTE, pointing to the page that is to
1180    be freed. */
1181 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1182                                             int level, struct dma_pte *pte,
1183                                             struct page *freelist)
1184 {
1185         struct page *pg;
1186
1187         pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1188         pg->freelist = freelist;
1189         freelist = pg;
1190
1191         if (level == 1)
1192                 return freelist;
1193
1194         pte = page_address(pg);
1195         do {
1196                 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1197                         freelist = dma_pte_list_pagetables(domain, level - 1,
1198                                                            pte, freelist);
1199                 pte++;
1200         } while (!first_pte_in_page(pte));
1201
1202         return freelist;
1203 }
1204
1205 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1206                                         struct dma_pte *pte, unsigned long pfn,
1207                                         unsigned long start_pfn,
1208                                         unsigned long last_pfn,
1209                                         struct page *freelist)
1210 {
1211         struct dma_pte *first_pte = NULL, *last_pte = NULL;
1212
1213         pfn = max(start_pfn, pfn);
1214         pte = &pte[pfn_level_offset(pfn, level)];
1215
1216         do {
1217                 unsigned long level_pfn;
1218
1219                 if (!dma_pte_present(pte))
1220                         goto next;
1221
1222                 level_pfn = pfn & level_mask(level);
1223
1224                 /* If range covers entire pagetable, free it */
1225                 if (start_pfn <= level_pfn &&
1226                     last_pfn >= level_pfn + level_size(level) - 1) {
1227                         /* These suborbinate page tables are going away entirely. Don't
1228                            bother to clear them; we're just going to *free* them. */
1229                         if (level > 1 && !dma_pte_superpage(pte))
1230                                 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1231
1232                         dma_clear_pte(pte);
1233                         if (!first_pte)
1234                                 first_pte = pte;
1235                         last_pte = pte;
1236                 } else if (level > 1) {
1237                         /* Recurse down into a level that isn't *entirely* obsolete */
1238                         freelist = dma_pte_clear_level(domain, level - 1,
1239                                                        phys_to_virt(dma_pte_addr(pte)),
1240                                                        level_pfn, start_pfn, last_pfn,
1241                                                        freelist);
1242                 }
1243 next:
1244                 pfn += level_size(level);
1245         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1246
1247         if (first_pte)
1248                 domain_flush_cache(domain, first_pte,
1249                                    (void *)++last_pte - (void *)first_pte);
1250
1251         return freelist;
1252 }
1253
1254 /* We can't just free the pages because the IOMMU may still be walking
1255    the page tables, and may have cached the intermediate levels. The
1256    pages can only be freed after the IOTLB flush has been done. */
1257 static struct page *domain_unmap(struct dmar_domain *domain,
1258                                  unsigned long start_pfn,
1259                                  unsigned long last_pfn)
1260 {
1261         struct page *freelist = NULL;
1262
1263         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1264         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1265         BUG_ON(start_pfn > last_pfn);
1266
1267         /* we don't need lock here; nobody else touches the iova range */
1268         freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1269                                        domain->pgd, 0, start_pfn, last_pfn, NULL);
1270
1271         /* free pgd */
1272         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1273                 struct page *pgd_page = virt_to_page(domain->pgd);
1274                 pgd_page->freelist = freelist;
1275                 freelist = pgd_page;
1276
1277                 domain->pgd = NULL;
1278         }
1279
1280         return freelist;
1281 }
1282
1283 static void dma_free_pagelist(struct page *freelist)
1284 {
1285         struct page *pg;
1286
1287         while ((pg = freelist)) {
1288                 freelist = pg->freelist;
1289                 free_pgtable_page(page_address(pg));
1290         }
1291 }
1292
1293 /* iommu handling */
1294 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1295 {
1296         struct root_entry *root;
1297         unsigned long flags;
1298
1299         root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1300         if (!root) {
1301                 pr_err("Allocating root entry for %s failed\n",
1302                         iommu->name);
1303                 return -ENOMEM;
1304         }
1305
1306         __iommu_flush_cache(iommu, root, ROOT_SIZE);
1307
1308         spin_lock_irqsave(&iommu->lock, flags);
1309         iommu->root_entry = root;
1310         spin_unlock_irqrestore(&iommu->lock, flags);
1311
1312         return 0;
1313 }
1314
1315 static void iommu_set_root_entry(struct intel_iommu *iommu)
1316 {
1317         u64 addr;
1318         u32 sts;
1319         unsigned long flag;
1320
1321         addr = virt_to_phys(iommu->root_entry);
1322         if (ecs_enabled(iommu))
1323                 addr |= DMA_RTADDR_RTT;
1324
1325         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1326         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1327
1328         writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1329
1330         /* Make sure hardware complete it */
1331         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1332                       readl, (sts & DMA_GSTS_RTPS), sts);
1333
1334         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1335 }
1336
1337 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1338 {
1339         u32 val;
1340         unsigned long flag;
1341
1342         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1343                 return;
1344
1345         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1346         writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1347
1348         /* Make sure hardware complete it */
1349         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1350                       readl, (!(val & DMA_GSTS_WBFS)), val);
1351
1352         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1353 }
1354
1355 /* return value determine if we need a write buffer flush */
1356 static void __iommu_flush_context(struct intel_iommu *iommu,
1357                                   u16 did, u16 source_id, u8 function_mask,
1358                                   u64 type)
1359 {
1360         u64 val = 0;
1361         unsigned long flag;
1362
1363         switch (type) {
1364         case DMA_CCMD_GLOBAL_INVL:
1365                 val = DMA_CCMD_GLOBAL_INVL;
1366                 break;
1367         case DMA_CCMD_DOMAIN_INVL:
1368                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1369                 break;
1370         case DMA_CCMD_DEVICE_INVL:
1371                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1372                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1373                 break;
1374         default:
1375                 BUG();
1376         }
1377         val |= DMA_CCMD_ICC;
1378
1379         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1380         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1381
1382         /* Make sure hardware complete it */
1383         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1384                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1385
1386         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1387 }
1388
1389 /* return value determine if we need a write buffer flush */
1390 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1391                                 u64 addr, unsigned int size_order, u64 type)
1392 {
1393         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1394         u64 val = 0, val_iva = 0;
1395         unsigned long flag;
1396
1397         switch (type) {
1398         case DMA_TLB_GLOBAL_FLUSH:
1399                 /* global flush doesn't need set IVA_REG */
1400                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1401                 break;
1402         case DMA_TLB_DSI_FLUSH:
1403                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1404                 break;
1405         case DMA_TLB_PSI_FLUSH:
1406                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1407                 /* IH bit is passed in as part of address */
1408                 val_iva = size_order | addr;
1409                 break;
1410         default:
1411                 BUG();
1412         }
1413         /* Note: set drain read/write */
1414 #if 0
1415         /*
1416          * This is probably to be super secure.. Looks like we can
1417          * ignore it without any impact.
1418          */
1419         if (cap_read_drain(iommu->cap))
1420                 val |= DMA_TLB_READ_DRAIN;
1421 #endif
1422         if (cap_write_drain(iommu->cap))
1423                 val |= DMA_TLB_WRITE_DRAIN;
1424
1425         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1426         /* Note: Only uses first TLB reg currently */
1427         if (val_iva)
1428                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1429         dmar_writeq(iommu->reg + tlb_offset + 8, val);
1430
1431         /* Make sure hardware complete it */
1432         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1433                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1434
1435         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1436
1437         /* check IOTLB invalidation granularity */
1438         if (DMA_TLB_IAIG(val) == 0)
1439                 pr_err("Flush IOTLB failed\n");
1440         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1441                 pr_debug("TLB flush request %Lx, actual %Lx\n",
1442                         (unsigned long long)DMA_TLB_IIRG(type),
1443                         (unsigned long long)DMA_TLB_IAIG(val));
1444 }
1445
1446 static struct device_domain_info *
1447 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1448                          u8 bus, u8 devfn)
1449 {
1450         struct device_domain_info *info;
1451
1452         assert_spin_locked(&device_domain_lock);
1453
1454         if (!iommu->qi)
1455                 return NULL;
1456
1457         list_for_each_entry(info, &domain->devices, link)
1458                 if (info->iommu == iommu && info->bus == bus &&
1459                     info->devfn == devfn) {
1460                         if (info->ats_supported && info->dev)
1461                                 return info;
1462                         break;
1463                 }
1464
1465         return NULL;
1466 }
1467
1468 static void domain_update_iotlb(struct dmar_domain *domain)
1469 {
1470         struct device_domain_info *info;
1471         bool has_iotlb_device = false;
1472
1473         assert_spin_locked(&device_domain_lock);
1474
1475         list_for_each_entry(info, &domain->devices, link) {
1476                 struct pci_dev *pdev;
1477
1478                 if (!info->dev || !dev_is_pci(info->dev))
1479                         continue;
1480
1481                 pdev = to_pci_dev(info->dev);
1482                 if (pdev->ats_enabled) {
1483                         has_iotlb_device = true;
1484                         break;
1485                 }
1486         }
1487
1488         domain->has_iotlb_device = has_iotlb_device;
1489 }
1490
1491 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1492 {
1493         struct pci_dev *pdev;
1494
1495         assert_spin_locked(&device_domain_lock);
1496
1497         if (!info || !dev_is_pci(info->dev))
1498                 return;
1499
1500         pdev = to_pci_dev(info->dev);
1501
1502 #ifdef CONFIG_INTEL_IOMMU_SVM
1503         /* The PCIe spec, in its wisdom, declares that the behaviour of
1504            the device if you enable PASID support after ATS support is
1505            undefined. So always enable PASID support on devices which
1506            have it, even if we can't yet know if we're ever going to
1507            use it. */
1508         if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1509                 info->pasid_enabled = 1;
1510
1511         if (info->pri_supported && !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1512                 info->pri_enabled = 1;
1513 #endif
1514         if (info->ats_supported && !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1515                 info->ats_enabled = 1;
1516                 domain_update_iotlb(info->domain);
1517                 info->ats_qdep = pci_ats_queue_depth(pdev);
1518         }
1519 }
1520
1521 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1522 {
1523         struct pci_dev *pdev;
1524
1525         assert_spin_locked(&device_domain_lock);
1526
1527         if (!dev_is_pci(info->dev))
1528                 return;
1529
1530         pdev = to_pci_dev(info->dev);
1531
1532         if (info->ats_enabled) {
1533                 pci_disable_ats(pdev);
1534                 info->ats_enabled = 0;
1535                 domain_update_iotlb(info->domain);
1536         }
1537 #ifdef CONFIG_INTEL_IOMMU_SVM
1538         if (info->pri_enabled) {
1539                 pci_disable_pri(pdev);
1540                 info->pri_enabled = 0;
1541         }
1542         if (info->pasid_enabled) {
1543                 pci_disable_pasid(pdev);
1544                 info->pasid_enabled = 0;
1545         }
1546 #endif
1547 }
1548
1549 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1550                                   u64 addr, unsigned mask)
1551 {
1552         u16 sid, qdep;
1553         unsigned long flags;
1554         struct device_domain_info *info;
1555
1556         if (!domain->has_iotlb_device)
1557                 return;
1558
1559         spin_lock_irqsave(&device_domain_lock, flags);
1560         list_for_each_entry(info, &domain->devices, link) {
1561                 if (!info->ats_enabled)
1562                         continue;
1563
1564                 sid = info->bus << 8 | info->devfn;
1565                 qdep = info->ats_qdep;
1566                 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1567         }
1568         spin_unlock_irqrestore(&device_domain_lock, flags);
1569 }
1570
1571 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1572                                   struct dmar_domain *domain,
1573                                   unsigned long pfn, unsigned int pages,
1574                                   int ih, int map)
1575 {
1576         unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1577         uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1578         u16 did = domain->iommu_did[iommu->seq_id];
1579
1580         BUG_ON(pages == 0);
1581
1582         if (ih)
1583                 ih = 1 << 6;
1584         /*
1585          * Fallback to domain selective flush if no PSI support or the size is
1586          * too big.
1587          * PSI requires page size to be 2 ^ x, and the base address is naturally
1588          * aligned to the size
1589          */
1590         if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1591                 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1592                                                 DMA_TLB_DSI_FLUSH);
1593         else
1594                 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1595                                                 DMA_TLB_PSI_FLUSH);
1596
1597         /*
1598          * In caching mode, changes of pages from non-present to present require
1599          * flush. However, device IOTLB doesn't need to be flushed in this case.
1600          */
1601         if (!cap_caching_mode(iommu->cap) || !map)
1602                 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1603                                       addr, mask);
1604 }
1605
1606 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1607 {
1608         u32 pmen;
1609         unsigned long flags;
1610
1611         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1612         pmen = readl(iommu->reg + DMAR_PMEN_REG);
1613         pmen &= ~DMA_PMEN_EPM;
1614         writel(pmen, iommu->reg + DMAR_PMEN_REG);
1615
1616         /* wait for the protected region status bit to clear */
1617         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1618                 readl, !(pmen & DMA_PMEN_PRS), pmen);
1619
1620         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1621 }
1622
1623 static void iommu_enable_translation(struct intel_iommu *iommu)
1624 {
1625         u32 sts;
1626         unsigned long flags;
1627
1628         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1629         iommu->gcmd |= DMA_GCMD_TE;
1630         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1631
1632         /* Make sure hardware complete it */
1633         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1634                       readl, (sts & DMA_GSTS_TES), sts);
1635
1636         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1637 }
1638
1639 static void iommu_disable_translation(struct intel_iommu *iommu)
1640 {
1641         u32 sts;
1642         unsigned long flag;
1643
1644         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1645         iommu->gcmd &= ~DMA_GCMD_TE;
1646         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1647
1648         /* Make sure hardware complete it */
1649         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1650                       readl, (!(sts & DMA_GSTS_TES)), sts);
1651
1652         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1653 }
1654
1655
1656 static int iommu_init_domains(struct intel_iommu *iommu)
1657 {
1658         u32 ndomains, nlongs;
1659         size_t size;
1660
1661         ndomains = cap_ndoms(iommu->cap);
1662         pr_debug("%s: Number of Domains supported <%d>\n",
1663                  iommu->name, ndomains);
1664         nlongs = BITS_TO_LONGS(ndomains);
1665
1666         spin_lock_init(&iommu->lock);
1667
1668         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1669         if (!iommu->domain_ids) {
1670                 pr_err("%s: Allocating domain id array failed\n",
1671                        iommu->name);
1672                 return -ENOMEM;
1673         }
1674
1675         size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
1676         iommu->domains = kzalloc(size, GFP_KERNEL);
1677
1678         if (iommu->domains) {
1679                 size = 256 * sizeof(struct dmar_domain *);
1680                 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1681         }
1682
1683         if (!iommu->domains || !iommu->domains[0]) {
1684                 pr_err("%s: Allocating domain array failed\n",
1685                        iommu->name);
1686                 kfree(iommu->domain_ids);
1687                 kfree(iommu->domains);
1688                 iommu->domain_ids = NULL;
1689                 iommu->domains    = NULL;
1690                 return -ENOMEM;
1691         }
1692
1693
1694
1695         /*
1696          * If Caching mode is set, then invalid translations are tagged
1697          * with domain-id 0, hence we need to pre-allocate it. We also
1698          * use domain-id 0 as a marker for non-allocated domain-id, so
1699          * make sure it is not used for a real domain.
1700          */
1701         set_bit(0, iommu->domain_ids);
1702
1703         return 0;
1704 }
1705
1706 static void disable_dmar_iommu(struct intel_iommu *iommu)
1707 {
1708         struct device_domain_info *info, *tmp;
1709         unsigned long flags;
1710
1711         if (!iommu->domains || !iommu->domain_ids)
1712                 return;
1713
1714         spin_lock_irqsave(&device_domain_lock, flags);
1715         list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1716                 struct dmar_domain *domain;
1717
1718                 if (info->iommu != iommu)
1719                         continue;
1720
1721                 if (!info->dev || !info->domain)
1722                         continue;
1723
1724                 domain = info->domain;
1725
1726                 dmar_remove_one_dev_info(domain, info->dev);
1727
1728                 if (!domain_type_is_vm_or_si(domain))
1729                         domain_exit(domain);
1730         }
1731         spin_unlock_irqrestore(&device_domain_lock, flags);
1732
1733         if (iommu->gcmd & DMA_GCMD_TE)
1734                 iommu_disable_translation(iommu);
1735 }
1736
1737 static void free_dmar_iommu(struct intel_iommu *iommu)
1738 {
1739         if ((iommu->domains) && (iommu->domain_ids)) {
1740                 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
1741                 int i;
1742
1743                 for (i = 0; i < elems; i++)
1744                         kfree(iommu->domains[i]);
1745                 kfree(iommu->domains);
1746                 kfree(iommu->domain_ids);
1747                 iommu->domains = NULL;
1748                 iommu->domain_ids = NULL;
1749         }
1750
1751         g_iommus[iommu->seq_id] = NULL;
1752
1753         /* free context mapping */
1754         free_context_table(iommu);
1755
1756 #ifdef CONFIG_INTEL_IOMMU_SVM
1757         if (pasid_enabled(iommu)) {
1758                 if (ecap_prs(iommu->ecap))
1759                         intel_svm_finish_prq(iommu);
1760                 intel_svm_free_pasid_tables(iommu);
1761         }
1762 #endif
1763 }
1764
1765 static struct dmar_domain *alloc_domain(int flags)
1766 {
1767         struct dmar_domain *domain;
1768
1769         domain = alloc_domain_mem();
1770         if (!domain)
1771                 return NULL;
1772
1773         memset(domain, 0, sizeof(*domain));
1774         domain->nid = -1;
1775         domain->flags = flags;
1776         domain->has_iotlb_device = false;
1777         INIT_LIST_HEAD(&domain->devices);
1778
1779         return domain;
1780 }
1781
1782 /* Must be called with iommu->lock */
1783 static int domain_attach_iommu(struct dmar_domain *domain,
1784                                struct intel_iommu *iommu)
1785 {
1786         unsigned long ndomains;
1787         int num;
1788
1789         assert_spin_locked(&device_domain_lock);
1790         assert_spin_locked(&iommu->lock);
1791
1792         domain->iommu_refcnt[iommu->seq_id] += 1;
1793         domain->iommu_count += 1;
1794         if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1795                 ndomains = cap_ndoms(iommu->cap);
1796                 num      = find_first_zero_bit(iommu->domain_ids, ndomains);
1797
1798                 if (num >= ndomains) {
1799                         pr_err("%s: No free domain ids\n", iommu->name);
1800                         domain->iommu_refcnt[iommu->seq_id] -= 1;
1801                         domain->iommu_count -= 1;
1802                         return -ENOSPC;
1803                 }
1804
1805                 set_bit(num, iommu->domain_ids);
1806                 set_iommu_domain(iommu, num, domain);
1807
1808                 domain->iommu_did[iommu->seq_id] = num;
1809                 domain->nid                      = iommu->node;
1810
1811                 domain_update_iommu_cap(domain);
1812         }
1813
1814         return 0;
1815 }
1816
1817 static int domain_detach_iommu(struct dmar_domain *domain,
1818                                struct intel_iommu *iommu)
1819 {
1820         int num, count = INT_MAX;
1821
1822         assert_spin_locked(&device_domain_lock);
1823         assert_spin_locked(&iommu->lock);
1824
1825         domain->iommu_refcnt[iommu->seq_id] -= 1;
1826         count = --domain->iommu_count;
1827         if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1828                 num = domain->iommu_did[iommu->seq_id];
1829                 clear_bit(num, iommu->domain_ids);
1830                 set_iommu_domain(iommu, num, NULL);
1831
1832                 domain_update_iommu_cap(domain);
1833                 domain->iommu_did[iommu->seq_id] = 0;
1834         }
1835
1836         return count;
1837 }
1838
1839 static struct iova_domain reserved_iova_list;
1840 static struct lock_class_key reserved_rbtree_key;
1841
1842 static int dmar_init_reserved_ranges(void)
1843 {
1844         struct pci_dev *pdev = NULL;
1845         struct iova *iova;
1846         int i;
1847
1848         init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1849                         DMA_32BIT_PFN);
1850
1851         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1852                 &reserved_rbtree_key);
1853
1854         /* IOAPIC ranges shouldn't be accessed by DMA */
1855         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1856                 IOVA_PFN(IOAPIC_RANGE_END));
1857         if (!iova) {
1858                 pr_err("Reserve IOAPIC range failed\n");
1859                 return -ENODEV;
1860         }
1861
1862         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1863         for_each_pci_dev(pdev) {
1864                 struct resource *r;
1865
1866                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1867                         r = &pdev->resource[i];
1868                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1869                                 continue;
1870                         iova = reserve_iova(&reserved_iova_list,
1871                                             IOVA_PFN(r->start),
1872                                             IOVA_PFN(r->end));
1873                         if (!iova) {
1874                                 pr_err("Reserve iova failed\n");
1875                                 return -ENODEV;
1876                         }
1877                 }
1878         }
1879         return 0;
1880 }
1881
1882 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1883 {
1884         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1885 }
1886
1887 static inline int guestwidth_to_adjustwidth(int gaw)
1888 {
1889         int agaw;
1890         int r = (gaw - 12) % 9;
1891
1892         if (r == 0)
1893                 agaw = gaw;
1894         else
1895                 agaw = gaw + 9 - r;
1896         if (agaw > 64)
1897                 agaw = 64;
1898         return agaw;
1899 }
1900
1901 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1902                        int guest_width)
1903 {
1904         int adjust_width, agaw;
1905         unsigned long sagaw;
1906
1907         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1908                         DMA_32BIT_PFN);
1909         domain_reserve_special_ranges(domain);
1910
1911         /* calculate AGAW */
1912         if (guest_width > cap_mgaw(iommu->cap))
1913                 guest_width = cap_mgaw(iommu->cap);
1914         domain->gaw = guest_width;
1915         adjust_width = guestwidth_to_adjustwidth(guest_width);
1916         agaw = width_to_agaw(adjust_width);
1917         sagaw = cap_sagaw(iommu->cap);
1918         if (!test_bit(agaw, &sagaw)) {
1919                 /* hardware doesn't support it, choose a bigger one */
1920                 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1921                 agaw = find_next_bit(&sagaw, 5, agaw);
1922                 if (agaw >= 5)
1923                         return -ENODEV;
1924         }
1925         domain->agaw = agaw;
1926
1927         if (ecap_coherent(iommu->ecap))
1928                 domain->iommu_coherency = 1;
1929         else
1930                 domain->iommu_coherency = 0;
1931
1932         if (ecap_sc_support(iommu->ecap))
1933                 domain->iommu_snooping = 1;
1934         else
1935                 domain->iommu_snooping = 0;
1936
1937         if (intel_iommu_superpage)
1938                 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1939         else
1940                 domain->iommu_superpage = 0;
1941
1942         domain->nid = iommu->node;
1943
1944         /* always allocate the top pgd */
1945         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1946         if (!domain->pgd)
1947                 return -ENOMEM;
1948         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1949         return 0;
1950 }
1951
1952 static void domain_exit(struct dmar_domain *domain)
1953 {
1954         struct page *freelist = NULL;
1955
1956         /* Domain 0 is reserved, so dont process it */
1957         if (!domain)
1958                 return;
1959
1960         /* Flush any lazy unmaps that may reference this domain */
1961         if (!intel_iommu_strict) {
1962                 int cpu;
1963
1964                 for_each_possible_cpu(cpu)
1965                         flush_unmaps_timeout(cpu);
1966         }
1967
1968         /* Remove associated devices and clear attached or cached domains */
1969         rcu_read_lock();
1970         domain_remove_dev_info(domain);
1971         rcu_read_unlock();
1972
1973         /* destroy iovas */
1974         put_iova_domain(&domain->iovad);
1975
1976         freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1977
1978         dma_free_pagelist(freelist);
1979
1980         free_domain_mem(domain);
1981 }
1982
1983 static int domain_context_mapping_one(struct dmar_domain *domain,
1984                                       struct intel_iommu *iommu,
1985                                       u8 bus, u8 devfn)
1986 {
1987         u16 did = domain->iommu_did[iommu->seq_id];
1988         int translation = CONTEXT_TT_MULTI_LEVEL;
1989         struct device_domain_info *info = NULL;
1990         struct context_entry *context;
1991         unsigned long flags;
1992         struct dma_pte *pgd;
1993         int ret, agaw;
1994
1995         WARN_ON(did == 0);
1996
1997         if (hw_pass_through && domain_type_is_si(domain))
1998                 translation = CONTEXT_TT_PASS_THROUGH;
1999
2000         pr_debug("Set context mapping for %02x:%02x.%d\n",
2001                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
2002
2003         BUG_ON(!domain->pgd);
2004
2005         spin_lock_irqsave(&device_domain_lock, flags);
2006         spin_lock(&iommu->lock);
2007
2008         ret = -ENOMEM;
2009         context = iommu_context_addr(iommu, bus, devfn, 1);
2010         if (!context)
2011                 goto out_unlock;
2012
2013         ret = 0;
2014         if (context_present(context))
2015                 goto out_unlock;
2016
2017         pgd = domain->pgd;
2018
2019         context_clear_entry(context);
2020         context_set_domain_id(context, did);
2021
2022         /*
2023          * Skip top levels of page tables for iommu which has less agaw
2024          * than default.  Unnecessary for PT mode.
2025          */
2026         if (translation != CONTEXT_TT_PASS_THROUGH) {
2027                 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
2028                         ret = -ENOMEM;
2029                         pgd = phys_to_virt(dma_pte_addr(pgd));
2030                         if (!dma_pte_present(pgd))
2031                                 goto out_unlock;
2032                 }
2033
2034                 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2035                 if (info && info->ats_supported)
2036                         translation = CONTEXT_TT_DEV_IOTLB;
2037                 else
2038                         translation = CONTEXT_TT_MULTI_LEVEL;
2039
2040                 context_set_address_root(context, virt_to_phys(pgd));
2041                 context_set_address_width(context, iommu->agaw);
2042         } else {
2043                 /*
2044                  * In pass through mode, AW must be programmed to
2045                  * indicate the largest AGAW value supported by
2046                  * hardware. And ASR is ignored by hardware.
2047                  */
2048                 context_set_address_width(context, iommu->msagaw);
2049         }
2050
2051         context_set_translation_type(context, translation);
2052         context_set_fault_enable(context);
2053         context_set_present(context);
2054         domain_flush_cache(domain, context, sizeof(*context));
2055
2056         /*
2057          * It's a non-present to present mapping. If hardware doesn't cache
2058          * non-present entry we only need to flush the write-buffer. If the
2059          * _does_ cache non-present entries, then it does so in the special
2060          * domain #0, which we have to flush:
2061          */
2062         if (cap_caching_mode(iommu->cap)) {
2063                 iommu->flush.flush_context(iommu, 0,
2064                                            (((u16)bus) << 8) | devfn,
2065                                            DMA_CCMD_MASK_NOBIT,
2066                                            DMA_CCMD_DEVICE_INVL);
2067                 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2068         } else {
2069                 iommu_flush_write_buffer(iommu);
2070         }
2071         iommu_enable_dev_iotlb(info);
2072
2073         ret = 0;
2074
2075 out_unlock:
2076         spin_unlock(&iommu->lock);
2077         spin_unlock_irqrestore(&device_domain_lock, flags);
2078
2079         return ret;
2080 }
2081
2082 struct domain_context_mapping_data {
2083         struct dmar_domain *domain;
2084         struct intel_iommu *iommu;
2085 };
2086
2087 static int domain_context_mapping_cb(struct pci_dev *pdev,
2088                                      u16 alias, void *opaque)
2089 {
2090         struct domain_context_mapping_data *data = opaque;
2091
2092         return domain_context_mapping_one(data->domain, data->iommu,
2093                                           PCI_BUS_NUM(alias), alias & 0xff);
2094 }
2095
2096 static int
2097 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2098 {
2099         struct intel_iommu *iommu;
2100         u8 bus, devfn;
2101         struct domain_context_mapping_data data;
2102
2103         iommu = device_to_iommu(dev, &bus, &devfn);
2104         if (!iommu)
2105                 return -ENODEV;
2106
2107         if (!dev_is_pci(dev))
2108                 return domain_context_mapping_one(domain, iommu, bus, devfn);
2109
2110         data.domain = domain;
2111         data.iommu = iommu;
2112
2113         return pci_for_each_dma_alias(to_pci_dev(dev),
2114                                       &domain_context_mapping_cb, &data);
2115 }
2116
2117 static int domain_context_mapped_cb(struct pci_dev *pdev,
2118                                     u16 alias, void *opaque)
2119 {
2120         struct intel_iommu *iommu = opaque;
2121
2122         return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2123 }
2124
2125 static int domain_context_mapped(struct device *dev)
2126 {
2127         struct intel_iommu *iommu;
2128         u8 bus, devfn;
2129
2130         iommu = device_to_iommu(dev, &bus, &devfn);
2131         if (!iommu)
2132                 return -ENODEV;
2133
2134         if (!dev_is_pci(dev))
2135                 return device_context_mapped(iommu, bus, devfn);
2136
2137         return !pci_for_each_dma_alias(to_pci_dev(dev),
2138                                        domain_context_mapped_cb, iommu);
2139 }
2140
2141 /* Returns a number of VTD pages, but aligned to MM page size */
2142 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2143                                             size_t size)
2144 {
2145         host_addr &= ~PAGE_MASK;
2146         return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2147 }
2148
2149 /* Return largest possible superpage level for a given mapping */
2150 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2151                                           unsigned long iov_pfn,
2152                                           unsigned long phy_pfn,
2153                                           unsigned long pages)
2154 {
2155         int support, level = 1;
2156         unsigned long pfnmerge;
2157
2158         support = domain->iommu_superpage;
2159
2160         /* To use a large page, the virtual *and* physical addresses
2161            must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2162            of them will mean we have to use smaller pages. So just
2163            merge them and check both at once. */
2164         pfnmerge = iov_pfn | phy_pfn;
2165
2166         while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2167                 pages >>= VTD_STRIDE_SHIFT;
2168                 if (!pages)
2169                         break;
2170                 pfnmerge >>= VTD_STRIDE_SHIFT;
2171                 level++;
2172                 support--;
2173         }
2174         return level;
2175 }
2176
2177 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2178                             struct scatterlist *sg, unsigned long phys_pfn,
2179                             unsigned long nr_pages, int prot)
2180 {
2181         struct dma_pte *first_pte = NULL, *pte = NULL;
2182         phys_addr_t uninitialized_var(pteval);
2183         unsigned long sg_res = 0;
2184         unsigned int largepage_lvl = 0;
2185         unsigned long lvl_pages = 0;
2186
2187         BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2188
2189         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2190                 return -EINVAL;
2191
2192         prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2193
2194         if (!sg) {
2195                 sg_res = nr_pages;
2196                 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2197         }
2198
2199         while (nr_pages > 0) {
2200                 uint64_t tmp;
2201
2202                 if (!sg_res) {
2203                         sg_res = aligned_nrpages(sg->offset, sg->length);
2204                         sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2205                         sg->dma_length = sg->length;
2206                         pteval = page_to_phys(sg_page(sg)) | prot;
2207                         phys_pfn = pteval >> VTD_PAGE_SHIFT;
2208                 }
2209
2210                 if (!pte) {
2211                         largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2212
2213                         first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2214                         if (!pte)
2215                                 return -ENOMEM;
2216                         /* It is large page*/
2217                         if (largepage_lvl > 1) {
2218                                 unsigned long nr_superpages, end_pfn;
2219
2220                                 pteval |= DMA_PTE_LARGE_PAGE;
2221                                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2222
2223                                 nr_superpages = sg_res / lvl_pages;
2224                                 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2225
2226                                 /*
2227                                  * Ensure that old small page tables are
2228                                  * removed to make room for superpage(s).
2229                                  */
2230                                 dma_pte_free_pagetable(domain, iov_pfn, end_pfn);
2231                         } else {
2232                                 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2233                         }
2234
2235                 }
2236                 /* We don't need lock here, nobody else
2237                  * touches the iova range
2238                  */
2239                 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2240                 if (tmp) {
2241                         static int dumps = 5;
2242                         pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2243                                 iov_pfn, tmp, (unsigned long long)pteval);
2244                         if (dumps) {
2245                                 dumps--;
2246                                 debug_dma_dump_mappings(NULL);
2247                         }
2248                         WARN_ON(1);
2249                 }
2250
2251                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2252
2253                 BUG_ON(nr_pages < lvl_pages);
2254                 BUG_ON(sg_res < lvl_pages);
2255
2256                 nr_pages -= lvl_pages;
2257                 iov_pfn += lvl_pages;
2258                 phys_pfn += lvl_pages;
2259                 pteval += lvl_pages * VTD_PAGE_SIZE;
2260                 sg_res -= lvl_pages;
2261
2262                 /* If the next PTE would be the first in a new page, then we
2263                    need to flush the cache on the entries we've just written.
2264                    And then we'll need to recalculate 'pte', so clear it and
2265                    let it get set again in the if (!pte) block above.
2266
2267                    If we're done (!nr_pages) we need to flush the cache too.
2268
2269                    Also if we've been setting superpages, we may need to
2270                    recalculate 'pte' and switch back to smaller pages for the
2271                    end of the mapping, if the trailing size is not enough to
2272                    use another superpage (i.e. sg_res < lvl_pages). */
2273                 pte++;
2274                 if (!nr_pages || first_pte_in_page(pte) ||
2275                     (largepage_lvl > 1 && sg_res < lvl_pages)) {
2276                         domain_flush_cache(domain, first_pte,
2277                                            (void *)pte - (void *)first_pte);
2278                         pte = NULL;
2279                 }
2280
2281                 if (!sg_res && nr_pages)
2282                         sg = sg_next(sg);
2283         }
2284         return 0;
2285 }
2286
2287 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2288                                     struct scatterlist *sg, unsigned long nr_pages,
2289                                     int prot)
2290 {
2291         return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2292 }
2293
2294 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2295                                      unsigned long phys_pfn, unsigned long nr_pages,
2296                                      int prot)
2297 {
2298         return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2299 }
2300
2301 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2302 {
2303         if (!iommu)
2304                 return;
2305
2306         clear_context_table(iommu, bus, devfn);
2307         iommu->flush.flush_context(iommu, 0, 0, 0,
2308                                            DMA_CCMD_GLOBAL_INVL);
2309         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2310 }
2311
2312 static inline void unlink_domain_info(struct device_domain_info *info)
2313 {
2314         assert_spin_locked(&device_domain_lock);
2315         list_del(&info->link);
2316         list_del(&info->global);
2317         if (info->dev)
2318                 info->dev->archdata.iommu = NULL;
2319 }
2320
2321 static void domain_remove_dev_info(struct dmar_domain *domain)
2322 {
2323         struct device_domain_info *info, *tmp;
2324         unsigned long flags;
2325
2326         spin_lock_irqsave(&device_domain_lock, flags);
2327         list_for_each_entry_safe(info, tmp, &domain->devices, link)
2328                 __dmar_remove_one_dev_info(info);
2329         spin_unlock_irqrestore(&device_domain_lock, flags);
2330 }
2331
2332 /*
2333  * find_domain
2334  * Note: we use struct device->archdata.iommu stores the info
2335  */
2336 static struct dmar_domain *find_domain(struct device *dev)
2337 {
2338         struct device_domain_info *info;
2339
2340         /* No lock here, assumes no domain exit in normal case */
2341         info = dev->archdata.iommu;
2342         if (info)
2343                 return info->domain;
2344         return NULL;
2345 }
2346
2347 static inline struct device_domain_info *
2348 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2349 {
2350         struct device_domain_info *info;
2351
2352         list_for_each_entry(info, &device_domain_list, global)
2353                 if (info->iommu->segment == segment && info->bus == bus &&
2354                     info->devfn == devfn)
2355                         return info;
2356
2357         return NULL;
2358 }
2359
2360 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2361                                                     int bus, int devfn,
2362                                                     struct device *dev,
2363                                                     struct dmar_domain *domain)
2364 {
2365         struct dmar_domain *found = NULL;
2366         struct device_domain_info *info;
2367         unsigned long flags;
2368         int ret;
2369
2370         info = alloc_devinfo_mem();
2371         if (!info)
2372                 return NULL;
2373
2374         info->bus = bus;
2375         info->devfn = devfn;
2376         info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2377         info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2378         info->ats_qdep = 0;
2379         info->dev = dev;
2380         info->domain = domain;
2381         info->iommu = iommu;
2382
2383         if (dev && dev_is_pci(dev)) {
2384                 struct pci_dev *pdev = to_pci_dev(info->dev);
2385
2386                 if (ecap_dev_iotlb_support(iommu->ecap) &&
2387                     pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2388                     dmar_find_matched_atsr_unit(pdev))
2389                         info->ats_supported = 1;
2390
2391                 if (ecs_enabled(iommu)) {
2392                         if (pasid_enabled(iommu)) {
2393                                 int features = pci_pasid_features(pdev);
2394                                 if (features >= 0)
2395                                         info->pasid_supported = features | 1;
2396                         }
2397
2398                         if (info->ats_supported && ecap_prs(iommu->ecap) &&
2399                             pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2400                                 info->pri_supported = 1;
2401                 }
2402         }
2403
2404         spin_lock_irqsave(&device_domain_lock, flags);
2405         if (dev)
2406                 found = find_domain(dev);
2407
2408         if (!found) {
2409                 struct device_domain_info *info2;
2410                 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2411                 if (info2) {
2412                         found      = info2->domain;
2413                         info2->dev = dev;
2414                 }
2415         }
2416
2417         if (found) {
2418                 spin_unlock_irqrestore(&device_domain_lock, flags);
2419                 free_devinfo_mem(info);
2420                 /* Caller must free the original domain */
2421                 return found;
2422         }
2423
2424         spin_lock(&iommu->lock);
2425         ret = domain_attach_iommu(domain, iommu);
2426         spin_unlock(&iommu->lock);
2427
2428         if (ret) {
2429                 spin_unlock_irqrestore(&device_domain_lock, flags);
2430                 free_devinfo_mem(info);
2431                 return NULL;
2432         }
2433
2434         list_add(&info->link, &domain->devices);
2435         list_add(&info->global, &device_domain_list);
2436         if (dev)
2437                 dev->archdata.iommu = info;
2438         spin_unlock_irqrestore(&device_domain_lock, flags);
2439
2440         if (dev && domain_context_mapping(domain, dev)) {
2441                 pr_err("Domain context map for %s failed\n", dev_name(dev));
2442                 dmar_remove_one_dev_info(domain, dev);
2443                 return NULL;
2444         }
2445
2446         return domain;
2447 }
2448
2449 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2450 {
2451         *(u16 *)opaque = alias;
2452         return 0;
2453 }
2454
2455 static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
2456 {
2457         struct device_domain_info *info = NULL;
2458         struct dmar_domain *domain = NULL;
2459         struct intel_iommu *iommu;
2460         u16 req_id, dma_alias;
2461         unsigned long flags;
2462         u8 bus, devfn;
2463
2464         iommu = device_to_iommu(dev, &bus, &devfn);
2465         if (!iommu)
2466                 return NULL;
2467
2468         req_id = ((u16)bus << 8) | devfn;
2469
2470         if (dev_is_pci(dev)) {
2471                 struct pci_dev *pdev = to_pci_dev(dev);
2472
2473                 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2474
2475                 spin_lock_irqsave(&device_domain_lock, flags);
2476                 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2477                                                       PCI_BUS_NUM(dma_alias),
2478                                                       dma_alias & 0xff);
2479                 if (info) {
2480                         iommu = info->iommu;
2481                         domain = info->domain;
2482                 }
2483                 spin_unlock_irqrestore(&device_domain_lock, flags);
2484
2485                 /* DMA alias already has a domain, use it */
2486                 if (info)
2487                         goto out;
2488         }
2489
2490         /* Allocate and initialize new domain for the device */
2491         domain = alloc_domain(0);
2492         if (!domain)
2493                 return NULL;
2494         if (domain_init(domain, iommu, gaw)) {
2495                 domain_exit(domain);
2496                 return NULL;
2497         }
2498
2499 out:
2500
2501         return domain;
2502 }
2503
2504 static struct dmar_domain *set_domain_for_dev(struct device *dev,
2505                                               struct dmar_domain *domain)
2506 {
2507         struct intel_iommu *iommu;
2508         struct dmar_domain *tmp;
2509         u16 req_id, dma_alias;
2510         u8 bus, devfn;
2511
2512         iommu = device_to_iommu(dev, &bus, &devfn);
2513         if (!iommu)
2514                 return NULL;
2515
2516         req_id = ((u16)bus << 8) | devfn;
2517
2518         if (dev_is_pci(dev)) {
2519                 struct pci_dev *pdev = to_pci_dev(dev);
2520
2521                 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2522
2523                 /* register PCI DMA alias device */
2524                 if (req_id != dma_alias) {
2525                         tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2526                                         dma_alias & 0xff, NULL, domain);
2527
2528                         if (!tmp || tmp != domain)
2529                                 return tmp;
2530                 }
2531         }
2532
2533         tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2534         if (!tmp || tmp != domain)
2535                 return tmp;
2536
2537         return domain;
2538 }
2539
2540 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2541 {
2542         struct dmar_domain *domain, *tmp;
2543
2544         domain = find_domain(dev);
2545         if (domain)
2546                 goto out;
2547
2548         domain = find_or_alloc_domain(dev, gaw);
2549         if (!domain)
2550                 goto out;
2551
2552         tmp = set_domain_for_dev(dev, domain);
2553         if (!tmp || domain != tmp) {
2554                 domain_exit(domain);
2555                 domain = tmp;
2556         }
2557
2558 out:
2559
2560         return domain;
2561 }
2562
2563 static int iommu_domain_identity_map(struct dmar_domain *domain,
2564                                      unsigned long long start,
2565                                      unsigned long long end)
2566 {
2567         unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2568         unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2569
2570         if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2571                           dma_to_mm_pfn(last_vpfn))) {
2572                 pr_err("Reserving iova failed\n");
2573                 return -ENOMEM;
2574         }
2575
2576         pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2577         /*
2578          * RMRR range might have overlap with physical memory range,
2579          * clear it first
2580          */
2581         dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2582
2583         return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2584                                   last_vpfn - first_vpfn + 1,
2585                                   DMA_PTE_READ|DMA_PTE_WRITE);
2586 }
2587
2588 static int domain_prepare_identity_map(struct device *dev,
2589                                        struct dmar_domain *domain,
2590                                        unsigned long long start,
2591                                        unsigned long long end)
2592 {
2593         /* For _hardware_ passthrough, don't bother. But for software
2594            passthrough, we do it anyway -- it may indicate a memory
2595            range which is reserved in E820, so which didn't get set
2596            up to start with in si_domain */
2597         if (domain == si_domain && hw_pass_through) {
2598                 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2599                         dev_name(dev), start, end);
2600                 return 0;
2601         }
2602
2603         pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2604                 dev_name(dev), start, end);
2605
2606         if (end < start) {
2607                 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2608                         "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2609                         dmi_get_system_info(DMI_BIOS_VENDOR),
2610                         dmi_get_system_info(DMI_BIOS_VERSION),
2611                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2612                 return -EIO;
2613         }
2614
2615         if (end >> agaw_to_width(domain->agaw)) {
2616                 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2617                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2618                      agaw_to_width(domain->agaw),
2619                      dmi_get_system_info(DMI_BIOS_VENDOR),
2620                      dmi_get_system_info(DMI_BIOS_VERSION),
2621                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2622                 return -EIO;
2623         }
2624
2625         return iommu_domain_identity_map(domain, start, end);
2626 }
2627
2628 static int iommu_prepare_identity_map(struct device *dev,
2629                                       unsigned long long start,
2630                                       unsigned long long end)
2631 {
2632         struct dmar_domain *domain;
2633         int ret;
2634
2635         domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2636         if (!domain)
2637                 return -ENOMEM;
2638
2639         ret = domain_prepare_identity_map(dev, domain, start, end);
2640         if (ret)
2641                 domain_exit(domain);
2642
2643         return ret;
2644 }
2645
2646 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2647                                          struct device *dev)
2648 {
2649         if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2650                 return 0;
2651         return iommu_prepare_identity_map(dev, rmrr->base_address,
2652                                           rmrr->end_address);
2653 }
2654
2655 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2656 static inline void iommu_prepare_isa(void)
2657 {
2658         struct pci_dev *pdev;
2659         int ret;
2660
2661         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2662         if (!pdev)
2663                 return;
2664
2665         pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2666         ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2667
2668         if (ret)
2669                 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2670
2671         pci_dev_put(pdev);
2672 }
2673 #else
2674 static inline void iommu_prepare_isa(void)
2675 {
2676         return;
2677 }
2678 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2679
2680 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2681
2682 static int __init si_domain_init(int hw)
2683 {
2684         int nid, ret = 0;
2685
2686         si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2687         if (!si_domain)
2688                 return -EFAULT;
2689
2690         if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2691                 domain_exit(si_domain);
2692                 return -EFAULT;
2693         }
2694
2695         pr_debug("Identity mapping domain allocated\n");
2696
2697         if (hw)
2698                 return 0;
2699
2700         for_each_online_node(nid) {
2701                 unsigned long start_pfn, end_pfn;
2702                 int i;
2703
2704                 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2705                         ret = iommu_domain_identity_map(si_domain,
2706                                         PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2707                         if (ret)
2708                                 return ret;
2709                 }
2710         }
2711
2712         return 0;
2713 }
2714
2715 static int identity_mapping(struct device *dev)
2716 {
2717         struct device_domain_info *info;
2718
2719         if (likely(!iommu_identity_mapping))
2720                 return 0;
2721
2722         info = dev->archdata.iommu;
2723         if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2724                 return (info->domain == si_domain);
2725
2726         return 0;
2727 }
2728
2729 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2730 {
2731         struct dmar_domain *ndomain;
2732         struct intel_iommu *iommu;
2733         u8 bus, devfn;
2734
2735         iommu = device_to_iommu(dev, &bus, &devfn);
2736         if (!iommu)
2737                 return -ENODEV;
2738
2739         ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2740         if (ndomain != domain)
2741                 return -EBUSY;
2742
2743         return 0;
2744 }
2745
2746 static bool device_has_rmrr(struct device *dev)
2747 {
2748         struct dmar_rmrr_unit *rmrr;
2749         struct device *tmp;
2750         int i;
2751
2752         rcu_read_lock();
2753         for_each_rmrr_units(rmrr) {
2754                 /*
2755                  * Return TRUE if this RMRR contains the device that
2756                  * is passed in.
2757                  */
2758                 for_each_active_dev_scope(rmrr->devices,
2759                                           rmrr->devices_cnt, i, tmp)
2760                         if (tmp == dev) {
2761                                 rcu_read_unlock();
2762                                 return true;
2763                         }
2764         }
2765         rcu_read_unlock();
2766         return false;
2767 }
2768
2769 /*
2770  * There are a couple cases where we need to restrict the functionality of
2771  * devices associated with RMRRs.  The first is when evaluating a device for
2772  * identity mapping because problems exist when devices are moved in and out
2773  * of domains and their respective RMRR information is lost.  This means that
2774  * a device with associated RMRRs will never be in a "passthrough" domain.
2775  * The second is use of the device through the IOMMU API.  This interface
2776  * expects to have full control of the IOVA space for the device.  We cannot
2777  * satisfy both the requirement that RMRR access is maintained and have an
2778  * unencumbered IOVA space.  We also have no ability to quiesce the device's
2779  * use of the RMRR space or even inform the IOMMU API user of the restriction.
2780  * We therefore prevent devices associated with an RMRR from participating in
2781  * the IOMMU API, which eliminates them from device assignment.
2782  *
2783  * In both cases we assume that PCI USB devices with RMRRs have them largely
2784  * for historical reasons and that the RMRR space is not actively used post
2785  * boot.  This exclusion may change if vendors begin to abuse it.
2786  *
2787  * The same exception is made for graphics devices, with the requirement that
2788  * any use of the RMRR regions will be torn down before assigning the device
2789  * to a guest.
2790  */
2791 static bool device_is_rmrr_locked(struct device *dev)
2792 {
2793         if (!device_has_rmrr(dev))
2794                 return false;
2795
2796         if (dev_is_pci(dev)) {
2797                 struct pci_dev *pdev = to_pci_dev(dev);
2798
2799                 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2800                         return false;
2801         }
2802
2803         return true;
2804 }
2805
2806 static int iommu_should_identity_map(struct device *dev, int startup)
2807 {
2808
2809         if (dev_is_pci(dev)) {
2810                 struct pci_dev *pdev = to_pci_dev(dev);
2811
2812                 if (device_is_rmrr_locked(dev))
2813                         return 0;
2814
2815                 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2816                         return 1;
2817
2818                 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2819                         return 1;
2820
2821                 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2822                         return 0;
2823
2824                 /*
2825                  * We want to start off with all devices in the 1:1 domain, and
2826                  * take them out later if we find they can't access all of memory.
2827                  *
2828                  * However, we can't do this for PCI devices behind bridges,
2829                  * because all PCI devices behind the same bridge will end up
2830                  * with the same source-id on their transactions.
2831                  *
2832                  * Practically speaking, we can't change things around for these
2833                  * devices at run-time, because we can't be sure there'll be no
2834                  * DMA transactions in flight for any of their siblings.
2835                  *
2836                  * So PCI devices (unless they're on the root bus) as well as
2837                  * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2838                  * the 1:1 domain, just in _case_ one of their siblings turns out
2839                  * not to be able to map all of memory.
2840                  */
2841                 if (!pci_is_pcie(pdev)) {
2842                         if (!pci_is_root_bus(pdev->bus))
2843                                 return 0;
2844                         if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2845                                 return 0;
2846                 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2847                         return 0;
2848         } else {
2849                 if (device_has_rmrr(dev))
2850                         return 0;
2851         }
2852
2853         /*
2854          * At boot time, we don't yet know if devices will be 64-bit capable.
2855          * Assume that they will — if they turn out not to be, then we can
2856          * take them out of the 1:1 domain later.
2857          */
2858         if (!startup) {
2859                 /*
2860                  * If the device's dma_mask is less than the system's memory
2861                  * size then this is not a candidate for identity mapping.
2862                  */
2863                 u64 dma_mask = *dev->dma_mask;
2864
2865                 if (dev->coherent_dma_mask &&
2866                     dev->coherent_dma_mask < dma_mask)
2867                         dma_mask = dev->coherent_dma_mask;
2868
2869                 return dma_mask >= dma_get_required_mask(dev);
2870         }
2871
2872         return 1;
2873 }
2874
2875 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2876 {
2877         int ret;
2878
2879         if (!iommu_should_identity_map(dev, 1))
2880                 return 0;
2881
2882         ret = domain_add_dev_info(si_domain, dev);
2883         if (!ret)
2884                 pr_info("%s identity mapping for device %s\n",
2885                         hw ? "Hardware" : "Software", dev_name(dev));
2886         else if (ret == -ENODEV)
2887                 /* device not associated with an iommu */
2888                 ret = 0;
2889
2890         return ret;
2891 }
2892
2893
2894 static int __init iommu_prepare_static_identity_mapping(int hw)
2895 {
2896         struct pci_dev *pdev = NULL;
2897         struct dmar_drhd_unit *drhd;
2898         struct intel_iommu *iommu;
2899         struct device *dev;
2900         int i;
2901         int ret = 0;
2902
2903         for_each_pci_dev(pdev) {
2904                 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2905                 if (ret)
2906                         return ret;
2907         }
2908
2909         for_each_active_iommu(iommu, drhd)
2910                 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2911                         struct acpi_device_physical_node *pn;
2912                         struct acpi_device *adev;
2913
2914                         if (dev->bus != &acpi_bus_type)
2915                                 continue;
2916
2917                         adev= to_acpi_device(dev);
2918                         mutex_lock(&adev->physical_node_lock);
2919                         list_for_each_entry(pn, &adev->physical_node_list, node) {
2920                                 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2921                                 if (ret)
2922                                         break;
2923                         }
2924                         mutex_unlock(&adev->physical_node_lock);
2925                         if (ret)
2926                                 return ret;
2927                 }
2928
2929         return 0;
2930 }
2931
2932 static void intel_iommu_init_qi(struct intel_iommu *iommu)
2933 {
2934         /*
2935          * Start from the sane iommu hardware state.
2936          * If the queued invalidation is already initialized by us
2937          * (for example, while enabling interrupt-remapping) then
2938          * we got the things already rolling from a sane state.
2939          */
2940         if (!iommu->qi) {
2941                 /*
2942                  * Clear any previous faults.
2943                  */
2944                 dmar_fault(-1, iommu);
2945                 /*
2946                  * Disable queued invalidation if supported and already enabled
2947                  * before OS handover.
2948                  */
2949                 dmar_disable_qi(iommu);
2950         }
2951
2952         if (dmar_enable_qi(iommu)) {
2953                 /*
2954                  * Queued Invalidate not enabled, use Register Based Invalidate
2955                  */
2956                 iommu->flush.flush_context = __iommu_flush_context;
2957                 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2958                 pr_info("%s: Using Register based invalidation\n",
2959                         iommu->name);
2960         } else {
2961                 iommu->flush.flush_context = qi_flush_context;
2962                 iommu->flush.flush_iotlb = qi_flush_iotlb;
2963                 pr_info("%s: Using Queued invalidation\n", iommu->name);
2964         }
2965 }
2966
2967 static int copy_context_table(struct intel_iommu *iommu,
2968                               struct root_entry *old_re,
2969                               struct context_entry **tbl,
2970                               int bus, bool ext)
2971 {
2972         int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
2973         struct context_entry *new_ce = NULL, ce;
2974         struct context_entry *old_ce = NULL;
2975         struct root_entry re;
2976         phys_addr_t old_ce_phys;
2977
2978         tbl_idx = ext ? bus * 2 : bus;
2979         memcpy(&re, old_re, sizeof(re));
2980
2981         for (devfn = 0; devfn < 256; devfn++) {
2982                 /* First calculate the correct index */
2983                 idx = (ext ? devfn * 2 : devfn) % 256;
2984
2985                 if (idx == 0) {
2986                         /* First save what we may have and clean up */
2987                         if (new_ce) {
2988                                 tbl[tbl_idx] = new_ce;
2989                                 __iommu_flush_cache(iommu, new_ce,
2990                                                     VTD_PAGE_SIZE);
2991                                 pos = 1;
2992                         }
2993
2994                         if (old_ce)
2995                                 iounmap(old_ce);
2996
2997                         ret = 0;
2998                         if (devfn < 0x80)
2999                                 old_ce_phys = root_entry_lctp(&re);
3000                         else
3001                                 old_ce_phys = root_entry_uctp(&re);
3002
3003                         if (!old_ce_phys) {
3004                                 if (ext && devfn == 0) {
3005                                         /* No LCTP, try UCTP */
3006                                         devfn = 0x7f;
3007                                         continue;
3008                                 } else {
3009                                         goto out;
3010                                 }
3011                         }
3012
3013                         ret = -ENOMEM;
3014                         old_ce = memremap(old_ce_phys, PAGE_SIZE,
3015                                         MEMREMAP_WB);
3016                         if (!old_ce)
3017                                 goto out;
3018
3019                         new_ce = alloc_pgtable_page(iommu->node);
3020                         if (!new_ce)
3021                                 goto out_unmap;
3022
3023                         ret = 0;
3024                 }
3025
3026                 /* Now copy the context entry */
3027                 memcpy(&ce, old_ce + idx, sizeof(ce));
3028
3029                 if (!__context_present(&ce))
3030                         continue;
3031
3032                 did = context_domain_id(&ce);
3033                 if (did >= 0 && did < cap_ndoms(iommu->cap))
3034                         set_bit(did, iommu->domain_ids);
3035
3036                 /*
3037                  * We need a marker for copied context entries. This
3038                  * marker needs to work for the old format as well as
3039                  * for extended context entries.
3040                  *
3041                  * Bit 67 of the context entry is used. In the old
3042                  * format this bit is available to software, in the
3043                  * extended format it is the PGE bit, but PGE is ignored
3044                  * by HW if PASIDs are disabled (and thus still
3045                  * available).
3046                  *
3047                  * So disable PASIDs first and then mark the entry
3048                  * copied. This means that we don't copy PASID
3049                  * translations from the old kernel, but this is fine as
3050                  * faults there are not fatal.
3051                  */
3052                 context_clear_pasid_enable(&ce);
3053                 context_set_copied(&ce);
3054
3055                 new_ce[idx] = ce;
3056         }
3057
3058         tbl[tbl_idx + pos] = new_ce;
3059
3060         __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3061
3062 out_unmap:
3063         memunmap(old_ce);
3064
3065 out:
3066         return ret;
3067 }
3068
3069 static int copy_translation_tables(struct intel_iommu *iommu)
3070 {
3071         struct context_entry **ctxt_tbls;
3072         struct root_entry *old_rt;
3073         phys_addr_t old_rt_phys;
3074         int ctxt_table_entries;
3075         unsigned long flags;
3076         u64 rtaddr_reg;
3077         int bus, ret;
3078         bool new_ext, ext;
3079
3080         rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3081         ext        = !!(rtaddr_reg & DMA_RTADDR_RTT);
3082         new_ext    = !!ecap_ecs(iommu->ecap);
3083
3084         /*
3085          * The RTT bit can only be changed when translation is disabled,
3086          * but disabling translation means to open a window for data
3087          * corruption. So bail out and don't copy anything if we would
3088          * have to change the bit.
3089          */
3090         if (new_ext != ext)
3091                 return -EINVAL;
3092
3093         old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3094         if (!old_rt_phys)
3095                 return -EINVAL;
3096
3097         old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
3098         if (!old_rt)
3099                 return -ENOMEM;
3100
3101         /* This is too big for the stack - allocate it from slab */
3102         ctxt_table_entries = ext ? 512 : 256;
3103         ret = -ENOMEM;
3104         ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
3105         if (!ctxt_tbls)
3106                 goto out_unmap;
3107
3108         for (bus = 0; bus < 256; bus++) {
3109                 ret = copy_context_table(iommu, &old_rt[bus],
3110                                          ctxt_tbls, bus, ext);
3111                 if (ret) {
3112                         pr_err("%s: Failed to copy context table for bus %d\n",
3113                                 iommu->name, bus);
3114                         continue;
3115                 }
3116         }
3117
3118         spin_lock_irqsave(&iommu->lock, flags);
3119
3120         /* Context tables are copied, now write them to the root_entry table */
3121         for (bus = 0; bus < 256; bus++) {
3122                 int idx = ext ? bus * 2 : bus;
3123                 u64 val;
3124
3125                 if (ctxt_tbls[idx]) {
3126                         val = virt_to_phys(ctxt_tbls[idx]) | 1;
3127                         iommu->root_entry[bus].lo = val;
3128                 }
3129
3130                 if (!ext || !ctxt_tbls[idx + 1])
3131                         continue;
3132
3133                 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3134                 iommu->root_entry[bus].hi = val;
3135         }
3136
3137         spin_unlock_irqrestore(&iommu->lock, flags);
3138
3139         kfree(ctxt_tbls);
3140
3141         __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3142
3143         ret = 0;
3144
3145 out_unmap:
3146         memunmap(old_rt);
3147
3148         return ret;
3149 }
3150
3151 static int __init init_dmars(void)
3152 {
3153         struct dmar_drhd_unit *drhd;
3154         struct dmar_rmrr_unit *rmrr;
3155         bool copied_tables = false;
3156         struct device *dev;
3157         struct intel_iommu *iommu;
3158         int i, ret, cpu;
3159
3160         /*
3161          * for each drhd
3162          *    allocate root
3163          *    initialize and program root entry to not present
3164          * endfor
3165          */
3166         for_each_drhd_unit(drhd) {
3167                 /*
3168                  * lock not needed as this is only incremented in the single
3169                  * threaded kernel __init code path all other access are read
3170                  * only
3171                  */
3172                 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3173                         g_num_of_iommus++;
3174                         continue;
3175                 }
3176                 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3177         }
3178
3179         /* Preallocate enough resources for IOMMU hot-addition */
3180         if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3181                 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3182
3183         g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3184                         GFP_KERNEL);
3185         if (!g_iommus) {
3186                 pr_err("Allocating global iommu array failed\n");
3187                 ret = -ENOMEM;
3188                 goto error;
3189         }
3190
3191         for_each_possible_cpu(cpu) {
3192                 struct deferred_flush_data *dfd = per_cpu_ptr(&deferred_flush,
3193                                                               cpu);
3194
3195                 dfd->tables = kzalloc(g_num_of_iommus *
3196                                       sizeof(struct deferred_flush_table),
3197                                       GFP_KERNEL);
3198                 if (!dfd->tables) {
3199                         ret = -ENOMEM;
3200                         goto free_g_iommus;
3201                 }
3202
3203                 spin_lock_init(&dfd->lock);
3204                 setup_timer(&dfd->timer, flush_unmaps_timeout, cpu);
3205         }
3206
3207         for_each_active_iommu(iommu, drhd) {
3208                 g_iommus[iommu->seq_id] = iommu;
3209
3210                 intel_iommu_init_qi(iommu);
3211
3212                 ret = iommu_init_domains(iommu);
3213                 if (ret)
3214                         goto free_iommu;
3215
3216                 init_translation_status(iommu);
3217
3218                 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3219                         iommu_disable_translation(iommu);
3220                         clear_translation_pre_enabled(iommu);
3221                         pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3222                                 iommu->name);
3223                 }
3224
3225                 /*
3226                  * TBD:
3227                  * we could share the same root & context tables
3228                  * among all IOMMU's. Need to Split it later.
3229                  */
3230                 ret = iommu_alloc_root_entry(iommu);
3231                 if (ret)
3232                         goto free_iommu;
3233
3234                 if (translation_pre_enabled(iommu)) {
3235                         pr_info("Translation already enabled - trying to copy translation structures\n");
3236
3237                         ret = copy_translation_tables(iommu);
3238                         if (ret) {
3239                                 /*
3240                                  * We found the IOMMU with translation
3241                                  * enabled - but failed to copy over the
3242                                  * old root-entry table. Try to proceed
3243                                  * by disabling translation now and
3244                                  * allocating a clean root-entry table.
3245                                  * This might cause DMAR faults, but
3246                                  * probably the dump will still succeed.
3247                                  */
3248                                 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3249                                        iommu->name);
3250                                 iommu_disable_translation(iommu);
3251                                 clear_translation_pre_enabled(iommu);
3252                         } else {
3253                                 pr_info("Copied translation tables from previous kernel for %s\n",
3254                                         iommu->name);
3255                                 copied_tables = true;
3256                         }
3257                 }
3258
3259                 if (!ecap_pass_through(iommu->ecap))
3260                         hw_pass_through = 0;
3261 #ifdef CONFIG_INTEL_IOMMU_SVM
3262                 if (pasid_enabled(iommu))
3263                         intel_svm_alloc_pasid_tables(iommu);
3264 #endif
3265         }
3266
3267         /*
3268          * Now that qi is enabled on all iommus, set the root entry and flush
3269          * caches. This is required on some Intel X58 chipsets, otherwise the
3270          * flush_context function will loop forever and the boot hangs.
3271          */
3272         for_each_active_iommu(iommu, drhd) {
3273                 iommu_flush_write_buffer(iommu);
3274                 iommu_set_root_entry(iommu);
3275                 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3276                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3277         }
3278
3279         if (iommu_pass_through)
3280                 iommu_identity_mapping |= IDENTMAP_ALL;
3281
3282 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3283         iommu_identity_mapping |= IDENTMAP_GFX;
3284 #endif
3285
3286         if (iommu_identity_mapping) {
3287                 ret = si_domain_init(hw_pass_through);
3288                 if (ret)
3289                         goto free_iommu;
3290         }
3291
3292         check_tylersburg_isoch();
3293
3294         /*
3295          * If we copied translations from a previous kernel in the kdump
3296          * case, we can not assign the devices to domains now, as that
3297          * would eliminate the old mappings. So skip this part and defer
3298          * the assignment to device driver initialization time.
3299          */
3300         if (copied_tables)
3301                 goto domains_done;
3302
3303         /*
3304          * If pass through is not set or not enabled, setup context entries for
3305          * identity mappings for rmrr, gfx, and isa and may fall back to static
3306          * identity mapping if iommu_identity_mapping is set.
3307          */
3308         if (iommu_identity_mapping) {
3309                 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3310                 if (ret) {
3311                         pr_crit("Failed to setup IOMMU pass-through\n");
3312                         goto free_iommu;
3313                 }
3314         }
3315         /*
3316          * For each rmrr
3317          *   for each dev attached to rmrr
3318          *   do
3319          *     locate drhd for dev, alloc domain for dev
3320          *     allocate free domain
3321          *     allocate page table entries for rmrr
3322          *     if context not allocated for bus
3323          *           allocate and init context
3324          *           set present in root table for this bus
3325          *     init context with domain, translation etc
3326          *    endfor
3327          * endfor
3328          */
3329         pr_info("Setting RMRR:\n");
3330         for_each_rmrr_units(rmrr) {
3331                 /* some BIOS lists non-exist devices in DMAR table. */
3332                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3333                                           i, dev) {
3334                         ret = iommu_prepare_rmrr_dev(rmrr, dev);
3335                         if (ret)
3336                                 pr_err("Mapping reserved region failed\n");
3337                 }
3338         }
3339
3340         iommu_prepare_isa();
3341
3342 domains_done:
3343
3344         /*
3345          * for each drhd
3346          *   enable fault log
3347          *   global invalidate context cache
3348          *   global invalidate iotlb
3349          *   enable translation
3350          */
3351         for_each_iommu(iommu, drhd) {
3352                 if (drhd->ignored) {
3353                         /*
3354                          * we always have to disable PMRs or DMA may fail on
3355                          * this device
3356                          */
3357                         if (force_on)
3358                                 iommu_disable_protect_mem_regions(iommu);
3359                         continue;
3360                 }
3361
3362                 iommu_flush_write_buffer(iommu);
3363
3364 #ifdef CONFIG_INTEL_IOMMU_SVM
3365                 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
3366                         ret = intel_svm_enable_prq(iommu);
3367                         if (ret)
3368                                 goto free_iommu;
3369                 }
3370 #endif
3371                 ret = dmar_set_interrupt(iommu);
3372                 if (ret)
3373                         goto free_iommu;
3374
3375                 if (!translation_pre_enabled(iommu))
3376                         iommu_enable_translation(iommu);
3377
3378                 iommu_disable_protect_mem_regions(iommu);
3379         }
3380
3381         return 0;
3382
3383 free_iommu:
3384         for_each_active_iommu(iommu, drhd) {
3385                 disable_dmar_iommu(iommu);
3386                 free_dmar_iommu(iommu);
3387         }
3388 free_g_iommus:
3389         for_each_possible_cpu(cpu)
3390                 kfree(per_cpu_ptr(&deferred_flush, cpu)->tables);
3391         kfree(g_iommus);
3392 error:
3393         return ret;
3394 }
3395
3396 /* This takes a number of _MM_ pages, not VTD pages */
3397 static unsigned long intel_alloc_iova(struct device *dev,
3398                                      struct dmar_domain *domain,
3399                                      unsigned long nrpages, uint64_t dma_mask)
3400 {
3401         unsigned long iova_pfn = 0;
3402
3403         /* Restrict dma_mask to the width that the iommu can handle */
3404         dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3405         /* Ensure we reserve the whole size-aligned region */
3406         nrpages = __roundup_pow_of_two(nrpages);
3407
3408         if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
3409                 /*
3410                  * First try to allocate an io virtual address in
3411                  * DMA_BIT_MASK(32) and if that fails then try allocating
3412                  * from higher range
3413                  */
3414                 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3415                                            IOVA_PFN(DMA_BIT_MASK(32)));
3416                 if (iova_pfn)
3417                         return iova_pfn;
3418         }
3419         iova_pfn = alloc_iova_fast(&domain->iovad, nrpages, IOVA_PFN(dma_mask));
3420         if (unlikely(!iova_pfn)) {
3421                 pr_err("Allocating %ld-page iova for %s failed",
3422                        nrpages, dev_name(dev));
3423                 return 0;
3424         }
3425
3426         return iova_pfn;
3427 }
3428
3429 static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
3430 {
3431         struct dmar_domain *domain, *tmp;
3432         struct dmar_rmrr_unit *rmrr;
3433         struct device *i_dev;
3434         int i, ret;
3435
3436         domain = find_domain(dev);
3437         if (domain)
3438                 goto out;
3439
3440         domain = find_or_alloc_domain(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3441         if (!domain)
3442                 goto out;
3443
3444         /* We have a new domain - setup possible RMRRs for the device */
3445         rcu_read_lock();
3446         for_each_rmrr_units(rmrr) {
3447                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3448                                           i, i_dev) {
3449                         if (i_dev != dev)
3450                                 continue;
3451
3452                         ret = domain_prepare_identity_map(dev, domain,
3453                                                           rmrr->base_address,
3454                                                           rmrr->end_address);
3455                         if (ret)
3456                                 dev_err(dev, "Mapping reserved region failed\n");
3457                 }
3458         }
3459         rcu_read_unlock();
3460
3461         tmp = set_domain_for_dev(dev, domain);
3462         if (!tmp || domain != tmp) {
3463                 domain_exit(domain);
3464                 domain = tmp;
3465         }
3466
3467 out:
3468
3469         if (!domain)
3470                 pr_err("Allocating domain for %s failed\n", dev_name(dev));
3471
3472
3473         return domain;
3474 }
3475
3476 static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
3477 {
3478         struct device_domain_info *info;
3479
3480         /* No lock here, assumes no domain exit in normal case */
3481         info = dev->archdata.iommu;
3482         if (likely(info))
3483                 return info->domain;
3484
3485         return __get_valid_domain_for_dev(dev);
3486 }
3487
3488 /* Check if the dev needs to go through non-identity map and unmap process.*/
3489 static int iommu_no_mapping(struct device *dev)
3490 {
3491         int found;
3492
3493         if (iommu_dummy(dev))
3494                 return 1;
3495
3496         if (!iommu_identity_mapping)
3497                 return 0;
3498
3499         found = identity_mapping(dev);
3500         if (found) {
3501                 if (iommu_should_identity_map(dev, 0))
3502                         return 1;
3503                 else {
3504                         /*
3505                          * 32 bit DMA is removed from si_domain and fall back
3506                          * to non-identity mapping.
3507                          */
3508                         dmar_remove_one_dev_info(si_domain, dev);
3509                         pr_info("32bit %s uses non-identity mapping\n",
3510                                 dev_name(dev));
3511                         return 0;
3512                 }
3513         } else {
3514                 /*
3515                  * In case of a detached 64 bit DMA device from vm, the device
3516                  * is put into si_domain for identity mapping.
3517                  */
3518                 if (iommu_should_identity_map(dev, 0)) {
3519                         int ret;
3520                         ret = domain_add_dev_info(si_domain, dev);
3521                         if (!ret) {
3522                                 pr_info("64bit %s uses identity mapping\n",
3523                                         dev_name(dev));
3524                                 return 1;
3525                         }
3526                 }
3527         }
3528
3529         return 0;
3530 }
3531
3532 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3533                                      size_t size, int dir, u64 dma_mask)
3534 {
3535         struct dmar_domain *domain;
3536         phys_addr_t start_paddr;
3537         unsigned long iova_pfn;
3538         int prot = 0;
3539         int ret;
3540         struct intel_iommu *iommu;
3541         unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3542
3543         BUG_ON(dir == DMA_NONE);
3544
3545         if (iommu_no_mapping(dev))
3546                 return paddr;
3547
3548         domain = get_valid_domain_for_dev(dev);
3549         if (!domain)
3550                 return 0;
3551
3552         iommu = domain_get_iommu(domain);
3553         size = aligned_nrpages(paddr, size);
3554
3555         iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3556         if (!iova_pfn)
3557                 goto error;
3558
3559         /*
3560          * Check if DMAR supports zero-length reads on write only
3561          * mappings..
3562          */
3563         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3564                         !cap_zlr(iommu->cap))
3565                 prot |= DMA_PTE_READ;
3566         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3567                 prot |= DMA_PTE_WRITE;
3568         /*
3569          * paddr - (paddr + size) might be partial page, we should map the whole
3570          * page.  Note: if two part of one page are separately mapped, we
3571          * might have two guest_addr mapping to the same host paddr, but this
3572          * is not a big problem
3573          */
3574         ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
3575                                  mm_to_dma_pfn(paddr_pfn), size, prot);
3576         if (ret)
3577                 goto error;
3578
3579         /* it's a non-present to present mapping. Only flush if caching mode */
3580         if (cap_caching_mode(iommu->cap))
3581                 iommu_flush_iotlb_psi(iommu, domain,
3582                                       mm_to_dma_pfn(iova_pfn),
3583                                       size, 0, 1);
3584         else
3585                 iommu_flush_write_buffer(iommu);
3586
3587         start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
3588         start_paddr += paddr & ~PAGE_MASK;
3589         return start_paddr;
3590
3591 error:
3592         if (iova_pfn)
3593                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3594         pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
3595                 dev_name(dev), size, (unsigned long long)paddr, dir);
3596         return 0;
3597 }
3598
3599 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3600                                  unsigned long offset, size_t size,
3601                                  enum dma_data_direction dir,
3602                                  unsigned long attrs)
3603 {
3604         return __intel_map_single(dev, page_to_phys(page) + offset, size,
3605                                   dir, *dev->dma_mask);
3606 }
3607
3608 static void flush_unmaps(struct deferred_flush_data *flush_data)
3609 {
3610         int i, j;
3611
3612         flush_data->timer_on = 0;
3613
3614         /* just flush them all */
3615         for (i = 0; i < g_num_of_iommus; i++) {
3616                 struct intel_iommu *iommu = g_iommus[i];
3617                 struct deferred_flush_table *flush_table =
3618                                 &flush_data->tables[i];
3619                 if (!iommu)
3620                         continue;
3621
3622                 if (!flush_table->next)
3623                         continue;
3624
3625                 /* In caching mode, global flushes turn emulation expensive */
3626                 if (!cap_caching_mode(iommu->cap))
3627                         iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3628                                          DMA_TLB_GLOBAL_FLUSH);
3629                 for (j = 0; j < flush_table->next; j++) {
3630                         unsigned long mask;
3631                         struct deferred_flush_entry *entry =
3632                                                 &flush_table->entries[j];
3633                         unsigned long iova_pfn = entry->iova_pfn;
3634                         unsigned long nrpages = entry->nrpages;
3635                         struct dmar_domain *domain = entry->domain;
3636                         struct page *freelist = entry->freelist;
3637
3638                         /* On real hardware multiple invalidations are expensive */
3639                         if (cap_caching_mode(iommu->cap))
3640                                 iommu_flush_iotlb_psi(iommu, domain,
3641                                         mm_to_dma_pfn(iova_pfn),
3642                                         nrpages, !freelist, 0);
3643                         else {
3644                                 mask = ilog2(nrpages);
3645                                 iommu_flush_dev_iotlb(domain,
3646                                                 (uint64_t)iova_pfn << PAGE_SHIFT, mask);
3647                         }
3648                         free_iova_fast(&domain->iovad, iova_pfn, nrpages);
3649                         if (freelist)
3650                                 dma_free_pagelist(freelist);
3651                 }
3652                 flush_table->next = 0;
3653         }
3654
3655         flush_data->size = 0;
3656 }
3657
3658 static void flush_unmaps_timeout(unsigned long cpuid)
3659 {
3660         struct deferred_flush_data *flush_data = per_cpu_ptr(&deferred_flush, cpuid);
3661         unsigned long flags;
3662
3663         spin_lock_irqsave(&flush_data->lock, flags);
3664         flush_unmaps(flush_data);
3665         spin_unlock_irqrestore(&flush_data->lock, flags);
3666 }
3667
3668 static void add_unmap(struct dmar_domain *dom, unsigned long iova_pfn,
3669                       unsigned long nrpages, struct page *freelist)
3670 {
3671         unsigned long flags;
3672         int entry_id, iommu_id;
3673         struct intel_iommu *iommu;
3674         struct deferred_flush_entry *entry;
3675         struct deferred_flush_data *flush_data;
3676         unsigned int cpuid;
3677
3678         cpuid = get_cpu();
3679         flush_data = per_cpu_ptr(&deferred_flush, cpuid);
3680
3681         /* Flush all CPUs' entries to avoid deferring too much.  If
3682          * this becomes a bottleneck, can just flush us, and rely on
3683          * flush timer for the rest.
3684          */
3685         if (flush_data->size == HIGH_WATER_MARK) {
3686                 int cpu;
3687
3688                 for_each_online_cpu(cpu)
3689                         flush_unmaps_timeout(cpu);
3690         }
3691
3692         spin_lock_irqsave(&flush_data->lock, flags);
3693
3694         iommu = domain_get_iommu(dom);
3695         iommu_id = iommu->seq_id;
3696
3697         entry_id = flush_data->tables[iommu_id].next;
3698         ++(flush_data->tables[iommu_id].next);
3699
3700         entry = &flush_data->tables[iommu_id].entries[entry_id];
3701         entry->domain = dom;
3702         entry->iova_pfn = iova_pfn;
3703         entry->nrpages = nrpages;
3704         entry->freelist = freelist;
3705
3706         if (!flush_data->timer_on) {
3707                 mod_timer(&flush_data->timer, jiffies + msecs_to_jiffies(10));
3708                 flush_data->timer_on = 1;
3709         }
3710         flush_data->size++;
3711         spin_unlock_irqrestore(&flush_data->lock, flags);
3712
3713         put_cpu();
3714 }
3715
3716 static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
3717 {
3718         struct dmar_domain *domain;
3719         unsigned long start_pfn, last_pfn;
3720         unsigned long nrpages;
3721         unsigned long iova_pfn;
3722         struct intel_iommu *iommu;
3723         struct page *freelist;
3724
3725         if (iommu_no_mapping(dev))
3726                 return;
3727
3728         domain = find_domain(dev);
3729         BUG_ON(!domain);
3730
3731         iommu = domain_get_iommu(domain);
3732
3733         iova_pfn = IOVA_PFN(dev_addr);
3734
3735         nrpages = aligned_nrpages(dev_addr, size);
3736         start_pfn = mm_to_dma_pfn(iova_pfn);
3737         last_pfn = start_pfn + nrpages - 1;
3738
3739         pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3740                  dev_name(dev), start_pfn, last_pfn);
3741
3742         freelist = domain_unmap(domain, start_pfn, last_pfn);
3743
3744         if (intel_iommu_strict) {
3745                 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
3746                                       nrpages, !freelist, 0);
3747                 /* free iova */
3748                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
3749                 dma_free_pagelist(freelist);
3750         } else {
3751                 add_unmap(domain, iova_pfn, nrpages, freelist);
3752                 /*
3753                  * queue up the release of the unmap to save the 1/6th of the
3754                  * cpu used up by the iotlb flush operation...
3755                  */
3756         }
3757 }
3758
3759 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3760                              size_t size, enum dma_data_direction dir,
3761                              unsigned long attrs)
3762 {
3763         intel_unmap(dev, dev_addr, size);
3764 }
3765
3766 static void *intel_alloc_coherent(struct device *dev, size_t size,
3767                                   dma_addr_t *dma_handle, gfp_t flags,
3768                                   unsigned long attrs)
3769 {
3770         struct page *page = NULL;
3771         int order;
3772
3773         size = PAGE_ALIGN(size);
3774         order = get_order(size);
3775
3776         if (!iommu_no_mapping(dev))
3777                 flags &= ~(GFP_DMA | GFP_DMA32);
3778         else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3779                 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3780                         flags |= GFP_DMA;
3781                 else
3782                         flags |= GFP_DMA32;
3783         }
3784
3785         if (gfpflags_allow_blocking(flags)) {
3786                 unsigned int count = size >> PAGE_SHIFT;
3787
3788                 page = dma_alloc_from_contiguous(dev, count, order);
3789                 if (page && iommu_no_mapping(dev) &&
3790                     page_to_phys(page) + size > dev->coherent_dma_mask) {
3791                         dma_release_from_contiguous(dev, page, count);
3792                         page = NULL;
3793                 }
3794         }
3795
3796         if (!page)
3797                 page = alloc_pages(flags, order);
3798         if (!page)
3799                 return NULL;
3800         memset(page_address(page), 0, size);
3801
3802         *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3803                                          DMA_BIDIRECTIONAL,
3804                                          dev->coherent_dma_mask);
3805         if (*dma_handle)
3806                 return page_address(page);
3807         if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3808                 __free_pages(page, order);
3809
3810         return NULL;
3811 }
3812
3813 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3814                                 dma_addr_t dma_handle, unsigned long attrs)
3815 {
3816         int order;
3817         struct page *page = virt_to_page(vaddr);
3818
3819         size = PAGE_ALIGN(size);
3820         order = get_order(size);
3821
3822         intel_unmap(dev, dma_handle, size);
3823         if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3824                 __free_pages(page, order);
3825 }
3826
3827 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3828                            int nelems, enum dma_data_direction dir,
3829                            unsigned long attrs)
3830 {
3831         dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
3832         unsigned long nrpages = 0;
3833         struct scatterlist *sg;
3834         int i;
3835
3836         for_each_sg(sglist, sg, nelems, i) {
3837                 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
3838         }
3839
3840         intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
3841 }
3842
3843 static int intel_nontranslate_map_sg(struct device *hddev,
3844         struct scatterlist *sglist, int nelems, int dir)
3845 {
3846         int i;
3847         struct scatterlist *sg;
3848
3849         for_each_sg(sglist, sg, nelems, i) {
3850                 BUG_ON(!sg_page(sg));
3851                 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
3852                 sg->dma_length = sg->length;
3853         }
3854         return nelems;
3855 }
3856
3857 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3858                         enum dma_data_direction dir, unsigned long attrs)
3859 {
3860         int i;
3861         struct dmar_domain *domain;
3862         size_t size = 0;
3863         int prot = 0;
3864         unsigned long iova_pfn;
3865         int ret;
3866         struct scatterlist *sg;
3867         unsigned long start_vpfn;
3868         struct intel_iommu *iommu;
3869
3870         BUG_ON(dir == DMA_NONE);
3871         if (iommu_no_mapping(dev))
3872                 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3873
3874         domain = get_valid_domain_for_dev(dev);
3875         if (!domain)
3876                 return 0;
3877
3878         iommu = domain_get_iommu(domain);
3879
3880         for_each_sg(sglist, sg, nelems, i)
3881                 size += aligned_nrpages(sg->offset, sg->length);
3882
3883         iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3884                                 *dev->dma_mask);
3885         if (!iova_pfn) {
3886                 sglist->dma_length = 0;
3887                 return 0;
3888         }
3889
3890         /*
3891          * Check if DMAR supports zero-length reads on write only
3892          * mappings..
3893          */
3894         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3895                         !cap_zlr(iommu->cap))
3896                 prot |= DMA_PTE_READ;
3897         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3898                 prot |= DMA_PTE_WRITE;
3899
3900         start_vpfn = mm_to_dma_pfn(iova_pfn);
3901
3902         ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3903         if (unlikely(ret)) {
3904                 dma_pte_free_pagetable(domain, start_vpfn,
3905                                        start_vpfn + size - 1);
3906                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3907                 return 0;
3908         }
3909
3910         /* it's a non-present to present mapping. Only flush if caching mode */
3911         if (cap_caching_mode(iommu->cap))
3912                 iommu_flush_iotlb_psi(iommu, domain, start_vpfn, size, 0, 1);
3913         else
3914                 iommu_flush_write_buffer(iommu);
3915
3916         return nelems;
3917 }
3918
3919 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3920 {
3921         return !dma_addr;
3922 }
3923
3924 struct dma_map_ops intel_dma_ops = {
3925         .alloc = intel_alloc_coherent,
3926         .free = intel_free_coherent,
3927         .map_sg = intel_map_sg,
3928         .unmap_sg = intel_unmap_sg,
3929         .map_page = intel_map_page,
3930         .unmap_page = intel_unmap_page,
3931         .mapping_error = intel_mapping_error,
3932 };
3933
3934 static inline int iommu_domain_cache_init(void)
3935 {
3936         int ret = 0;
3937
3938         iommu_domain_cache = kmem_cache_create("iommu_domain",
3939                                          sizeof(struct dmar_domain),
3940                                          0,
3941                                          SLAB_HWCACHE_ALIGN,
3942
3943                                          NULL);
3944         if (!iommu_domain_cache) {
3945                 pr_err("Couldn't create iommu_domain cache\n");
3946                 ret = -ENOMEM;
3947         }
3948
3949         return ret;
3950 }
3951
3952 static inline int iommu_devinfo_cache_init(void)
3953 {
3954         int ret = 0;
3955
3956         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3957                                          sizeof(struct device_domain_info),
3958                                          0,
3959                                          SLAB_HWCACHE_ALIGN,
3960                                          NULL);
3961         if (!iommu_devinfo_cache) {
3962                 pr_err("Couldn't create devinfo cache\n");
3963                 ret = -ENOMEM;
3964         }
3965
3966         return ret;
3967 }
3968
3969 static int __init iommu_init_mempool(void)
3970 {
3971         int ret;
3972         ret = iova_cache_get();
3973         if (ret)
3974                 return ret;
3975
3976         ret = iommu_domain_cache_init();
3977         if (ret)
3978                 goto domain_error;
3979
3980         ret = iommu_devinfo_cache_init();
3981         if (!ret)
3982                 return ret;
3983
3984         kmem_cache_destroy(iommu_domain_cache);
3985 domain_error:
3986         iova_cache_put();
3987
3988         return -ENOMEM;
3989 }
3990
3991 static void __init iommu_exit_mempool(void)
3992 {
3993         kmem_cache_destroy(iommu_devinfo_cache);
3994         kmem_cache_destroy(iommu_domain_cache);
3995         iova_cache_put();
3996 }
3997
3998 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3999 {
4000         struct dmar_drhd_unit *drhd;
4001         u32 vtbar;
4002         int rc;
4003
4004         /* We know that this device on this chipset has its own IOMMU.
4005          * If we find it under a different IOMMU, then the BIOS is lying
4006          * to us. Hope that the IOMMU for this device is actually
4007          * disabled, and it needs no translation...
4008          */
4009         rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
4010         if (rc) {
4011                 /* "can't" happen */
4012                 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
4013                 return;
4014         }
4015         vtbar &= 0xffff0000;
4016
4017         /* we know that the this iommu should be at offset 0xa000 from vtbar */
4018         drhd = dmar_find_matched_drhd_unit(pdev);
4019         if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
4020                             TAINT_FIRMWARE_WORKAROUND,
4021                             "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
4022                 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4023 }
4024 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
4025
4026 static void __init init_no_remapping_devices(void)
4027 {
4028         struct dmar_drhd_unit *drhd;
4029         struct device *dev;
4030         int i;
4031
4032         for_each_drhd_unit(drhd) {
4033                 if (!drhd->include_all) {
4034                         for_each_active_dev_scope(drhd->devices,
4035                                                   drhd->devices_cnt, i, dev)
4036                                 break;
4037                         /* ignore DMAR unit if no devices exist */
4038                         if (i == drhd->devices_cnt)
4039                                 drhd->ignored = 1;
4040                 }
4041         }
4042
4043         for_each_active_drhd_unit(drhd) {
4044                 if (drhd->include_all)
4045                         continue;
4046
4047                 for_each_active_dev_scope(drhd->devices,
4048                                           drhd->devices_cnt, i, dev)
4049                         if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
4050                                 break;
4051                 if (i < drhd->devices_cnt)
4052                         continue;
4053
4054                 /* This IOMMU has *only* gfx devices. Either bypass it or
4055                    set the gfx_mapped flag, as appropriate */
4056                 if (dmar_map_gfx) {
4057                         intel_iommu_gfx_mapped = 1;
4058                 } else {
4059                         drhd->ignored = 1;
4060                         for_each_active_dev_scope(drhd->devices,
4061                                                   drhd->devices_cnt, i, dev)
4062                                 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4063                 }
4064         }
4065 }
4066
4067 #ifdef CONFIG_SUSPEND
4068 static int init_iommu_hw(void)
4069 {
4070         struct dmar_drhd_unit *drhd;
4071         struct intel_iommu *iommu = NULL;
4072
4073         for_each_active_iommu(iommu, drhd)
4074                 if (iommu->qi)
4075                         dmar_reenable_qi(iommu);
4076
4077         for_each_iommu(iommu, drhd) {
4078                 if (drhd->ignored) {
4079                         /*
4080                          * we always have to disable PMRs or DMA may fail on
4081                          * this device
4082                          */
4083                         if (force_on)
4084                                 iommu_disable_protect_mem_regions(iommu);
4085                         continue;
4086                 }
4087
4088                 iommu_flush_write_buffer(iommu);
4089
4090                 iommu_set_root_entry(iommu);
4091
4092                 iommu->flush.flush_context(iommu, 0, 0, 0,
4093                                            DMA_CCMD_GLOBAL_INVL);
4094                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4095                 iommu_enable_translation(iommu);
4096                 iommu_disable_protect_mem_regions(iommu);
4097         }
4098
4099         return 0;
4100 }
4101
4102 static void iommu_flush_all(void)
4103 {
4104         struct dmar_drhd_unit *drhd;
4105         struct intel_iommu *iommu;
4106
4107         for_each_active_iommu(iommu, drhd) {
4108                 iommu->flush.flush_context(iommu, 0, 0, 0,
4109                                            DMA_CCMD_GLOBAL_INVL);
4110                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
4111                                          DMA_TLB_GLOBAL_FLUSH);
4112         }
4113 }
4114
4115 static int iommu_suspend(void)
4116 {
4117         struct dmar_drhd_unit *drhd;
4118         struct intel_iommu *iommu = NULL;
4119         unsigned long flag;
4120
4121         for_each_active_iommu(iommu, drhd) {
4122                 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
4123                                                  GFP_ATOMIC);
4124                 if (!iommu->iommu_state)
4125                         goto nomem;
4126         }
4127
4128         iommu_flush_all();
4129
4130         for_each_active_iommu(iommu, drhd) {
4131                 iommu_disable_translation(iommu);
4132
4133                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4134
4135                 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4136                         readl(iommu->reg + DMAR_FECTL_REG);
4137                 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4138                         readl(iommu->reg + DMAR_FEDATA_REG);
4139                 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4140                         readl(iommu->reg + DMAR_FEADDR_REG);
4141                 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4142                         readl(iommu->reg + DMAR_FEUADDR_REG);
4143
4144                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4145         }
4146         return 0;
4147
4148 nomem:
4149         for_each_active_iommu(iommu, drhd)
4150                 kfree(iommu->iommu_state);
4151
4152         return -ENOMEM;
4153 }
4154
4155 static void iommu_resume(void)
4156 {
4157         struct dmar_drhd_unit *drhd;
4158         struct intel_iommu *iommu = NULL;
4159         unsigned long flag;
4160
4161         if (init_iommu_hw()) {
4162                 if (force_on)
4163                         panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4164                 else
4165                         WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
4166                 return;
4167         }
4168
4169         for_each_active_iommu(iommu, drhd) {
4170
4171                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4172
4173                 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4174                         iommu->reg + DMAR_FECTL_REG);
4175                 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4176                         iommu->reg + DMAR_FEDATA_REG);
4177                 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4178                         iommu->reg + DMAR_FEADDR_REG);
4179                 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4180                         iommu->reg + DMAR_FEUADDR_REG);
4181
4182                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4183         }
4184
4185         for_each_active_iommu(iommu, drhd)
4186                 kfree(iommu->iommu_state);
4187 }
4188
4189 static struct syscore_ops iommu_syscore_ops = {
4190         .resume         = iommu_resume,
4191         .suspend        = iommu_suspend,
4192 };
4193
4194 static void __init init_iommu_pm_ops(void)
4195 {
4196         register_syscore_ops(&iommu_syscore_ops);
4197 }
4198
4199 #else
4200 static inline void init_iommu_pm_ops(void) {}
4201 #endif  /* CONFIG_PM */
4202
4203
4204 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
4205 {
4206         struct acpi_dmar_reserved_memory *rmrr;
4207         struct dmar_rmrr_unit *rmrru;
4208
4209         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4210         if (!rmrru)
4211                 return -ENOMEM;
4212
4213         rmrru->hdr = header;
4214         rmrr = (struct acpi_dmar_reserved_memory *)header;
4215         rmrru->base_address = rmrr->base_address;
4216         rmrru->end_address = rmrr->end_address;
4217         rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4218                                 ((void *)rmrr) + rmrr->header.length,
4219                                 &rmrru->devices_cnt);
4220         if (rmrru->devices_cnt && rmrru->devices == NULL) {
4221                 kfree(rmrru);
4222                 return -ENOMEM;
4223         }
4224
4225         list_add(&rmrru->list, &dmar_rmrr_units);
4226
4227         return 0;
4228 }
4229
4230 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4231 {
4232         struct dmar_atsr_unit *atsru;
4233         struct acpi_dmar_atsr *tmp;
4234
4235         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4236                 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4237                 if (atsr->segment != tmp->segment)
4238                         continue;
4239                 if (atsr->header.length != tmp->header.length)
4240                         continue;
4241                 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4242                         return atsru;
4243         }
4244
4245         return NULL;
4246 }
4247
4248 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4249 {
4250         struct acpi_dmar_atsr *atsr;
4251         struct dmar_atsr_unit *atsru;
4252
4253         if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
4254                 return 0;
4255
4256         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4257         atsru = dmar_find_atsr(atsr);
4258         if (atsru)
4259                 return 0;
4260
4261         atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
4262         if (!atsru)
4263                 return -ENOMEM;
4264
4265         /*
4266          * If memory is allocated from slab by ACPI _DSM method, we need to
4267          * copy the memory content because the memory buffer will be freed
4268          * on return.
4269          */
4270         atsru->hdr = (void *)(atsru + 1);
4271         memcpy(atsru->hdr, hdr, hdr->length);
4272         atsru->include_all = atsr->flags & 0x1;
4273         if (!atsru->include_all) {
4274                 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4275                                 (void *)atsr + atsr->header.length,
4276                                 &atsru->devices_cnt);
4277                 if (atsru->devices_cnt && atsru->devices == NULL) {
4278                         kfree(atsru);
4279                         return -ENOMEM;
4280                 }
4281         }
4282
4283         list_add_rcu(&atsru->list, &dmar_atsr_units);
4284
4285         return 0;
4286 }
4287
4288 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4289 {
4290         dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4291         kfree(atsru);
4292 }
4293
4294 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4295 {
4296         struct acpi_dmar_atsr *atsr;
4297         struct dmar_atsr_unit *atsru;
4298
4299         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4300         atsru = dmar_find_atsr(atsr);
4301         if (atsru) {
4302                 list_del_rcu(&atsru->list);
4303                 synchronize_rcu();
4304                 intel_iommu_free_atsr(atsru);
4305         }
4306
4307         return 0;
4308 }
4309
4310 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4311 {
4312         int i;
4313         struct device *dev;
4314         struct acpi_dmar_atsr *atsr;
4315         struct dmar_atsr_unit *atsru;
4316
4317         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4318         atsru = dmar_find_atsr(atsr);
4319         if (!atsru)
4320                 return 0;
4321
4322         if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
4323                 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4324                                           i, dev)
4325                         return -EBUSY;
4326         }
4327
4328         return 0;
4329 }
4330
4331 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4332 {
4333         int sp, ret = 0;
4334         struct intel_iommu *iommu = dmaru->iommu;
4335
4336         if (g_iommus[iommu->seq_id])
4337                 return 0;
4338
4339         if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
4340                 pr_warn("%s: Doesn't support hardware pass through.\n",
4341                         iommu->name);
4342                 return -ENXIO;
4343         }
4344         if (!ecap_sc_support(iommu->ecap) &&
4345             domain_update_iommu_snooping(iommu)) {
4346                 pr_warn("%s: Doesn't support snooping.\n",
4347                         iommu->name);
4348                 return -ENXIO;
4349         }
4350         sp = domain_update_iommu_superpage(iommu) - 1;
4351         if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4352                 pr_warn("%s: Doesn't support large page.\n",
4353                         iommu->name);
4354                 return -ENXIO;
4355         }
4356
4357         /*
4358          * Disable translation if already enabled prior to OS handover.
4359          */
4360         if (iommu->gcmd & DMA_GCMD_TE)
4361                 iommu_disable_translation(iommu);
4362
4363         g_iommus[iommu->seq_id] = iommu;
4364         ret = iommu_init_domains(iommu);
4365         if (ret == 0)
4366                 ret = iommu_alloc_root_entry(iommu);
4367         if (ret)
4368                 goto out;
4369
4370 #ifdef CONFIG_INTEL_IOMMU_SVM
4371         if (pasid_enabled(iommu))
4372                 intel_svm_alloc_pasid_tables(iommu);
4373 #endif
4374
4375         if (dmaru->ignored) {
4376                 /*
4377                  * we always have to disable PMRs or DMA may fail on this device
4378                  */
4379                 if (force_on)
4380                         iommu_disable_protect_mem_regions(iommu);
4381                 return 0;
4382         }
4383
4384         intel_iommu_init_qi(iommu);
4385         iommu_flush_write_buffer(iommu);
4386
4387 #ifdef CONFIG_INTEL_IOMMU_SVM
4388         if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
4389                 ret = intel_svm_enable_prq(iommu);
4390                 if (ret)
4391                         goto disable_iommu;
4392         }
4393 #endif
4394         ret = dmar_set_interrupt(iommu);
4395         if (ret)
4396                 goto disable_iommu;
4397
4398         iommu_set_root_entry(iommu);
4399         iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4400         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4401         iommu_enable_translation(iommu);
4402
4403         iommu_disable_protect_mem_regions(iommu);
4404         return 0;
4405
4406 disable_iommu:
4407         disable_dmar_iommu(iommu);
4408 out:
4409         free_dmar_iommu(iommu);
4410         return ret;
4411 }
4412
4413 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4414 {
4415         int ret = 0;
4416         struct intel_iommu *iommu = dmaru->iommu;
4417
4418         if (!intel_iommu_enabled)
4419                 return 0;
4420         if (iommu == NULL)
4421                 return -EINVAL;
4422
4423         if (insert) {
4424                 ret = intel_iommu_add(dmaru);
4425         } else {
4426                 disable_dmar_iommu(iommu);
4427                 free_dmar_iommu(iommu);
4428         }
4429
4430         return ret;
4431 }
4432
4433 static void intel_iommu_free_dmars(void)
4434 {
4435         struct dmar_rmrr_unit *rmrru, *rmrr_n;
4436         struct dmar_atsr_unit *atsru, *atsr_n;
4437
4438         list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4439                 list_del(&rmrru->list);
4440                 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4441                 kfree(rmrru);
4442         }
4443
4444         list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4445                 list_del(&atsru->list);
4446                 intel_iommu_free_atsr(atsru);
4447         }
4448 }
4449
4450 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4451 {
4452         int i, ret = 1;
4453         struct pci_bus *bus;
4454         struct pci_dev *bridge = NULL;
4455         struct device *tmp;
4456         struct acpi_dmar_atsr *atsr;
4457         struct dmar_atsr_unit *atsru;
4458
4459         dev = pci_physfn(dev);
4460         for (bus = dev->bus; bus; bus = bus->parent) {
4461                 bridge = bus->self;
4462                 /* If it's an integrated device, allow ATS */
4463                 if (!bridge)
4464                         return 1;
4465                 /* Connected via non-PCIe: no ATS */
4466                 if (!pci_is_pcie(bridge) ||
4467                     pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
4468                         return 0;
4469                 /* If we found the root port, look it up in the ATSR */
4470                 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
4471                         break;
4472         }
4473
4474         rcu_read_lock();
4475         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4476                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4477                 if (atsr->segment != pci_domain_nr(dev->bus))
4478                         continue;
4479
4480                 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
4481                         if (tmp == &bridge->dev)
4482                                 goto out;
4483
4484                 if (atsru->include_all)
4485                         goto out;
4486         }
4487         ret = 0;
4488 out:
4489         rcu_read_unlock();
4490
4491         return ret;
4492 }
4493
4494 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4495 {
4496         int ret = 0;
4497         struct dmar_rmrr_unit *rmrru;
4498         struct dmar_atsr_unit *atsru;
4499         struct acpi_dmar_atsr *atsr;
4500         struct acpi_dmar_reserved_memory *rmrr;
4501
4502         if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
4503                 return 0;
4504
4505         list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4506                 rmrr = container_of(rmrru->hdr,
4507                                     struct acpi_dmar_reserved_memory, header);
4508                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4509                         ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4510                                 ((void *)rmrr) + rmrr->header.length,
4511                                 rmrr->segment, rmrru->devices,
4512                                 rmrru->devices_cnt);
4513                         if(ret < 0)
4514                                 return ret;
4515                 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4516                         dmar_remove_dev_scope(info, rmrr->segment,
4517                                 rmrru->devices, rmrru->devices_cnt);
4518                 }
4519         }
4520
4521         list_for_each_entry(atsru, &dmar_atsr_units, list) {
4522                 if (atsru->include_all)
4523                         continue;
4524
4525                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4526                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4527                         ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4528                                         (void *)atsr + atsr->header.length,
4529                                         atsr->segment, atsru->devices,
4530                                         atsru->devices_cnt);
4531                         if (ret > 0)
4532                                 break;
4533                         else if(ret < 0)
4534                                 return ret;
4535                 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4536                         if (dmar_remove_dev_scope(info, atsr->segment,
4537                                         atsru->devices, atsru->devices_cnt))
4538                                 break;
4539                 }
4540         }
4541
4542         return 0;
4543 }
4544
4545 /*
4546  * Here we only respond to action of unbound device from driver.
4547  *
4548  * Added device is not attached to its DMAR domain here yet. That will happen
4549  * when mapping the device to iova.
4550  */
4551 static int device_notifier(struct notifier_block *nb,
4552                                   unsigned long action, void *data)
4553 {
4554         struct device *dev = data;
4555         struct dmar_domain *domain;
4556
4557         if (iommu_dummy(dev))
4558                 return 0;
4559
4560         if (action != BUS_NOTIFY_REMOVED_DEVICE)
4561                 return 0;
4562
4563         domain = find_domain(dev);
4564         if (!domain)
4565                 return 0;
4566
4567         dmar_remove_one_dev_info(domain, dev);
4568         if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4569                 domain_exit(domain);
4570
4571         return 0;
4572 }
4573
4574 static struct notifier_block device_nb = {
4575         .notifier_call = device_notifier,
4576 };
4577
4578 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4579                                        unsigned long val, void *v)
4580 {
4581         struct memory_notify *mhp = v;
4582         unsigned long long start, end;
4583         unsigned long start_vpfn, last_vpfn;
4584
4585         switch (val) {
4586         case MEM_GOING_ONLINE:
4587                 start = mhp->start_pfn << PAGE_SHIFT;
4588                 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4589                 if (iommu_domain_identity_map(si_domain, start, end)) {
4590                         pr_warn("Failed to build identity map for [%llx-%llx]\n",
4591                                 start, end);
4592                         return NOTIFY_BAD;
4593                 }
4594                 break;
4595
4596         case MEM_OFFLINE:
4597         case MEM_CANCEL_ONLINE:
4598                 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4599                 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4600                 while (start_vpfn <= last_vpfn) {
4601                         struct iova *iova;
4602                         struct dmar_drhd_unit *drhd;
4603                         struct intel_iommu *iommu;
4604                         struct page *freelist;
4605
4606                         iova = find_iova(&si_domain->iovad, start_vpfn);
4607                         if (iova == NULL) {
4608                                 pr_debug("Failed get IOVA for PFN %lx\n",
4609                                          start_vpfn);
4610                                 break;
4611                         }
4612
4613                         iova = split_and_remove_iova(&si_domain->iovad, iova,
4614                                                      start_vpfn, last_vpfn);
4615                         if (iova == NULL) {
4616                                 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4617                                         start_vpfn, last_vpfn);
4618                                 return NOTIFY_BAD;
4619                         }
4620
4621                         freelist = domain_unmap(si_domain, iova->pfn_lo,
4622                                                iova->pfn_hi);
4623
4624                         rcu_read_lock();
4625                         for_each_active_iommu(iommu, drhd)
4626                                 iommu_flush_iotlb_psi(iommu, si_domain,
4627                                         iova->pfn_lo, iova_size(iova),
4628                                         !freelist, 0);
4629                         rcu_read_unlock();
4630                         dma_free_pagelist(freelist);
4631
4632                         start_vpfn = iova->pfn_hi + 1;
4633                         free_iova_mem(iova);
4634                 }
4635                 break;
4636         }
4637
4638         return NOTIFY_OK;
4639 }
4640
4641 static struct notifier_block intel_iommu_memory_nb = {
4642         .notifier_call = intel_iommu_memory_notifier,
4643         .priority = 0
4644 };
4645
4646 static void free_all_cpu_cached_iovas(unsigned int cpu)
4647 {
4648         int i;
4649
4650         for (i = 0; i < g_num_of_iommus; i++) {
4651                 struct intel_iommu *iommu = g_iommus[i];
4652                 struct dmar_domain *domain;
4653                 int did;
4654
4655                 if (!iommu)
4656                         continue;
4657
4658                 for (did = 0; did < cap_ndoms(iommu->cap); did++) {
4659                         domain = get_iommu_domain(iommu, (u16)did);
4660
4661                         if (!domain)
4662                                 continue;
4663                         free_cpu_cached_iovas(cpu, &domain->iovad);
4664                 }
4665         }
4666 }
4667
4668 static int intel_iommu_cpu_notifier(struct notifier_block *nfb,
4669                                     unsigned long action, void *v)
4670 {
4671         unsigned int cpu = (unsigned long)v;
4672
4673         switch (action) {
4674         case CPU_DEAD:
4675         case CPU_DEAD_FROZEN:
4676                 free_all_cpu_cached_iovas(cpu);
4677                 flush_unmaps_timeout(cpu);
4678                 break;
4679         }
4680         return NOTIFY_OK;
4681 }
4682
4683 static struct notifier_block intel_iommu_cpu_nb = {
4684         .notifier_call = intel_iommu_cpu_notifier,
4685 };
4686
4687 static ssize_t intel_iommu_show_version(struct device *dev,
4688                                         struct device_attribute *attr,
4689                                         char *buf)
4690 {
4691         struct intel_iommu *iommu = dev_get_drvdata(dev);
4692         u32 ver = readl(iommu->reg + DMAR_VER_REG);
4693         return sprintf(buf, "%d:%d\n",
4694                        DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4695 }
4696 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4697
4698 static ssize_t intel_iommu_show_address(struct device *dev,
4699                                         struct device_attribute *attr,
4700                                         char *buf)
4701 {
4702         struct intel_iommu *iommu = dev_get_drvdata(dev);
4703         return sprintf(buf, "%llx\n", iommu->reg_phys);
4704 }
4705 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4706
4707 static ssize_t intel_iommu_show_cap(struct device *dev,
4708                                     struct device_attribute *attr,
4709                                     char *buf)
4710 {
4711         struct intel_iommu *iommu = dev_get_drvdata(dev);
4712         return sprintf(buf, "%llx\n", iommu->cap);
4713 }
4714 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4715
4716 static ssize_t intel_iommu_show_ecap(struct device *dev,
4717                                     struct device_attribute *attr,
4718                                     char *buf)
4719 {
4720         struct intel_iommu *iommu = dev_get_drvdata(dev);
4721         return sprintf(buf, "%llx\n", iommu->ecap);
4722 }
4723 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4724
4725 static ssize_t intel_iommu_show_ndoms(struct device *dev,
4726                                       struct device_attribute *attr,
4727                                       char *buf)
4728 {
4729         struct intel_iommu *iommu = dev_get_drvdata(dev);
4730         return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4731 }
4732 static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4733
4734 static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4735                                            struct device_attribute *attr,
4736                                            char *buf)
4737 {
4738         struct intel_iommu *iommu = dev_get_drvdata(dev);
4739         return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4740                                                   cap_ndoms(iommu->cap)));
4741 }
4742 static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4743
4744 static struct attribute *intel_iommu_attrs[] = {
4745         &dev_attr_version.attr,
4746         &dev_attr_address.attr,
4747         &dev_attr_cap.attr,
4748         &dev_attr_ecap.attr,
4749         &dev_attr_domains_supported.attr,
4750         &dev_attr_domains_used.attr,
4751         NULL,
4752 };
4753
4754 static struct attribute_group intel_iommu_group = {
4755         .name = "intel-iommu",
4756         .attrs = intel_iommu_attrs,
4757 };
4758
4759 const struct attribute_group *intel_iommu_groups[] = {
4760         &intel_iommu_group,
4761         NULL,
4762 };
4763
4764 int __init intel_iommu_init(void)
4765 {
4766         int ret = -ENODEV;
4767         struct dmar_drhd_unit *drhd;
4768         struct intel_iommu *iommu;
4769
4770         /* VT-d is required for a TXT/tboot launch, so enforce that */
4771         force_on = tboot_force_iommu();
4772
4773         if (iommu_init_mempool()) {
4774                 if (force_on)
4775                         panic("tboot: Failed to initialize iommu memory\n");
4776                 return -ENOMEM;
4777         }
4778
4779         down_write(&dmar_global_lock);
4780         if (dmar_table_init()) {
4781                 if (force_on)
4782                         panic("tboot: Failed to initialize DMAR table\n");
4783                 goto out_free_dmar;
4784         }
4785
4786         if (dmar_dev_scope_init() < 0) {
4787                 if (force_on)
4788                         panic("tboot: Failed to initialize DMAR device scope\n");
4789                 goto out_free_dmar;
4790         }
4791
4792         if (no_iommu || dmar_disabled)
4793                 goto out_free_dmar;
4794
4795         if (list_empty(&dmar_rmrr_units))
4796                 pr_info("No RMRR found\n");
4797
4798         if (list_empty(&dmar_atsr_units))
4799                 pr_info("No ATSR found\n");
4800
4801         if (dmar_init_reserved_ranges()) {
4802                 if (force_on)
4803                         panic("tboot: Failed to reserve iommu ranges\n");
4804                 goto out_free_reserved_range;
4805         }
4806
4807         init_no_remapping_devices();
4808
4809         ret = init_dmars();
4810         if (ret) {
4811                 if (force_on)
4812                         panic("tboot: Failed to initialize DMARs\n");
4813                 pr_err("Initialization failed\n");
4814                 goto out_free_reserved_range;
4815         }
4816         up_write(&dmar_global_lock);
4817         pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4818
4819 #ifdef CONFIG_SWIOTLB
4820         swiotlb = 0;
4821 #endif
4822         dma_ops = &intel_dma_ops;
4823
4824         init_iommu_pm_ops();
4825
4826         for_each_active_iommu(iommu, drhd)
4827                 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4828                                                        intel_iommu_groups,
4829                                                        "%s", iommu->name);
4830
4831         bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4832         bus_register_notifier(&pci_bus_type, &device_nb);
4833         if (si_domain && !hw_pass_through)
4834                 register_memory_notifier(&intel_iommu_memory_nb);
4835         register_hotcpu_notifier(&intel_iommu_cpu_nb);
4836
4837         intel_iommu_enabled = 1;
4838
4839         return 0;
4840
4841 out_free_reserved_range:
4842         put_iova_domain(&reserved_iova_list);
4843 out_free_dmar:
4844         intel_iommu_free_dmars();
4845         up_write(&dmar_global_lock);
4846         iommu_exit_mempool();
4847         return ret;
4848 }
4849
4850 static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4851 {
4852         struct intel_iommu *iommu = opaque;
4853
4854         domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4855         return 0;
4856 }
4857
4858 /*
4859  * NB - intel-iommu lacks any sort of reference counting for the users of
4860  * dependent devices.  If multiple endpoints have intersecting dependent
4861  * devices, unbinding the driver from any one of them will possibly leave
4862  * the others unable to operate.
4863  */
4864 static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
4865 {
4866         if (!iommu || !dev || !dev_is_pci(dev))
4867                 return;
4868
4869         pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
4870 }
4871
4872 static void __dmar_remove_one_dev_info(struct device_domain_info *info)
4873 {
4874         struct intel_iommu *iommu;
4875         unsigned long flags;
4876
4877         assert_spin_locked(&device_domain_lock);
4878
4879         if (WARN_ON(!info))
4880                 return;
4881
4882         iommu = info->iommu;
4883
4884         if (info->dev) {
4885                 iommu_disable_dev_iotlb(info);
4886                 domain_context_clear(iommu, info->dev);
4887         }
4888
4889         unlink_domain_info(info);
4890
4891         spin_lock_irqsave(&iommu->lock, flags);
4892         domain_detach_iommu(info->domain, iommu);
4893         spin_unlock_irqrestore(&iommu->lock, flags);
4894
4895         free_devinfo_mem(info);
4896 }
4897
4898 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4899                                      struct device *dev)
4900 {
4901         struct device_domain_info *info;
4902         unsigned long flags;
4903
4904         spin_lock_irqsave(&device_domain_lock, flags);
4905         info = dev->archdata.iommu;
4906         __dmar_remove_one_dev_info(info);
4907         spin_unlock_irqrestore(&device_domain_lock, flags);
4908 }
4909
4910 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4911 {
4912         int adjust_width;
4913
4914         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4915                         DMA_32BIT_PFN);
4916         domain_reserve_special_ranges(domain);
4917
4918         /* calculate AGAW */
4919         domain->gaw = guest_width;
4920         adjust_width = guestwidth_to_adjustwidth(guest_width);
4921         domain->agaw = width_to_agaw(adjust_width);
4922
4923         domain->iommu_coherency = 0;
4924         domain->iommu_snooping = 0;
4925         domain->iommu_superpage = 0;
4926         domain->max_addr = 0;
4927
4928         /* always allocate the top pgd */
4929         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
4930         if (!domain->pgd)
4931                 return -ENOMEM;
4932         domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4933         return 0;
4934 }
4935
4936 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
4937 {
4938         struct dmar_domain *dmar_domain;
4939         struct iommu_domain *domain;
4940
4941         if (type != IOMMU_DOMAIN_UNMANAGED)
4942                 return NULL;
4943
4944         dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
4945         if (!dmar_domain) {
4946                 pr_err("Can't allocate dmar_domain\n");
4947                 return NULL;
4948         }
4949         if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
4950                 pr_err("Domain initialization failed\n");
4951                 domain_exit(dmar_domain);
4952                 return NULL;
4953         }
4954         domain_update_iommu_cap(dmar_domain);
4955
4956         domain = &dmar_domain->domain;
4957         domain->geometry.aperture_start = 0;
4958         domain->geometry.aperture_end   = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4959         domain->geometry.force_aperture = true;
4960
4961         return domain;
4962 }
4963
4964 static void intel_iommu_domain_free(struct iommu_domain *domain)
4965 {
4966         domain_exit(to_dmar_domain(domain));
4967 }
4968
4969 static int intel_iommu_attach_device(struct iommu_domain *domain,
4970                                      struct device *dev)
4971 {
4972         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4973         struct intel_iommu *iommu;
4974         int addr_width;
4975         u8 bus, devfn;
4976
4977         if (device_is_rmrr_locked(dev)) {
4978                 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement.  Contact your platform vendor.\n");
4979                 return -EPERM;
4980         }
4981
4982         /* normally dev is not mapped */
4983         if (unlikely(domain_context_mapped(dev))) {
4984                 struct dmar_domain *old_domain;
4985
4986                 old_domain = find_domain(dev);
4987                 if (old_domain) {
4988                         rcu_read_lock();
4989                         dmar_remove_one_dev_info(old_domain, dev);
4990                         rcu_read_unlock();
4991
4992                         if (!domain_type_is_vm_or_si(old_domain) &&
4993                              list_empty(&old_domain->devices))
4994                                 domain_exit(old_domain);
4995                 }
4996         }
4997
4998         iommu = device_to_iommu(dev, &bus, &devfn);
4999         if (!iommu)
5000                 return -ENODEV;
5001
5002         /* check if this iommu agaw is sufficient for max mapped address */
5003         addr_width = agaw_to_width(iommu->agaw);
5004         if (addr_width > cap_mgaw(iommu->cap))
5005                 addr_width = cap_mgaw(iommu->cap);
5006
5007         if (dmar_domain->max_addr > (1LL << addr_width)) {
5008                 pr_err("%s: iommu width (%d) is not "
5009                        "sufficient for the mapped address (%llx)\n",
5010                        __func__, addr_width, dmar_domain->max_addr);
5011                 return -EFAULT;
5012         }
5013         dmar_domain->gaw = addr_width;
5014
5015         /*
5016          * Knock out extra levels of page tables if necessary
5017          */
5018         while (iommu->agaw < dmar_domain->agaw) {
5019                 struct dma_pte *pte;
5020
5021                 pte = dmar_domain->pgd;
5022                 if (dma_pte_present(pte)) {
5023                         dmar_domain->pgd = (struct dma_pte *)
5024                                 phys_to_virt(dma_pte_addr(pte));
5025                         free_pgtable_page(pte);
5026                 }
5027                 dmar_domain->agaw--;
5028         }
5029
5030         return domain_add_dev_info(dmar_domain, dev);
5031 }
5032
5033 static void intel_iommu_detach_device(struct iommu_domain *domain,
5034                                       struct device *dev)
5035 {
5036         dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
5037 }
5038
5039 static int intel_iommu_map(struct iommu_domain *domain,
5040                            unsigned long iova, phys_addr_t hpa,
5041                            size_t size, int iommu_prot)
5042 {
5043         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5044         u64 max_addr;
5045         int prot = 0;
5046         int ret;
5047
5048         if (iommu_prot & IOMMU_READ)
5049                 prot |= DMA_PTE_READ;
5050         if (iommu_prot & IOMMU_WRITE)
5051                 prot |= DMA_PTE_WRITE;
5052         if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
5053                 prot |= DMA_PTE_SNP;
5054
5055         max_addr = iova + size;
5056         if (dmar_domain->max_addr < max_addr) {
5057                 u64 end;
5058
5059                 /* check if minimum agaw is sufficient for mapped address */
5060                 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
5061                 if (end < max_addr) {
5062                         pr_err("%s: iommu width (%d) is not "
5063                                "sufficient for the mapped address (%llx)\n",
5064                                __func__, dmar_domain->gaw, max_addr);
5065                         return -EFAULT;
5066                 }
5067                 dmar_domain->max_addr = max_addr;
5068         }
5069         /* Round up size to next multiple of PAGE_SIZE, if it and
5070            the low bits of hpa would take us onto the next page */
5071         size = aligned_nrpages(hpa, size);
5072         ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
5073                                  hpa >> VTD_PAGE_SHIFT, size, prot);
5074         return ret;
5075 }
5076
5077 static size_t intel_iommu_unmap(struct iommu_domain *domain,
5078                                 unsigned long iova, size_t size)
5079 {
5080         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5081         struct page *freelist = NULL;
5082         struct intel_iommu *iommu;
5083         unsigned long start_pfn, last_pfn;
5084         unsigned int npages;
5085         int iommu_id, level = 0;
5086
5087         /* Cope with horrid API which requires us to unmap more than the
5088            size argument if it happens to be a large-page mapping. */
5089         BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
5090
5091         if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
5092                 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
5093
5094         start_pfn = iova >> VTD_PAGE_SHIFT;
5095         last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
5096
5097         freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
5098
5099         npages = last_pfn - start_pfn + 1;
5100
5101         for_each_domain_iommu(iommu_id, dmar_domain) {
5102                 iommu = g_iommus[iommu_id];
5103
5104                 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5105                                       start_pfn, npages, !freelist, 0);
5106         }
5107
5108         dma_free_pagelist(freelist);
5109
5110         if (dmar_domain->max_addr == iova + size)
5111                 dmar_domain->max_addr = iova;
5112
5113         return size;
5114 }
5115
5116 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
5117                                             dma_addr_t iova)
5118 {
5119         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5120         struct dma_pte *pte;
5121         int level = 0;
5122         u64 phys = 0;
5123
5124         pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
5125         if (pte)
5126                 phys = dma_pte_addr(pte);
5127
5128         return phys;
5129 }
5130
5131 static bool intel_iommu_capable(enum iommu_cap cap)
5132 {
5133         if (cap == IOMMU_CAP_CACHE_COHERENCY)
5134                 return domain_update_iommu_snooping(NULL) == 1;
5135         if (cap == IOMMU_CAP_INTR_REMAP)
5136                 return irq_remapping_enabled == 1;
5137
5138         return false;
5139 }
5140
5141 static int intel_iommu_add_device(struct device *dev)
5142 {
5143         struct intel_iommu *iommu;
5144         struct iommu_group *group;
5145         u8 bus, devfn;
5146
5147         iommu = device_to_iommu(dev, &bus, &devfn);
5148         if (!iommu)
5149                 return -ENODEV;
5150
5151         iommu_device_link(iommu->iommu_dev, dev);
5152
5153         group = iommu_group_get_for_dev(dev);
5154
5155         if (IS_ERR(group))
5156                 return PTR_ERR(group);
5157
5158         iommu_group_put(group);
5159         return 0;
5160 }
5161
5162 static void intel_iommu_remove_device(struct device *dev)
5163 {
5164         struct intel_iommu *iommu;
5165         u8 bus, devfn;
5166
5167         iommu = device_to_iommu(dev, &bus, &devfn);
5168         if (!iommu)
5169                 return;
5170
5171         iommu_group_remove_device(dev);
5172
5173         iommu_device_unlink(iommu->iommu_dev, dev);
5174 }
5175
5176 #ifdef CONFIG_INTEL_IOMMU_SVM
5177 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
5178 {
5179         struct device_domain_info *info;
5180         struct context_entry *context;
5181         struct dmar_domain *domain;
5182         unsigned long flags;
5183         u64 ctx_lo;
5184         int ret;
5185
5186         domain = get_valid_domain_for_dev(sdev->dev);
5187         if (!domain)
5188                 return -EINVAL;
5189
5190         spin_lock_irqsave(&device_domain_lock, flags);
5191         spin_lock(&iommu->lock);
5192
5193         ret = -EINVAL;
5194         info = sdev->dev->archdata.iommu;
5195         if (!info || !info->pasid_supported)
5196                 goto out;
5197
5198         context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5199         if (WARN_ON(!context))
5200                 goto out;
5201
5202         ctx_lo = context[0].lo;
5203
5204         sdev->did = domain->iommu_did[iommu->seq_id];
5205         sdev->sid = PCI_DEVID(info->bus, info->devfn);
5206
5207         if (!(ctx_lo & CONTEXT_PASIDE)) {
5208                 context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
5209                 context[1].lo = (u64)virt_to_phys(iommu->pasid_table) | ecap_pss(iommu->ecap);
5210                 wmb();
5211                 /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both
5212                  * extended to permit requests-with-PASID if the PASIDE bit
5213                  * is set. which makes sense. For CONTEXT_TT_PASS_THROUGH,
5214                  * however, the PASIDE bit is ignored and requests-with-PASID
5215                  * are unconditionally blocked. Which makes less sense.
5216                  * So convert from CONTEXT_TT_PASS_THROUGH to one of the new
5217                  * "guest mode" translation types depending on whether ATS
5218                  * is available or not. Annoyingly, we can't use the new
5219                  * modes *unless* PASIDE is set. */
5220                 if ((ctx_lo & CONTEXT_TT_MASK) == (CONTEXT_TT_PASS_THROUGH << 2)) {
5221                         ctx_lo &= ~CONTEXT_TT_MASK;
5222                         if (info->ats_supported)
5223                                 ctx_lo |= CONTEXT_TT_PT_PASID_DEV_IOTLB << 2;
5224                         else
5225                                 ctx_lo |= CONTEXT_TT_PT_PASID << 2;
5226                 }
5227                 ctx_lo |= CONTEXT_PASIDE;
5228                 if (iommu->pasid_state_table)
5229                         ctx_lo |= CONTEXT_DINVE;
5230                 if (info->pri_supported)
5231                         ctx_lo |= CONTEXT_PRS;
5232                 context[0].lo = ctx_lo;
5233                 wmb();
5234                 iommu->flush.flush_context(iommu, sdev->did, sdev->sid,
5235                                            DMA_CCMD_MASK_NOBIT,
5236                                            DMA_CCMD_DEVICE_INVL);
5237         }
5238
5239         /* Enable PASID support in the device, if it wasn't already */
5240         if (!info->pasid_enabled)
5241                 iommu_enable_dev_iotlb(info);
5242
5243         if (info->ats_enabled) {
5244                 sdev->dev_iotlb = 1;
5245                 sdev->qdep = info->ats_qdep;
5246                 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
5247                         sdev->qdep = 0;
5248         }
5249         ret = 0;
5250
5251  out:
5252         spin_unlock(&iommu->lock);
5253         spin_unlock_irqrestore(&device_domain_lock, flags);
5254
5255         return ret;
5256 }
5257
5258 struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5259 {
5260         struct intel_iommu *iommu;
5261         u8 bus, devfn;
5262
5263         if (iommu_dummy(dev)) {
5264                 dev_warn(dev,
5265                          "No IOMMU translation for device; cannot enable SVM\n");
5266                 return NULL;
5267         }
5268
5269         iommu = device_to_iommu(dev, &bus, &devfn);
5270         if ((!iommu)) {
5271                 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
5272                 return NULL;
5273         }
5274
5275         if (!iommu->pasid_table) {
5276                 dev_err(dev, "PASID not enabled on IOMMU; cannot enable SVM\n");
5277                 return NULL;
5278         }
5279
5280         return iommu;
5281 }
5282 #endif /* CONFIG_INTEL_IOMMU_SVM */
5283
5284 static const struct iommu_ops intel_iommu_ops = {
5285         .capable        = intel_iommu_capable,
5286         .domain_alloc   = intel_iommu_domain_alloc,
5287         .domain_free    = intel_iommu_domain_free,
5288         .attach_dev     = intel_iommu_attach_device,
5289         .detach_dev     = intel_iommu_detach_device,
5290         .map            = intel_iommu_map,
5291         .unmap          = intel_iommu_unmap,
5292         .map_sg         = default_iommu_map_sg,
5293         .iova_to_phys   = intel_iommu_iova_to_phys,
5294         .add_device     = intel_iommu_add_device,
5295         .remove_device  = intel_iommu_remove_device,
5296         .device_group   = pci_device_group,
5297         .pgsize_bitmap  = INTEL_IOMMU_PGSIZES,
5298 };
5299
5300 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
5301 {
5302         /* G4x/GM45 integrated gfx dmar support is totally busted. */
5303         pr_info("Disabling IOMMU for graphics on this chipset\n");
5304         dmar_map_gfx = 0;
5305 }
5306
5307 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
5308 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
5309 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
5310 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
5311 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
5312 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
5313 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
5314
5315 static void quirk_iommu_rwbf(struct pci_dev *dev)
5316 {
5317         /*
5318          * Mobile 4 Series Chipset neglects to set RWBF capability,
5319          * but needs it. Same seems to hold for the desktop versions.
5320          */
5321         pr_info("Forcing write-buffer flush capability\n");
5322         rwbf_quirk = 1;
5323 }
5324
5325 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
5326 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5327 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5328 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5329 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5330 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5331 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
5332
5333 #define GGC 0x52
5334 #define GGC_MEMORY_SIZE_MASK    (0xf << 8)
5335 #define GGC_MEMORY_SIZE_NONE    (0x0 << 8)
5336 #define GGC_MEMORY_SIZE_1M      (0x1 << 8)
5337 #define GGC_MEMORY_SIZE_2M      (0x3 << 8)
5338 #define GGC_MEMORY_VT_ENABLED   (0x8 << 8)
5339 #define GGC_MEMORY_SIZE_2M_VT   (0x9 << 8)
5340 #define GGC_MEMORY_SIZE_3M_VT   (0xa << 8)
5341 #define GGC_MEMORY_SIZE_4M_VT   (0xb << 8)
5342
5343 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
5344 {
5345         unsigned short ggc;
5346
5347         if (pci_read_config_word(dev, GGC, &ggc))
5348                 return;
5349
5350         if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
5351                 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
5352                 dmar_map_gfx = 0;
5353         } else if (dmar_map_gfx) {
5354                 /* we have to ensure the gfx device is idle before we flush */
5355                 pr_info("Disabling batched IOTLB flush on Ironlake\n");
5356                 intel_iommu_strict = 1;
5357        }
5358 }
5359 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5360 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5361 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5362 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5363
5364 /* On Tylersburg chipsets, some BIOSes have been known to enable the
5365    ISOCH DMAR unit for the Azalia sound device, but not give it any
5366    TLB entries, which causes it to deadlock. Check for that.  We do
5367    this in a function called from init_dmars(), instead of in a PCI
5368    quirk, because we don't want to print the obnoxious "BIOS broken"
5369    message if VT-d is actually disabled.
5370 */
5371 static void __init check_tylersburg_isoch(void)
5372 {
5373         struct pci_dev *pdev;
5374         uint32_t vtisochctrl;
5375
5376         /* If there's no Azalia in the system anyway, forget it. */
5377         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5378         if (!pdev)
5379                 return;
5380         pci_dev_put(pdev);
5381
5382         /* System Management Registers. Might be hidden, in which case
5383            we can't do the sanity check. But that's OK, because the
5384            known-broken BIOSes _don't_ actually hide it, so far. */
5385         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5386         if (!pdev)
5387                 return;
5388
5389         if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5390                 pci_dev_put(pdev);
5391                 return;
5392         }
5393
5394         pci_dev_put(pdev);
5395
5396         /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5397         if (vtisochctrl & 1)
5398                 return;
5399
5400         /* Drop all bits other than the number of TLB entries */
5401         vtisochctrl &= 0x1c;
5402
5403         /* If we have the recommended number of TLB entries (16), fine. */
5404         if (vtisochctrl == 0x10)
5405                 return;
5406
5407         /* Zero TLB entries? You get to ride the short bus to school. */
5408         if (!vtisochctrl) {
5409                 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5410                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5411                      dmi_get_system_info(DMI_BIOS_VENDOR),
5412                      dmi_get_system_info(DMI_BIOS_VERSION),
5413                      dmi_get_system_info(DMI_PRODUCT_VERSION));
5414                 iommu_identity_mapping |= IDENTMAP_AZALIA;
5415                 return;
5416         }
5417
5418         pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
5419                vtisochctrl);
5420 }