drivers/iommu/intel/svm.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright © 2015 Intel Corporation.
   4  *
   5  * Authors: David Woodhouse <[email protected]>
   6  */
   7
   8 #include <linux/intel-iommu.h>
   9 #include <linux/mmu_notifier.h>
  10 #include <linux/sched.h>
  11 #include <linux/sched/mm.h>
  12 #include <linux/slab.h>
  13 #include <linux/intel-svm.h>
  14 #include <linux/rculist.h>
  15 #include <linux/pci.h>
  16 #include <linux/pci-ats.h>
  17 #include <linux/dmar.h>
  18 #include <linux/interrupt.h>
  19 #include <linux/mm_types.h>
  20 #include <linux/ioasid.h>
  21 #include <asm/page.h>
  22
  23 #include "pasid.h"
  24
  25 static irqreturn_t prq_event_thread(int irq, void *d);
  26 static void intel_svm_drain_prq(struct device *dev, int pasid);
  27
  28 #define PRQ_ORDER 0
  29
  30 int intel_svm_enable_prq(struct intel_iommu *iommu)
  31 {
  32         struct page *pages;
  33         int irq, ret;
  34
  35         pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, PRQ_ORDER);
  36         if (!pages) {
  37                 pr_warn("IOMMU: %s: Failed to allocate page request queue\n",
  38                         iommu->name);
  39                 return -ENOMEM;
  40         }
  41         iommu->prq = page_address(pages);
  42
  43         irq = dmar_alloc_hwirq(DMAR_UNITS_SUPPORTED + iommu->seq_id, iommu->node, iommu);
  44         if (irq <= 0) {
  45                 pr_err("IOMMU: %s: Failed to create IRQ vector for page request queue\n",
  46                        iommu->name);
  47                 ret = -EINVAL;
  48         err:
  49                 free_pages((unsigned long)iommu->prq, PRQ_ORDER);
  50                 iommu->prq = NULL;
  51                 return ret;
  52         }
  53         iommu->pr_irq = irq;
  54
  55         snprintf(iommu->prq_name, sizeof(iommu->prq_name), "dmar%d-prq", iommu->seq_id);
  56
  57         ret = request_threaded_irq(irq, NULL, prq_event_thread, IRQF_ONESHOT,
  58                                    iommu->prq_name, iommu);
  59         if (ret) {
  60                 pr_err("IOMMU: %s: Failed to request IRQ for page request queue\n",
  61                        iommu->name);
  62                 dmar_free_hwirq(irq);
  63                 iommu->pr_irq = 0;
  64                 goto err;
  65         }
  66         dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL);
  67         dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL);
  68         dmar_writeq(iommu->reg + DMAR_PQA_REG, virt_to_phys(iommu->prq) | PRQ_ORDER);
  69
  70         init_completion(&iommu->prq_complete);
  71
  72         return 0;
  73 }
  74
  75 int intel_svm_finish_prq(struct intel_iommu *iommu)
  76 {
  77         dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL);
  78         dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL);
  79         dmar_writeq(iommu->reg + DMAR_PQA_REG, 0ULL);
  80
  81         if (iommu->pr_irq) {
  82                 free_irq(iommu->pr_irq, iommu);
  83                 dmar_free_hwirq(iommu->pr_irq);
  84                 iommu->pr_irq = 0;
  85         }
  86
  87         free_pages((unsigned long)iommu->prq, PRQ_ORDER);
  88         iommu->prq = NULL;
  89
  90         return 0;
  91 }
  92
  93 static inline bool intel_svm_capable(struct intel_iommu *iommu)
  94 {
  95         return iommu->flags & VTD_FLAG_SVM_CAPABLE;
  96 }
  97
  98 void intel_svm_check(struct intel_iommu *iommu)
  99 {
 100         if (!pasid_supported(iommu))
 101                 return;
 102
 103         if (cpu_feature_enabled(X86_FEATURE_GBPAGES) &&
 104             !cap_fl1gp_support(iommu->cap)) {
 105                 pr_err("%s SVM disabled, incompatible 1GB page capability\n",
 106                        iommu->name);
 107                 return;
 108         }
 109
 110         if (cpu_feature_enabled(X86_FEATURE_LA57) &&
 111             !cap_5lp_support(iommu->cap)) {
 112                 pr_err("%s SVM disabled, incompatible paging mode\n",
 113                        iommu->name);
 114                 return;
 115         }
 116
 117         iommu->flags |= VTD_FLAG_SVM_CAPABLE;
 118 }
 119
 120 static void intel_flush_svm_range_dev (struct intel_svm *svm, struct intel_svm_dev *sdev,
 121                                 unsigned long address, unsigned long pages, int ih)
 122 {
 123         struct qi_desc desc;
 124
 125         if (pages == -1) {
 126                 desc.qw0 = QI_EIOTLB_PASID(svm->pasid) |
 127                         QI_EIOTLB_DID(sdev->did) |
 128                         QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) |
 129                         QI_EIOTLB_TYPE;
 130                 desc.qw1 = 0;
 131         } else {
 132                 int mask = ilog2(__roundup_pow_of_two(pages));
 133
 134                 desc.qw0 = QI_EIOTLB_PASID(svm->pasid) |
 135                                 QI_EIOTLB_DID(sdev->did) |
 136                                 QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) |
 137                                 QI_EIOTLB_TYPE;
 138                 desc.qw1 = QI_EIOTLB_ADDR(address) |
 139                                 QI_EIOTLB_IH(ih) |
 140                                 QI_EIOTLB_AM(mask);
 141         }
 142         desc.qw2 = 0;
 143         desc.qw3 = 0;
 144         qi_submit_sync(svm->iommu, &desc, 1, 0);
 145
 146         if (sdev->dev_iotlb) {
 147                 desc.qw0 = QI_DEV_EIOTLB_PASID(svm->pasid) |
 148                                 QI_DEV_EIOTLB_SID(sdev->sid) |
 149                                 QI_DEV_EIOTLB_QDEP(sdev->qdep) |
 150                                 QI_DEIOTLB_TYPE;
 151                 if (pages == -1) {
 152                         desc.qw1 = QI_DEV_EIOTLB_ADDR(-1ULL >> 1) |
 153                                         QI_DEV_EIOTLB_SIZE;
 154                 } else if (pages > 1) {
 155                         /* The least significant zero bit indicates the size. So,
 156                          * for example, an "address" value of 0x12345f000 will
 157                          * flush from 0x123440000 to 0x12347ffff (256KiB). */
 158                         unsigned long last = address + ((unsigned long)(pages - 1) << VTD_PAGE_SHIFT);
 159                         unsigned long mask = __rounddown_pow_of_two(address ^ last);
 160
 161                         desc.qw1 = QI_DEV_EIOTLB_ADDR((address & ~mask) |
 162                                         (mask - 1)) | QI_DEV_EIOTLB_SIZE;
 163                 } else {
 164                         desc.qw1 = QI_DEV_EIOTLB_ADDR(address);
 165                 }
 166                 desc.qw2 = 0;
 167                 desc.qw3 = 0;
 168                 qi_submit_sync(svm->iommu, &desc, 1, 0);
 169         }
 170 }
 171
 172 static void intel_flush_svm_range(struct intel_svm *svm, unsigned long address,
 173                                 unsigned long pages, int ih)
 174 {
 175         struct intel_svm_dev *sdev;
 176
 177         rcu_read_lock();
 178         list_for_each_entry_rcu(sdev, &svm->devs, list)
 179                 intel_flush_svm_range_dev(svm, sdev, address, pages, ih);
 180         rcu_read_unlock();
 181 }
 182
 183 /* Pages have been freed at this point */
 184 static void intel_invalidate_range(struct mmu_notifier *mn,
 185                                    struct mm_struct *mm,
 186                                    unsigned long start, unsigned long end)
 187 {
 188         struct intel_svm *svm = container_of(mn, struct intel_svm, notifier);
 189
 190         intel_flush_svm_range(svm, start,
 191                               (end - start + PAGE_SIZE - 1) >> VTD_PAGE_SHIFT, 0);
 192 }
 193
 194 static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
 195 {
 196         struct intel_svm *svm = container_of(mn, struct intel_svm, notifier);
 197         struct intel_svm_dev *sdev;
 198
 199         /* This might end up being called from exit_mmap(), *before* the page
 200          * tables are cleared. And __mmu_notifier_release() will delete us from
 201          * the list of notifiers so that our invalidate_range() callback doesn't
 202          * get called when the page tables are cleared. So we need to protect
 203          * against hardware accessing those page tables.
 204          *
 205          * We do it by clearing the entry in the PASID table and then flushing
 206          * the IOTLB and the PASID table caches. This might upset hardware;
 207          * perhaps we'll want to point the PASID to a dummy PGD (like the zero
 208          * page) so that we end up taking a fault that the hardware really
 209          * *has* to handle gracefully without affecting other processes.
 210          */
 211         rcu_read_lock();
 212         list_for_each_entry_rcu(sdev, &svm->devs, list)
 213                 intel_pasid_tear_down_entry(svm->iommu, sdev->dev,
 214                                             svm->pasid, true);
 215         rcu_read_unlock();
 216
 217 }
 218
 219 static const struct mmu_notifier_ops intel_mmuops = {
 220         .release = intel_mm_release,
 221         .invalidate_range = intel_invalidate_range,
 222 };
 223
 224 static DEFINE_MUTEX(pasid_mutex);
 225 static LIST_HEAD(global_svm_list);
 226
 227 #define for_each_svm_dev(sdev, svm, d)                  \
 228         list_for_each_entry((sdev), &(svm)->devs, list) \
 229                 if ((d) != (sdev)->dev) {} else
 230
 231 static int pasid_to_svm_sdev(struct device *dev, unsigned int pasid,
 232                              struct intel_svm **rsvm,
 233                              struct intel_svm_dev **rsdev)
 234 {
 235         struct intel_svm_dev *d, *sdev = NULL;
 236         struct intel_svm *svm;
 237
 238         /* The caller should hold the pasid_mutex lock */
 239         if (WARN_ON(!mutex_is_locked(&pasid_mutex)))
 240                 return -EINVAL;
 241
 242         if (pasid == INVALID_IOASID || pasid >= PASID_MAX)
 243                 return -EINVAL;
 244
 245         svm = ioasid_find(NULL, pasid, NULL);
 246         if (IS_ERR(svm))
 247                 return PTR_ERR(svm);
 248
 249         if (!svm)
 250                 goto out;
 251
 252         /*
 253          * If we found svm for the PASID, there must be at least one device
 254          * bond.
 255          */
 256         if (WARN_ON(list_empty(&svm->devs)))
 257                 return -EINVAL;
 258
 259         rcu_read_lock();
 260         list_for_each_entry_rcu(d, &svm->devs, list) {
 261                 if (d->dev == dev) {
 262                         sdev = d;
 263                         break;
 264                 }
 265         }
 266         rcu_read_unlock();
 267
 268 out:
 269         *rsvm = svm;
 270         *rsdev = sdev;
 271
 272         return 0;
 273 }
 274
 275 int intel_svm_bind_gpasid(struct iommu_domain *domain, struct device *dev,
 276                           struct iommu_gpasid_bind_data *data)
 277 {
 278         struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL);
 279         struct intel_svm_dev *sdev = NULL;
 280         struct dmar_domain *dmar_domain;
 281         struct intel_svm *svm = NULL;
 282         int ret = 0;
 283
 284         if (WARN_ON(!iommu) || !data)
 285                 return -EINVAL;
 286
 287         if (data->format != IOMMU_PASID_FORMAT_INTEL_VTD)
 288                 return -EINVAL;
 289
 290         /* IOMMU core ensures argsz is more than the start of the union */
 291         if (data->argsz < offsetofend(struct iommu_gpasid_bind_data, vendor.vtd))
 292                 return -EINVAL;
 293
 294         /* Make sure no undefined flags are used in vendor data */
 295         if (data->vendor.vtd.flags & ~(IOMMU_SVA_VTD_GPASID_LAST - 1))
 296                 return -EINVAL;
 297
 298         if (!dev_is_pci(dev))
 299                 return -ENOTSUPP;
 300
 301         /* VT-d supports devices with full 20 bit PASIDs only */
 302         if (pci_max_pasids(to_pci_dev(dev)) != PASID_MAX)
 303                 return -EINVAL;
 304
 305         /*
 306          * We only check host PASID range, we have no knowledge to check
 307          * guest PASID range.
 308          */
 309         if (data->hpasid <= 0 || data->hpasid >= PASID_MAX)
 310                 return -EINVAL;
 311
 312         dmar_domain = to_dmar_domain(domain);
 313
 314         mutex_lock(&pasid_mutex);
 315         ret = pasid_to_svm_sdev(dev, data->hpasid, &svm, &sdev);
 316         if (ret)
 317                 goto out;
 318
 319         if (sdev) {
 320                 /*
 321                  * Do not allow multiple bindings of the same device-PASID since
 322                  * there is only one SL page tables per PASID. We may revisit
 323                  * once sharing PGD across domains are supported.
 324                  */
 325                 dev_warn_ratelimited(dev, "Already bound with PASID %u\n",
 326                                      svm->pasid);
 327                 ret = -EBUSY;
 328                 goto out;
 329         }
 330
 331         if (!svm) {
 332                 /* We come here when PASID has never been bond to a device. */
 333                 svm = kzalloc(sizeof(*svm), GFP_KERNEL);
 334                 if (!svm) {
 335                         ret = -ENOMEM;
 336                         goto out;
 337                 }
 338                 /* REVISIT: upper layer/VFIO can track host process that bind
 339                  * the PASID. ioasid_set = mm might be sufficient for vfio to
 340                  * check pasid VMM ownership. We can drop the following line
 341                  * once VFIO and IOASID set check is in place.
 342                  */
 343                 svm->mm = get_task_mm(current);
 344                 svm->pasid = data->hpasid;
 345                 if (data->flags & IOMMU_SVA_GPASID_VAL) {
 346                         svm->gpasid = data->gpasid;
 347                         svm->flags |= SVM_FLAG_GUEST_PASID;
 348                 }
 349                 ioasid_set_data(data->hpasid, svm);
 350                 INIT_LIST_HEAD_RCU(&svm->devs);
 351                 mmput(svm->mm);
 352         }
 353         sdev = kzalloc(sizeof(*sdev), GFP_KERNEL);
 354         if (!sdev) {
 355                 ret = -ENOMEM;
 356                 goto out;
 357         }
 358         sdev->dev = dev;
 359
 360         /* Only count users if device has aux domains */
 361         if (iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX))
 362                 sdev->users = 1;
 363
 364         /* Set up device context entry for PASID if not enabled already */
 365         ret = intel_iommu_enable_pasid(iommu, sdev->dev);
 366         if (ret) {
 367                 dev_err_ratelimited(dev, "Failed to enable PASID capability\n");
 368                 kfree(sdev);
 369                 goto out;
 370         }
 371
 372         /*
 373          * PASID table is per device for better security. Therefore, for
 374          * each bind of a new device even with an existing PASID, we need to
 375          * call the nested mode setup function here.
 376          */
 377         spin_lock(&iommu->lock);
 378         ret = intel_pasid_setup_nested(iommu, dev,
 379                                        (pgd_t *)(uintptr_t)data->gpgd,
 380                                        data->hpasid, &data->vendor.vtd, dmar_domain,
 381                                        data->addr_width);
 382         spin_unlock(&iommu->lock);
 383         if (ret) {
 384                 dev_err_ratelimited(dev, "Failed to set up PASID %llu in nested mode, Err %d\n",
 385                                     data->hpasid, ret);
 386                 /*
 387                  * PASID entry should be in cleared state if nested mode
 388                  * set up failed. So we only need to clear IOASID tracking
 389                  * data such that free call will succeed.
 390                  */
 391                 kfree(sdev);
 392                 goto out;
 393         }
 394
 395         svm->flags |= SVM_FLAG_GUEST_MODE;
 396
 397         init_rcu_head(&sdev->rcu);
 398         list_add_rcu(&sdev->list, &svm->devs);
 399  out:
 400         if (!IS_ERR_OR_NULL(svm) && list_empty(&svm->devs)) {
 401                 ioasid_set_data(data->hpasid, NULL);
 402                 kfree(svm);
 403         }
 404
 405         mutex_unlock(&pasid_mutex);
 406         return ret;
 407 }
 408
 409 int intel_svm_unbind_gpasid(struct device *dev, int pasid)
 410 {
 411         struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL);
 412         struct intel_svm_dev *sdev;
 413         struct intel_svm *svm;
 414         int ret;
 415
 416         if (WARN_ON(!iommu))
 417                 return -EINVAL;
 418
 419         mutex_lock(&pasid_mutex);
 420         ret = pasid_to_svm_sdev(dev, pasid, &svm, &sdev);
 421         if (ret)
 422                 goto out;
 423
 424         if (sdev) {
 425                 if (iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX))
 426                         sdev->users--;
 427                 if (!sdev->users) {
 428                         list_del_rcu(&sdev->list);
 429                         intel_pasid_tear_down_entry(iommu, dev,
 430                                                     svm->pasid, false);
 431                         intel_svm_drain_prq(dev, svm->pasid);
 432                         kfree_rcu(sdev, rcu);
 433
 434                         if (list_empty(&svm->devs)) {
 435                                 /*
 436                                  * We do not free the IOASID here in that
 437                                  * IOMMU driver did not allocate it.
 438                                  * Unlike native SVM, IOASID for guest use was
 439                                  * allocated prior to the bind call.
 440                                  * In any case, if the free call comes before
 441                                  * the unbind, IOMMU driver will get notified
 442                                  * and perform cleanup.
 443                                  */
 444                                 ioasid_set_data(pasid, NULL);
 445                                 kfree(svm);
 446                         }
 447                 }
 448         }
 449 out:
 450         mutex_unlock(&pasid_mutex);
 451         return ret;
 452 }
 453
 454 /* Caller must hold pasid_mutex, mm reference */
 455 static int
 456 intel_svm_bind_mm(struct device *dev, int flags, struct svm_dev_ops *ops,
 457                   struct mm_struct *mm, struct intel_svm_dev **sd)
 458 {
 459         struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL);
 460         struct device_domain_info *info;
 461         struct intel_svm_dev *sdev;
 462         struct intel_svm *svm = NULL;
 463         int pasid_max;
 464         int ret;
 465
 466         if (!iommu || dmar_disabled)
 467                 return -EINVAL;
 468
 469         if (!intel_svm_capable(iommu))
 470                 return -ENOTSUPP;
 471
 472         if (dev_is_pci(dev)) {
 473                 pasid_max = pci_max_pasids(to_pci_dev(dev));
 474                 if (pasid_max < 0)
 475                         return -EINVAL;
 476         } else
 477                 pasid_max = 1 << 20;
 478
 479         /* Bind supervisor PASID shuld have mm = NULL */
 480         if (flags & SVM_FLAG_SUPERVISOR_MODE) {
 481                 if (!ecap_srs(iommu->ecap) || mm) {
 482                         pr_err("Supervisor PASID with user provided mm.\n");
 483                         return -EINVAL;
 484                 }
 485         }
 486
 487         if (!(flags & SVM_FLAG_PRIVATE_PASID)) {
 488                 struct intel_svm *t;
 489
 490                 list_for_each_entry(t, &global_svm_list, list) {
 491                         if (t->mm != mm || (t->flags & SVM_FLAG_PRIVATE_PASID))
 492                                 continue;
 493
 494                         svm = t;
 495                         if (svm->pasid >= pasid_max) {
 496                                 dev_warn(dev,
 497                                          "Limited PASID width. Cannot use existing PASID %d\n",
 498                                          svm->pasid);
 499                                 ret = -ENOSPC;
 500                                 goto out;
 501                         }
 502
 503                         /* Find the matching device in svm list */
 504                         for_each_svm_dev(sdev, svm, dev) {
 505                                 if (sdev->ops != ops) {
 506                                         ret = -EBUSY;
 507                                         goto out;
 508                                 }
 509                                 sdev->users++;
 510                                 goto success;
 511                         }
 512
 513                         break;
 514                 }
 515         }
 516
 517         sdev = kzalloc(sizeof(*sdev), GFP_KERNEL);
 518         if (!sdev) {
 519                 ret = -ENOMEM;
 520                 goto out;
 521         }
 522         sdev->dev = dev;
 523
 524         ret = intel_iommu_enable_pasid(iommu, dev);
 525         if (ret) {
 526                 kfree(sdev);
 527                 goto out;
 528         }
 529
 530         info = get_domain_info(dev);
 531         sdev->did = FLPT_DEFAULT_DID;
 532         sdev->sid = PCI_DEVID(info->bus, info->devfn);
 533         if (info->ats_enabled) {
 534                 sdev->dev_iotlb = 1;
 535                 sdev->qdep = info->ats_qdep;
 536                 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
 537                         sdev->qdep = 0;
 538         }
 539
 540         /* Finish the setup now we know we're keeping it */
 541         sdev->users = 1;
 542         sdev->ops = ops;
 543         init_rcu_head(&sdev->rcu);
 544
 545         if (!svm) {
 546                 svm = kzalloc(sizeof(*svm), GFP_KERNEL);
 547                 if (!svm) {
 548                         ret = -ENOMEM;
 549                         kfree(sdev);
 550                         goto out;
 551                 }
 552                 svm->iommu = iommu;
 553
 554                 if (pasid_max > intel_pasid_max_id)
 555                         pasid_max = intel_pasid_max_id;
 556
 557                 /* Do not use PASID 0, reserved for RID to PASID */
 558                 svm->pasid = ioasid_alloc(NULL, PASID_MIN,
 559                                           pasid_max - 1, svm);
 560                 if (svm->pasid == INVALID_IOASID) {
 561                         kfree(svm);
 562                         kfree(sdev);
 563                         ret = -ENOSPC;
 564                         goto out;
 565                 }
 566                 svm->notifier.ops = &intel_mmuops;
 567                 svm->mm = mm;
 568                 svm->flags = flags;
 569                 INIT_LIST_HEAD_RCU(&svm->devs);
 570                 INIT_LIST_HEAD(&svm->list);
 571                 ret = -ENOMEM;
 572                 if (mm) {
 573                         ret = mmu_notifier_register(&svm->notifier, mm);
 574                         if (ret) {
 575                                 ioasid_free(svm->pasid);
 576                                 kfree(svm);
 577                                 kfree(sdev);
 578                                 goto out;
 579                         }
 580                 }
 581
 582                 spin_lock(&iommu->lock);
 583                 ret = intel_pasid_setup_first_level(iommu, dev,
 584                                 mm ? mm->pgd : init_mm.pgd,
 585                                 svm->pasid, FLPT_DEFAULT_DID,
 586                                 (mm ? 0 : PASID_FLAG_SUPERVISOR_MODE) |
 587                                 (cpu_feature_enabled(X86_FEATURE_LA57) ?
 588                                  PASID_FLAG_FL5LP : 0));
 589                 spin_unlock(&iommu->lock);
 590                 if (ret) {
 591                         if (mm)
 592                                 mmu_notifier_unregister(&svm->notifier, mm);
 593                         ioasid_free(svm->pasid);
 594                         kfree(svm);
 595                         kfree(sdev);
 596                         goto out;
 597                 }
 598
 599                 list_add_tail(&svm->list, &global_svm_list);
 600         } else {
 601                 /*
 602                  * Binding a new device with existing PASID, need to setup
 603                  * the PASID entry.
 604                  */
 605                 spin_lock(&iommu->lock);
 606                 ret = intel_pasid_setup_first_level(iommu, dev,
 607                                                 mm ? mm->pgd : init_mm.pgd,
 608                                                 svm->pasid, FLPT_DEFAULT_DID,
 609                                                 (mm ? 0 : PASID_FLAG_SUPERVISOR_MODE) |
 610                                                 (cpu_feature_enabled(X86_FEATURE_LA57) ?
 611                                                 PASID_FLAG_FL5LP : 0));
 612                 spin_unlock(&iommu->lock);
 613                 if (ret) {
 614                         kfree(sdev);
 615                         goto out;
 616                 }
 617         }
 618         list_add_rcu(&sdev->list, &svm->devs);
 619 success:
 620         sdev->pasid = svm->pasid;
 621         sdev->sva.dev = dev;
 622         if (sd)
 623                 *sd = sdev;
 624         ret = 0;
 625 out:
 626         return ret;
 627 }
 628
 629 /* Caller must hold pasid_mutex */
 630 static int intel_svm_unbind_mm(struct device *dev, int pasid)
 631 {
 632         struct intel_svm_dev *sdev;
 633         struct intel_iommu *iommu;
 634         struct intel_svm *svm;
 635         int ret = -EINVAL;
 636
 637         iommu = device_to_iommu(dev, NULL, NULL);
 638         if (!iommu)
 639                 goto out;
 640
 641         ret = pasid_to_svm_sdev(dev, pasid, &svm, &sdev);
 642         if (ret)
 643                 goto out;
 644
 645         if (sdev) {
 646                 sdev->users--;
 647                 if (!sdev->users) {
 648                         list_del_rcu(&sdev->list);
 649                         /* Flush the PASID cache and IOTLB for this device.
 650                          * Note that we do depend on the hardware *not* using
 651                          * the PASID any more. Just as we depend on other
 652                          * devices never using PASIDs that they have no right
 653                          * to use. We have a *shared* PASID table, because it's
 654                          * large and has to be physically contiguous. So it's
 655                          * hard to be as defensive as we might like. */
 656                         intel_pasid_tear_down_entry(iommu, dev,
 657                                                     svm->pasid, false);
 658                         intel_svm_drain_prq(dev, svm->pasid);
 659                         kfree_rcu(sdev, rcu);
 660
 661                         if (list_empty(&svm->devs)) {
 662                                 ioasid_free(svm->pasid);
 663                                 if (svm->mm)
 664                                         mmu_notifier_unregister(&svm->notifier, svm->mm);
 665                                 list_del(&svm->list);
 666                                 /* We mandate that no page faults may be outstanding
 667                                  * for the PASID when intel_svm_unbind_mm() is called.
 668                                  * If that is not obeyed, subtle errors will happen.
 669                                  * Let's make them less subtle... */
 670                                 memset(svm, 0x6b, sizeof(*svm));
 671                                 kfree(svm);
 672                         }
 673                 }
 674         }
 675 out:
 676         return ret;
 677 }
 678
 679 /* Page request queue descriptor */
 680 struct page_req_dsc {
 681         union {
 682                 struct {
 683                         u64 type:8;
 684                         u64 pasid_present:1;
 685                         u64 priv_data_present:1;
 686                         u64 rsvd:6;
 687                         u64 rid:16;
 688                         u64 pasid:20;
 689                         u64 exe_req:1;
 690                         u64 pm_req:1;
 691                         u64 rsvd2:10;
 692                 };
 693                 u64 qw_0;
 694         };
 695         union {
 696                 struct {
 697                         u64 rd_req:1;
 698                         u64 wr_req:1;
 699                         u64 lpig:1;
 700                         u64 prg_index:9;
 701                         u64 addr:52;
 702                 };
 703                 u64 qw_1;
 704         };
 705         u64 priv_data[2];
 706 };
 707
 708 #define PRQ_RING_MASK   ((0x1000 << PRQ_ORDER) - 0x20)
 709
 710 static bool access_error(struct vm_area_struct *vma, struct page_req_dsc *req)
 711 {
 712         unsigned long requested = 0;
 713
 714         if (req->exe_req)
 715                 requested |= VM_EXEC;
 716
 717         if (req->rd_req)
 718                 requested |= VM_READ;
 719
 720         if (req->wr_req)
 721                 requested |= VM_WRITE;
 722
 723         return (requested & ~vma->vm_flags) != 0;
 724 }
 725
 726 static bool is_canonical_address(u64 addr)
 727 {
 728         int shift = 64 - (__VIRTUAL_MASK_SHIFT + 1);
 729         long saddr = (long) addr;
 730
 731         return (((saddr << shift) >> shift) == saddr);
 732 }
 733
 734 /**
 735  * intel_svm_drain_prq - Drain page requests and responses for a pasid
 736  * @dev: target device
 737  * @pasid: pasid for draining
 738  *
 739  * Drain all pending page requests and responses related to @pasid in both
 740  * software and hardware. This is supposed to be called after the device
 741  * driver has stopped DMA, the pasid entry has been cleared, and both IOTLB
 742  * and DevTLB have been invalidated.
 743  *
 744  * It waits until all pending page requests for @pasid in the page fault
 745  * queue are completed by the prq handling thread. Then follow the steps
 746  * described in VT-d spec CH7.10 to drain all page requests and page
 747  * responses pending in the hardware.
 748  */
 749 static void intel_svm_drain_prq(struct device *dev, int pasid)
 750 {
 751         struct device_domain_info *info;
 752         struct dmar_domain *domain;
 753         struct intel_iommu *iommu;
 754         struct qi_desc desc[3];
 755         struct pci_dev *pdev;
 756         int head, tail;
 757         u16 sid, did;
 758         int qdep;
 759
 760         info = get_domain_info(dev);
 761         if (WARN_ON(!info || !dev_is_pci(dev)))
 762                 return;
 763
 764         if (!info->pri_enabled)
 765                 return;
 766
 767         iommu = info->iommu;
 768         domain = info->domain;
 769         pdev = to_pci_dev(dev);
 770         sid = PCI_DEVID(info->bus, info->devfn);
 771         did = domain->iommu_did[iommu->seq_id];
 772         qdep = pci_ats_queue_depth(pdev);
 773
 774         /*
 775          * Check and wait until all pending page requests in the queue are
 776          * handled by the prq handling thread.
 777          */
 778 prq_retry:
 779         reinit_completion(&iommu->prq_complete);
 780         tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK;
 781         head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK;
 782         while (head != tail) {
 783                 struct page_req_dsc *req;
 784
 785                 req = &iommu->prq[head / sizeof(*req)];
 786                 if (!req->pasid_present || req->pasid != pasid) {
 787                         head = (head + sizeof(*req)) & PRQ_RING_MASK;
 788                         continue;
 789                 }
 790
 791                 wait_for_completion(&iommu->prq_complete);
 792                 goto prq_retry;
 793         }
 794
 795         /*
 796          * Perform steps described in VT-d spec CH7.10 to drain page
 797          * requests and responses in hardware.
 798          */
 799         memset(desc, 0, sizeof(desc));
 800         desc[0].qw0 = QI_IWD_STATUS_DATA(QI_DONE) |
 801                         QI_IWD_FENCE |
 802                         QI_IWD_TYPE;
 803         desc[1].qw0 = QI_EIOTLB_PASID(pasid) |
 804                         QI_EIOTLB_DID(did) |
 805                         QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) |
 806                         QI_EIOTLB_TYPE;
 807         desc[2].qw0 = QI_DEV_EIOTLB_PASID(pasid) |
 808                         QI_DEV_EIOTLB_SID(sid) |
 809                         QI_DEV_EIOTLB_QDEP(qdep) |
 810                         QI_DEIOTLB_TYPE |
 811                         QI_DEV_IOTLB_PFSID(info->pfsid);
 812 qi_retry:
 813         reinit_completion(&iommu->prq_complete);
 814         qi_submit_sync(iommu, desc, 3, QI_OPT_WAIT_DRAIN);
 815         if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) {
 816                 wait_for_completion(&iommu->prq_complete);
 817                 goto qi_retry;
 818         }
 819 }
 820
 821 static int prq_to_iommu_prot(struct page_req_dsc *req)
 822 {
 823         int prot = 0;
 824
 825         if (req->rd_req)
 826                 prot |= IOMMU_FAULT_PERM_READ;
 827         if (req->wr_req)
 828                 prot |= IOMMU_FAULT_PERM_WRITE;
 829         if (req->exe_req)
 830                 prot |= IOMMU_FAULT_PERM_EXEC;
 831         if (req->pm_req)
 832                 prot |= IOMMU_FAULT_PERM_PRIV;
 833
 834         return prot;
 835 }
 836
 837 static int
 838 intel_svm_prq_report(struct device *dev, struct page_req_dsc *desc)
 839 {
 840         struct iommu_fault_event event;
 841
 842         if (!dev || !dev_is_pci(dev))
 843                 return -ENODEV;
 844
 845         /* Fill in event data for device specific processing */
 846         memset(&event, 0, sizeof(struct iommu_fault_event));
 847         event.fault.type = IOMMU_FAULT_PAGE_REQ;
 848         event.fault.prm.addr = desc->addr;
 849         event.fault.prm.pasid = desc->pasid;
 850         event.fault.prm.grpid = desc->prg_index;
 851         event.fault.prm.perm = prq_to_iommu_prot(desc);
 852
 853         if (desc->lpig)
 854                 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE;
 855         if (desc->pasid_present) {
 856                 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID;
 857                 event.fault.prm.flags |= IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID;
 858         }
 859         if (desc->priv_data_present) {
 860                 /*
 861                  * Set last page in group bit if private data is present,
 862                  * page response is required as it does for LPIG.
 863                  * iommu_report_device_fault() doesn't understand this vendor
 864                  * specific requirement thus we set last_page as a workaround.
 865                  */
 866                 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE;
 867                 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA;
 868                 memcpy(event.fault.prm.private_data, desc->priv_data,
 869                        sizeof(desc->priv_data));
 870         }
 871
 872         return iommu_report_device_fault(dev, &event);
 873 }
 874
 875 static irqreturn_t prq_event_thread(int irq, void *d)
 876 {
 877         struct intel_svm_dev *sdev = NULL;
 878         struct intel_iommu *iommu = d;
 879         struct intel_svm *svm = NULL;
 880         int head, tail, handled = 0;
 881
 882         /* Clear PPR bit before reading head/tail registers, to
 883          * ensure that we get a new interrupt if needed. */
 884         writel(DMA_PRS_PPR, iommu->reg + DMAR_PRS_REG);
 885
 886         tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK;
 887         head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK;
 888         while (head != tail) {
 889                 struct vm_area_struct *vma;
 890                 struct page_req_dsc *req;
 891                 struct qi_desc resp;
 892                 int result;
 893                 vm_fault_t ret;
 894                 u64 address;
 895
 896                 handled = 1;
 897
 898                 req = &iommu->prq[head / sizeof(*req)];
 899
 900                 result = QI_RESP_FAILURE;
 901                 address = (u64)req->addr << VTD_PAGE_SHIFT;
 902                 if (!req->pasid_present) {
 903                         pr_err("%s: Page request without PASID: %08llx %08llx\n",
 904                                iommu->name, ((unsigned long long *)req)[0],
 905                                ((unsigned long long *)req)[1]);
 906                         goto no_pasid;
 907                 }
 908
 909                 if (!svm || svm->pasid != req->pasid) {
 910                         rcu_read_lock();
 911                         svm = ioasid_find(NULL, req->pasid, NULL);
 912                         /* It *can't* go away, because the driver is not permitted
 913                          * to unbind the mm while any page faults are outstanding.
 914                          * So we only need RCU to protect the internal idr code. */
 915                         rcu_read_unlock();
 916                         if (IS_ERR_OR_NULL(svm)) {
 917                                 pr_err("%s: Page request for invalid PASID %d: %08llx %08llx\n",
 918                                        iommu->name, req->pasid, ((unsigned long long *)req)[0],
 919                                        ((unsigned long long *)req)[1]);
 920                                 goto no_pasid;
 921                         }
 922                 }
 923
 924                 if (!sdev || sdev->sid != req->rid) {
 925                         struct intel_svm_dev *t;
 926
 927                         sdev = NULL;
 928                         rcu_read_lock();
 929                         list_for_each_entry_rcu(t, &svm->devs, list) {
 930                                 if (t->sid == req->rid) {
 931                                         sdev = t;
 932                                         break;
 933                                 }
 934                         }
 935                         rcu_read_unlock();
 936                 }
 937
 938                 result = QI_RESP_INVALID;
 939                 /* Since we're using init_mm.pgd directly, we should never take
 940                  * any faults on kernel addresses. */
 941                 if (!svm->mm)
 942                         goto bad_req;
 943
 944                 /* If address is not canonical, return invalid response */
 945                 if (!is_canonical_address(address))
 946                         goto bad_req;
 947
 948                 /*
 949                  * If prq is to be handled outside iommu driver via receiver of
 950                  * the fault notifiers, we skip the page response here.
 951                  */
 952                 if (svm->flags & SVM_FLAG_GUEST_MODE) {
 953                         if (sdev && !intel_svm_prq_report(sdev->dev, req))
 954                                 goto prq_advance;
 955                         else
 956                                 goto bad_req;
 957                 }
 958
 959                 /* If the mm is already defunct, don't handle faults. */
 960                 if (!mmget_not_zero(svm->mm))
 961                         goto bad_req;
 962
 963                 mmap_read_lock(svm->mm);
 964                 vma = find_extend_vma(svm->mm, address);
 965                 if (!vma || address < vma->vm_start)
 966                         goto invalid;
 967
 968                 if (access_error(vma, req))
 969                         goto invalid;
 970
 971                 ret = handle_mm_fault(vma, address,
 972                                       req->wr_req ? FAULT_FLAG_WRITE : 0,
 973                                       NULL);
 974                 if (ret & VM_FAULT_ERROR)
 975                         goto invalid;
 976
 977                 result = QI_RESP_SUCCESS;
 978 invalid:
 979                 mmap_read_unlock(svm->mm);
 980                 mmput(svm->mm);
 981 bad_req:
 982                 WARN_ON(!sdev);
 983                 if (sdev && sdev->ops && sdev->ops->fault_cb) {
 984                         int rwxp = (req->rd_req << 3) | (req->wr_req << 2) |
 985                                 (req->exe_req << 1) | (req->pm_req);
 986                         sdev->ops->fault_cb(sdev->dev, req->pasid, req->addr,
 987                                             req->priv_data, rwxp, result);
 988                 }
 989                 /* We get here in the error case where the PASID lookup failed,
 990                    and these can be NULL. Do not use them below this point! */
 991                 sdev = NULL;
 992                 svm = NULL;
 993 no_pasid:
 994                 if (req->lpig || req->priv_data_present) {
 995                         /*
 996                          * Per VT-d spec. v3.0 ch7.7, system software must
 997                          * respond with page group response if private data
 998                          * is present (PDP) or last page in group (LPIG) bit
 999                          * is set. This is an additional VT-d feature beyond
1000                          * PCI ATS spec.
1001                          */
1002                         resp.qw0 = QI_PGRP_PASID(req->pasid) |
1003                                 QI_PGRP_DID(req->rid) |
1004                                 QI_PGRP_PASID_P(req->pasid_present) |
1005                                 QI_PGRP_PDP(req->pasid_present) |
1006                                 QI_PGRP_RESP_CODE(result) |
1007                                 QI_PGRP_RESP_TYPE;
1008                         resp.qw1 = QI_PGRP_IDX(req->prg_index) |
1009                                 QI_PGRP_LPIG(req->lpig);
1010
1011                         if (req->priv_data_present)
1012                                 memcpy(&resp.qw2, req->priv_data,
1013                                        sizeof(req->priv_data));
1014                         resp.qw2 = 0;
1015                         resp.qw3 = 0;
1016                         qi_submit_sync(iommu, &resp, 1, 0);
1017                 }
1018 prq_advance:
1019                 head = (head + sizeof(*req)) & PRQ_RING_MASK;
1020         }
1021
1022         dmar_writeq(iommu->reg + DMAR_PQH_REG, tail);
1023
1024         /*
1025          * Clear the page request overflow bit and wake up all threads that
1026          * are waiting for the completion of this handling.
1027          */
1028         if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO)
1029                 writel(DMA_PRS_PRO, iommu->reg + DMAR_PRS_REG);
1030
1031         if (!completion_done(&iommu->prq_complete))
1032                 complete(&iommu->prq_complete);
1033
1034         return IRQ_RETVAL(handled);
1035 }
1036
1037 #define to_intel_svm_dev(handle) container_of(handle, struct intel_svm_dev, sva)
1038 struct iommu_sva *
1039 intel_svm_bind(struct device *dev, struct mm_struct *mm, void *drvdata)
1040 {
1041         struct iommu_sva *sva = ERR_PTR(-EINVAL);
1042         struct intel_svm_dev *sdev = NULL;
1043         int flags = 0;
1044         int ret;
1045
1046         /*
1047          * TODO: Consolidate with generic iommu-sva bind after it is merged.
1048          * It will require shared SVM data structures, i.e. combine io_mm
1049          * and intel_svm etc.
1050          */
1051         if (drvdata)
1052                 flags = *(int *)drvdata;
1053         mutex_lock(&pasid_mutex);
1054         ret = intel_svm_bind_mm(dev, flags, NULL, mm, &sdev);
1055         if (ret)
1056                 sva = ERR_PTR(ret);
1057         else if (sdev)
1058                 sva = &sdev->sva;
1059         else
1060                 WARN(!sdev, "SVM bind succeeded with no sdev!\n");
1061
1062         mutex_unlock(&pasid_mutex);
1063
1064         return sva;
1065 }
1066
1067 void intel_svm_unbind(struct iommu_sva *sva)
1068 {
1069         struct intel_svm_dev *sdev;
1070
1071         mutex_lock(&pasid_mutex);
1072         sdev = to_intel_svm_dev(sva);
1073         intel_svm_unbind_mm(sdev->dev, sdev->pasid);
1074         mutex_unlock(&pasid_mutex);
1075 }
1076
1077 int intel_svm_get_pasid(struct iommu_sva *sva)
1078 {
1079         struct intel_svm_dev *sdev;
1080         int pasid;
1081
1082         mutex_lock(&pasid_mutex);
1083         sdev = to_intel_svm_dev(sva);
1084         pasid = sdev->pasid;
1085         mutex_unlock(&pasid_mutex);
1086
1087         return pasid;
1088 }
1089
1090 int intel_svm_page_response(struct device *dev,
1091                             struct iommu_fault_event *evt,
1092                             struct iommu_page_response *msg)
1093 {
1094         struct iommu_fault_page_request *prm;
1095         struct intel_svm_dev *sdev = NULL;
1096         struct intel_svm *svm = NULL;
1097         struct intel_iommu *iommu;
1098         bool private_present;
1099         bool pasid_present;
1100         bool last_page;
1101         u8 bus, devfn;
1102         int ret = 0;
1103         u16 sid;
1104
1105         if (!dev || !dev_is_pci(dev))
1106                 return -ENODEV;
1107
1108         iommu = device_to_iommu(dev, &bus, &devfn);
1109         if (!iommu)
1110                 return -ENODEV;
1111
1112         if (!msg || !evt)
1113                 return -EINVAL;
1114
1115         mutex_lock(&pasid_mutex);
1116
1117         prm = &evt->fault.prm;
1118         sid = PCI_DEVID(bus, devfn);
1119         pasid_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID;
1120         private_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA;
1121         last_page = prm->flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE;
1122
1123         if (!pasid_present) {
1124                 ret = -EINVAL;
1125                 goto out;
1126         }
1127
1128         if (prm->pasid == 0 || prm->pasid >= PASID_MAX) {
1129                 ret = -EINVAL;
1130                 goto out;
1131         }
1132
1133         ret = pasid_to_svm_sdev(dev, prm->pasid, &svm, &sdev);
1134         if (ret || !sdev) {
1135                 ret = -ENODEV;
1136                 goto out;
1137         }
1138
1139         /*
1140          * For responses from userspace, need to make sure that the
1141          * pasid has been bound to its mm.
1142          */
1143         if (svm->flags & SVM_FLAG_GUEST_MODE) {
1144                 struct mm_struct *mm;
1145
1146                 mm = get_task_mm(current);
1147                 if (!mm) {
1148                         ret = -EINVAL;
1149                         goto out;
1150                 }
1151
1152                 if (mm != svm->mm) {
1153                         ret = -ENODEV;
1154                         mmput(mm);
1155                         goto out;
1156                 }
1157
1158                 mmput(mm);
1159         }
1160
1161         /*
1162          * Per VT-d spec. v3.0 ch7.7, system software must respond
1163          * with page group response if private data is present (PDP)
1164          * or last page in group (LPIG) bit is set. This is an
1165          * additional VT-d requirement beyond PCI ATS spec.
1166          */
1167         if (last_page || private_present) {
1168                 struct qi_desc desc;
1169
1170                 desc.qw0 = QI_PGRP_PASID(prm->pasid) | QI_PGRP_DID(sid) |
1171                                 QI_PGRP_PASID_P(pasid_present) |
1172                                 QI_PGRP_PDP(private_present) |
1173                                 QI_PGRP_RESP_CODE(msg->code) |
1174                                 QI_PGRP_RESP_TYPE;
1175                 desc.qw1 = QI_PGRP_IDX(prm->grpid) | QI_PGRP_LPIG(last_page);
1176                 desc.qw2 = 0;
1177                 desc.qw3 = 0;
1178                 if (private_present)
1179                         memcpy(&desc.qw2, prm->private_data,
1180                                sizeof(prm->private_data));
1181
1182                 qi_submit_sync(iommu, &desc, 1, 0);
1183         }
1184 out:
1185         mutex_unlock(&pasid_mutex);
1186         return ret;
1187 }