Merge tag 'cxl-for-6.0' of git://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl

[linux.git] / drivers / iommu / intel / pasid.c

diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c
index 17cad7c1f62d6e14e21a8602b706a56811c9946a..c5e7e8b020a5727f5168d7837d2847e99a6d8b37 100644 (file)
--- a/drivers/iommu/intel/pasid.c
+++ b/drivers/iommu/intel/pasid.c
@@ -12,13 +12,13 @@
 #include <linux/bitops.h>
 #include <linux/cpufeature.h>
 #include <linux/dmar.h>
-#include <linux/intel-iommu.h>
 #include <linux/iommu.h>
 #include <linux/memory.h>
 #include <linux/pci.h>
 #include <linux/pci-ats.h>
 #include <linux/spinlock.h>
 
+#include "iommu.h"
 #include "pasid.h"
 
 /*
@@ -450,17 +450,17 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev,
        struct pasid_entry *pte;
        u16 did, pgtt;
 
+       spin_lock(&iommu->lock);
        pte = intel_pasid_get_entry(dev, pasid);
-       if (WARN_ON(!pte))
-               return;
-
-       if (!pasid_pte_is_present(pte))
+       if (WARN_ON(!pte) || !pasid_pte_is_present(pte)) {
+               spin_unlock(&iommu->lock);
                return;
+       }
 
        did = pasid_get_domain_id(pte);
        pgtt = pasid_pte_get_pgtt(pte);
-
        intel_pasid_clear_entry(dev, pasid, fault_ignore);
+       spin_unlock(&iommu->lock);
 
        if (!ecap_coherent(iommu->ecap))
                clflush_cache_range(pte, sizeof(*pte));
@@ -496,22 +496,6 @@ static void pasid_flush_caches(struct intel_iommu *iommu,
        }
 }
 
-static inline int pasid_enable_wpe(struct pasid_entry *pte)
-{
-#ifdef CONFIG_X86
-       unsigned long cr0 = read_cr0();
-
-       /* CR0.WP is normally set but just to be sure */
-       if (unlikely(!(cr0 & X86_CR0_WP))) {
-               pr_err_ratelimited("No CPU write protect!\n");
-               return -EINVAL;
-       }
-#endif
-       pasid_set_wpe(pte);
-
-       return 0;
-};
-
 /*
  * Set up the scalable mode pasid table entry for first only
  * translation type.
@@ -528,39 +512,52 @@ int intel_pasid_setup_first_level(struct intel_iommu *iommu,
                return -EINVAL;
        }
 
-       pte = intel_pasid_get_entry(dev, pasid);
-       if (WARN_ON(!pte))
+       if (flags & PASID_FLAG_SUPERVISOR_MODE) {
+#ifdef CONFIG_X86
+               unsigned long cr0 = read_cr0();
+
+               /* CR0.WP is normally set but just to be sure */
+               if (unlikely(!(cr0 & X86_CR0_WP))) {
+                       pr_err("No CPU write protect!\n");
+                       return -EINVAL;
+               }
+#endif
+               if (!ecap_srs(iommu->ecap)) {
+                       pr_err("No supervisor request support on %s\n",
+                              iommu->name);
+                       return -EINVAL;
+               }
+       }
+
+       if ((flags & PASID_FLAG_FL5LP) && !cap_5lp_support(iommu->cap)) {
+               pr_err("No 5-level paging support for first-level on %s\n",
+                      iommu->name);
                return -EINVAL;
+       }
 
-       /* Caller must ensure PASID entry is not in use. */
-       if (pasid_pte_is_present(pte))
+       spin_lock(&iommu->lock);
+       pte = intel_pasid_get_entry(dev, pasid);
+       if (!pte) {
+               spin_unlock(&iommu->lock);
+               return -ENODEV;
+       }
+
+       if (pasid_pte_is_present(pte)) {
+               spin_unlock(&iommu->lock);
                return -EBUSY;
+       }
 
        pasid_clear_entry(pte);
 
        /* Setup the first level page table pointer: */
        pasid_set_flptr(pte, (u64)__pa(pgd));
        if (flags & PASID_FLAG_SUPERVISOR_MODE) {
-               if (!ecap_srs(iommu->ecap)) {
-                       pr_err("No supervisor request support on %s\n",
-                              iommu->name);
-                       return -EINVAL;
-               }
                pasid_set_sre(pte);
-               if (pasid_enable_wpe(pte))
-                       return -EINVAL;
-
+               pasid_set_wpe(pte);
        }
 
-       if (flags & PASID_FLAG_FL5LP) {
-               if (cap_5lp_support(iommu->cap)) {
-                       pasid_set_flpm(pte, 1);
-               } else {
-                       pr_err("No 5-level paging support for first-level\n");
-                       pasid_clear_entry(pte);
-                       return -EINVAL;
-               }
-       }
+       if (flags & PASID_FLAG_FL5LP)
+               pasid_set_flpm(pte, 1);
 
        if (flags & PASID_FLAG_PAGE_SNOOP)
                pasid_set_pgsnp(pte);
@@ -572,6 +569,8 @@ int intel_pasid_setup_first_level(struct intel_iommu *iommu,
        /* Setup Present and PASID Granular Transfer Type: */
        pasid_set_translation_type(pte, PASID_ENTRY_PGTT_FL_ONLY);
        pasid_set_present(pte);
+       spin_unlock(&iommu->lock);
+
        pasid_flush_caches(iommu, pte, pasid, did);
 
        return 0;
@@ -627,17 +626,19 @@ int intel_pasid_setup_second_level(struct intel_iommu *iommu,
        }
 
        pgd_val = virt_to_phys(pgd);
-       did = domain->iommu_did[iommu->seq_id];
+       did = domain_id_iommu(domain, iommu);
 
+       spin_lock(&iommu->lock);
        pte = intel_pasid_get_entry(dev, pasid);
        if (!pte) {
-               dev_err(dev, "Failed to get pasid entry of PASID %d\n", pasid);
+               spin_unlock(&iommu->lock);
                return -ENODEV;
        }
 
-       /* Caller must ensure PASID entry is not in use. */
-       if (pasid_pte_is_present(pte))
+       if (pasid_pte_is_present(pte)) {
+               spin_unlock(&iommu->lock);
                return -EBUSY;
+       }
 
        pasid_clear_entry(pte);
        pasid_set_domain_id(pte, did);
@@ -654,6 +655,8 @@ int intel_pasid_setup_second_level(struct intel_iommu *iommu,
        if (pasid != PASID_RID2PASID)
                pasid_set_sre(pte);
        pasid_set_present(pte);
+       spin_unlock(&iommu->lock);
+
        pasid_flush_caches(iommu, pte, pasid, did);
 
        return 0;
@@ -669,15 +672,17 @@ int intel_pasid_setup_pass_through(struct intel_iommu *iommu,
        u16 did = FLPT_DEFAULT_DID;
        struct pasid_entry *pte;
 
+       spin_lock(&iommu->lock);
        pte = intel_pasid_get_entry(dev, pasid);
        if (!pte) {
-               dev_err(dev, "Failed to get pasid entry of PASID %d\n", pasid);
+               spin_unlock(&iommu->lock);
                return -ENODEV;
        }
 
-       /* Caller must ensure PASID entry is not in use. */
-       if (pasid_pte_is_present(pte))
+       if (pasid_pte_is_present(pte)) {
+               spin_unlock(&iommu->lock);
                return -EBUSY;
+       }
 
        pasid_clear_entry(pte);
        pasid_set_domain_id(pte, did);
@@ -692,6 +697,8 @@ int intel_pasid_setup_pass_through(struct intel_iommu *iommu,
         */
        pasid_set_sre(pte);
        pasid_set_present(pte);
+       spin_unlock(&iommu->lock);
+
        pasid_flush_caches(iommu, pte, pasid, did);
 
        return 0;