source "drivers/iommu/amd/Kconfig"
source "drivers/iommu/intel/Kconfig"
source "drivers/iommu/iommufd/Kconfig"
+++source "drivers/iommu/riscv/Kconfig"
config IRQ_REMAP
bool "Support for Interrupt Remapping"
Say Y here if your system supports SVA extensions such as PCIe PASID
and PRI.
+++ config ARM_SMMU_V3_IOMMUFD
+++ bool "Enable IOMMUFD features for ARM SMMUv3 (EXPERIMENTAL)"
+++ depends on IOMMUFD
+++ help
+++ Support for IOMMUFD features intended to support virtual machines
+++ with accelerated virtual IOMMUs.
+++
+++ Say Y here if you are doing development and testing on this feature.
+++
config ARM_SMMU_V3_KUNIT_TEST
tristate "KUnit tests for arm-smmu-v3 driver" if !KUNIT_ALL_TESTS
depends on KUNIT
extern unsigned long amd_iommu_pgsize_bitmap;
/* Protection domain ops */
+ +void amd_iommu_init_identity_domain(void);
struct protection_domain *protection_domain_alloc(unsigned int type, int nid);
void protection_domain_free(struct protection_domain *domain);
struct iommu_domain *amd_iommu_domain_alloc_sva(struct device *dev,
struct mm_struct *mm);
void amd_iommu_domain_free(struct iommu_domain *dom);
int iommu_sva_set_dev_pasid(struct iommu_domain *domain,
- -- struct device *dev, ioasid_t pasid);
+ ++ struct device *dev, ioasid_t pasid,
+ ++ struct iommu_domain *old);
void amd_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid,
struct iommu_domain *domain);
return (amd_iommu_efr2 & mask);
}
+ +static inline bool amd_iommu_v2_pgtbl_supported(void)
+ +{
+ + return (check_feature(FEATURE_GIOSUP) && check_feature(FEATURE_GT));
+ +}
+ +
static inline bool amd_iommu_gt_ppr_supported(void)
{
- - return (check_feature(FEATURE_GT) &&
+ + return (amd_iommu_v2_pgtbl_supported() &&
check_feature(FEATURE_PPR) &&
check_feature(FEATURE_EPHSUP));
}
cd_table->s1fmt = STRTAB_STE_0_S1FMT_LINEAR;
cd_table->linear.num_ents = max_contexts;
--- l1size = max_contexts * sizeof(struct arm_smmu_cd),
+++ l1size = max_contexts * sizeof(struct arm_smmu_cd);
cd_table->linear.table = dma_alloc_coherent(smmu->dev, l1size,
&cd_table->cdtab_dma,
GFP_KERNEL);
}
}
--- VISIBLE_IF_KUNIT
void arm_smmu_make_abort_ste(struct arm_smmu_ste *target)
{
memset(target, 0, sizeof(*target));
}
EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_cdtable_ste);
--- VISIBLE_IF_KUNIT
void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target,
struct arm_smmu_master *master,
struct arm_smmu_domain *smmu_domain,
case IOMMU_CAP_CACHE_COHERENCY:
/* Assume that a coherent TCU implies coherent TBUs */
return master->smmu->features & ARM_SMMU_FEAT_COHERENCY;
+++ case IOMMU_CAP_ENFORCE_CACHE_COHERENCY:
+++ return arm_smmu_master_canwbs(master);
case IOMMU_CAP_NOEXEC:
case IOMMU_CAP_DEFERRED_FLUSH:
return true;
}
}
+++ static bool arm_smmu_enforce_cache_coherency(struct iommu_domain *domain)
+++ {
+++ struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+++ struct arm_smmu_master_domain *master_domain;
+++ unsigned long flags;
+++ bool ret = true;
+++
+++ spin_lock_irqsave(&smmu_domain->devices_lock, flags);
+++ list_for_each_entry(master_domain, &smmu_domain->devices,
+++ devices_elm) {
+++ if (!arm_smmu_master_canwbs(master_domain->master)) {
+++ ret = false;
+++ break;
+++ }
+++ }
+++ smmu_domain->enforce_cache_coherency = ret;
+++ spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
+++ return ret;
+++ }
+++
struct arm_smmu_domain *arm_smmu_domain_alloc(void)
{
struct arm_smmu_domain *smmu_domain;
}
}
--- static void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master,
--- const struct arm_smmu_ste *target)
+++ void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master,
+++ const struct arm_smmu_ste *target)
{
int i, j;
struct arm_smmu_device *smmu = master->smmu;
spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
}
--- struct arm_smmu_attach_state {
--- /* Inputs */
--- struct iommu_domain *old_domain;
--- struct arm_smmu_master *master;
--- bool cd_needs_ats;
--- ioasid_t ssid;
--- /* Resulting state */
--- bool ats_enabled;
--- };
---
/*
* Start the sequence to attach a domain to a master. The sequence contains three
* steps:
* new_domain can be a non-paging domain. In this case ATS will not be enabled,
* and invalidations won't be tracked.
*/
--- static int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state,
--- struct iommu_domain *new_domain)
+++ int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state,
+++ struct iommu_domain *new_domain)
{
struct arm_smmu_master *master = state->master;
struct arm_smmu_master_domain *master_domain;
* one of them.
*/
spin_lock_irqsave(&smmu_domain->devices_lock, flags);
+++ if (smmu_domain->enforce_cache_coherency &&
+++ !arm_smmu_master_canwbs(master)) {
+++ spin_unlock_irqrestore(&smmu_domain->devices_lock,
+++ flags);
+++ kfree(master_domain);
+++ return -EINVAL;
+++ }
+++
if (state->ats_enabled)
atomic_inc(&smmu_domain->nr_ats_masters);
list_add(&master_domain->devices_elm, &smmu_domain->devices);
* completes synchronizing the PCI device's ATC and finishes manipulating the
* smmu_domain->devices list.
*/
--- static void arm_smmu_attach_commit(struct arm_smmu_attach_state *state)
+++ void arm_smmu_attach_commit(struct arm_smmu_attach_state *state)
{
struct arm_smmu_master *master = state->master;
}
static int arm_smmu_s1_set_dev_pasid(struct iommu_domain *domain,
- -- struct device *dev, ioasid_t id)
+ ++ struct device *dev, ioasid_t id,
+ ++ struct iommu_domain *old)
{
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
struct arm_smmu_master *master = dev_iommu_priv_get(dev);
*/
arm_smmu_make_s1_cd(&target_cd, master, smmu_domain);
return arm_smmu_set_pasid(master, to_smmu_domain(domain), id,
- -- &target_cd);
+ ++ &target_cd, old);
}
static void arm_smmu_update_ste(struct arm_smmu_master *master,
int arm_smmu_set_pasid(struct arm_smmu_master *master,
struct arm_smmu_domain *smmu_domain, ioasid_t pasid,
- -- struct arm_smmu_cd *cd)
+ ++ struct arm_smmu_cd *cd, struct iommu_domain *old)
{
struct iommu_domain *sid_domain = iommu_get_domain_for_dev(master->dev);
struct arm_smmu_attach_state state = {
.master = master,
- -- /*
- -- * For now the core code prevents calling this when a domain is
- -- * already attached, no need to set old_domain.
- -- */
.ssid = pasid,
+ ++ .old_domain = old,
};
struct arm_smmu_cd *cdptr;
int ret;
const struct iommu_user_data *user_data)
{
struct arm_smmu_master *master = dev_iommu_priv_get(dev);
- const u32 PAGING_FLAGS = IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
+ const u32 PAGING_FLAGS = IOMMU_HWPT_ALLOC_DIRTY_TRACKING |
- - IOMMU_HWPT_ALLOC_PASID;
++++ IOMMU_HWPT_ALLOC_PASID |
+++ IOMMU_HWPT_ALLOC_NEST_PARENT;
struct arm_smmu_domain *smmu_domain;
int ret;
if (parent || user_data)
return ERR_PTR(-EOPNOTSUPP);
+ + if (flags & IOMMU_HWPT_ALLOC_PASID)
+ + return arm_smmu_domain_alloc_paging(dev);
+ +
smmu_domain = arm_smmu_domain_alloc();
if (IS_ERR(smmu_domain))
return ERR_CAST(smmu_domain);
+++ if (flags & IOMMU_HWPT_ALLOC_NEST_PARENT) {
+++ if (!(master->smmu->features & ARM_SMMU_FEAT_NESTING)) {
+++ ret = -EOPNOTSUPP;
+++ goto err_free;
+++ }
+++ smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
+++ }
+++
smmu_domain->domain.type = IOMMU_DOMAIN_UNMANAGED;
smmu_domain->domain.ops = arm_smmu_ops.default_domain_ops;
ret = arm_smmu_domain_finalise(smmu_domain, master->smmu, flags);
return group;
}
--- static int arm_smmu_enable_nesting(struct iommu_domain *domain)
--- {
--- struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
--- int ret = 0;
---
--- mutex_lock(&smmu_domain->init_mutex);
--- if (smmu_domain->smmu)
--- ret = -EPERM;
--- else
--- smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
--- mutex_unlock(&smmu_domain->init_mutex);
---
--- return ret;
--- }
---
static int arm_smmu_of_xlate(struct device *dev,
const struct of_phandle_args *args)
{
.identity_domain = &arm_smmu_identity_domain,
.blocked_domain = &arm_smmu_blocked_domain,
.capable = arm_smmu_capable,
+++ .hw_info = arm_smmu_hw_info,
.domain_alloc_paging = arm_smmu_domain_alloc_paging,
.domain_alloc_sva = arm_smmu_sva_domain_alloc,
.domain_alloc_user = arm_smmu_domain_alloc_user,
.owner = THIS_MODULE,
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = arm_smmu_attach_dev,
+++ .enforce_cache_coherency = arm_smmu_enforce_cache_coherency,
.set_dev_pasid = arm_smmu_s1_set_dev_pasid,
.map_pages = arm_smmu_map_pages,
.unmap_pages = arm_smmu_unmap_pages,
.flush_iotlb_all = arm_smmu_flush_iotlb_all,
.iotlb_sync = arm_smmu_iotlb_sync,
.iova_to_phys = arm_smmu_iova_to_phys,
--- .enable_nesting = arm_smmu_enable_nesting,
.free = arm_smmu_domain_free_paging,
}
};
u32 l1size;
struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
unsigned int last_sid_idx =
--- arm_smmu_strtab_l1_idx((1 << smmu->sid_bits) - 1);
+++ arm_smmu_strtab_l1_idx((1ULL << smmu->sid_bits) - 1);
/* Calculate the L1 size, capped to the SIDSIZE. */
cfg->l2.num_l1_ents = min(last_sid_idx + 1, STRTAB_MAX_L1_ENTRIES);
#define IIDR_REVISION GENMASK(15, 12)
#define IIDR_IMPLEMENTER GENMASK(11, 0)
+++ #define ARM_SMMU_AIDR 0x1C
+++
#define ARM_SMMU_CR0 0x20
#define CR0_ATSCHK (1 << 4)
#define CR0_CMDQEN (1 << 3)
/* List of struct arm_smmu_master_domain */
struct list_head devices;
spinlock_t devices_lock;
+++ bool enforce_cache_coherency : 1;
struct mmu_notifier mmu_notifier;
};
void (*sync)(struct arm_smmu_entry_writer *writer);
};
+++ void arm_smmu_make_abort_ste(struct arm_smmu_ste *target);
+++ void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target,
+++ struct arm_smmu_master *master,
+++ struct arm_smmu_domain *smmu_domain,
+++ bool ats_enabled);
+++
#if IS_ENABLED(CONFIG_KUNIT)
void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits);
void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *cur,
const __le64 *target);
void arm_smmu_get_cd_used(const __le64 *ent, __le64 *used_bits);
--- void arm_smmu_make_abort_ste(struct arm_smmu_ste *target);
void arm_smmu_make_bypass_ste(struct arm_smmu_device *smmu,
struct arm_smmu_ste *target);
void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target,
struct arm_smmu_master *master, bool ats_enabled,
unsigned int s1dss);
--- void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target,
--- struct arm_smmu_master *master,
--- struct arm_smmu_domain *smmu_domain,
--- bool ats_enabled);
void arm_smmu_make_sva_cd(struct arm_smmu_cd *target,
struct arm_smmu_master *master, struct mm_struct *mm,
u16 asid);
int arm_smmu_set_pasid(struct arm_smmu_master *master,
struct arm_smmu_domain *smmu_domain, ioasid_t pasid,
- -- struct arm_smmu_cd *cd);
+ ++ struct arm_smmu_cd *cd, struct iommu_domain *old);
void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid);
void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
int arm_smmu_cmdq_init(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq *cmdq);
+++ static inline bool arm_smmu_master_canwbs(struct arm_smmu_master *master)
+++ {
+++ return dev_iommu_fwspec_get(master->dev)->flags &
+++ IOMMU_FWSPEC_PCI_RC_CANWBS;
+++ }
+++
+++ struct arm_smmu_attach_state {
+++ /* Inputs */
+++ struct iommu_domain *old_domain;
+++ struct arm_smmu_master *master;
+++ bool cd_needs_ats;
+++ ioasid_t ssid;
+++ /* Resulting state */
+++ bool ats_enabled;
+++ };
+++
+++ int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state,
+++ struct iommu_domain *new_domain);
+++ void arm_smmu_attach_commit(struct arm_smmu_attach_state *state);
+++ void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master,
+++ const struct arm_smmu_ste *target);
+++
#ifdef CONFIG_ARM_SMMU_V3_SVA
bool arm_smmu_sva_supported(struct arm_smmu_device *smmu);
bool arm_smmu_master_sva_supported(struct arm_smmu_master *master);
return ERR_PTR(-ENODEV);
}
#endif /* CONFIG_TEGRA241_CMDQV */
+++
+++ #if IS_ENABLED(CONFIG_ARM_SMMU_V3_IOMMUFD)
+++ void *arm_smmu_hw_info(struct device *dev, u32 *length, u32 *type);
+++ #else
+++ #define arm_smmu_hw_info NULL
+++ #endif /* CONFIG_ARM_SMMU_V3_IOMMUFD */
+++
#endif /* _ARM_SMMU_V3_H */
goto out_free;
} else {
smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
+++
+++ /*
+++ * Defer probe if the relevant SMMU instance hasn't finished
+++ * probing yet. This is a fragile hack and we'd ideally
+++ * avoid this race in the core code. Until that's ironed
+++ * out, however, this is the most pragmatic option on the
+++ * table.
+++ */
+++ if (!smmu)
+++ return ERR_PTR(dev_err_probe(dev, -EPROBE_DEFER,
+++ "smmu dev has not bound yet\n"));
}
ret = -EINVAL;
return group;
}
--- static int arm_smmu_enable_nesting(struct iommu_domain *domain)
--- {
--- struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
--- int ret = 0;
---
--- mutex_lock(&smmu_domain->init_mutex);
--- if (smmu_domain->smmu)
--- ret = -EPERM;
--- else
--- smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
--- mutex_unlock(&smmu_domain->init_mutex);
---
--- return ret;
--- }
---
static int arm_smmu_set_pgtable_quirks(struct iommu_domain *domain,
unsigned long quirks)
{
.flush_iotlb_all = arm_smmu_flush_iotlb_all,
.iotlb_sync = arm_smmu_iotlb_sync,
.iova_to_phys = arm_smmu_iova_to_phys,
--- .enable_nesting = arm_smmu_enable_nesting,
.set_pgtable_quirks = arm_smmu_set_pgtable_quirks,
.free = arm_smmu_domain_free,
}
ecap_smpwc(iommu->ecap) : ecap_coherent(iommu->ecap);
}
- --static void domain_update_iommu_coherency(struct dmar_domain *domain)
- --{
- -- struct iommu_domain_info *info;
- -- struct dmar_drhd_unit *drhd;
- -- struct intel_iommu *iommu;
- -- bool found = false;
- -- unsigned long i;
- --
- -- domain->iommu_coherency = true;
- -- xa_for_each(&domain->iommu_array, i, info) {
- -- found = true;
- -- if (!iommu_paging_structure_coherency(info->iommu)) {
- -- domain->iommu_coherency = false;
- -- break;
- -- }
- -- }
- -- if (found)
- -- return;
- --
- -- /* No hardware attached; use lowest common denominator */
- -- rcu_read_lock();
- -- for_each_active_iommu(iommu, drhd) {
- -- if (!iommu_paging_structure_coherency(iommu)) {
- -- domain->iommu_coherency = false;
- -- break;
- -- }
- -- }
- -- rcu_read_unlock();
- --}
- --
- --static int domain_update_iommu_superpage(struct dmar_domain *domain,
- -- struct intel_iommu *skip)
- --{
- -- struct dmar_drhd_unit *drhd;
- -- struct intel_iommu *iommu;
- -- int mask = 0x3;
- --
- -- if (!intel_iommu_superpage)
- -- return 0;
- --
- -- /* set iommu_superpage to the smallest common denominator */
- -- rcu_read_lock();
- -- for_each_active_iommu(iommu, drhd) {
- -- if (iommu != skip) {
- -- if (domain && domain->use_first_level) {
- -- if (!cap_fl1gp_support(iommu->cap))
- -- mask = 0x1;
- -- } else {
- -- mask &= cap_super_page_val(iommu->cap);
- -- }
- --
- -- if (!mask)
- -- break;
- -- }
- -- }
- -- rcu_read_unlock();
- --
- -- return fls(mask);
- --}
- --
- --static int domain_update_device_node(struct dmar_domain *domain)
- --{
- -- struct device_domain_info *info;
- -- int nid = NUMA_NO_NODE;
- -- unsigned long flags;
- --
- -- spin_lock_irqsave(&domain->lock, flags);
- -- list_for_each_entry(info, &domain->devices, link) {
- -- /*
- -- * There could possibly be multiple device numa nodes as devices
- -- * within the same domain may sit behind different IOMMUs. There
- -- * isn't perfect answer in such situation, so we select first
- -- * come first served policy.
- -- */
- -- nid = dev_to_node(info->dev);
- -- if (nid != NUMA_NO_NODE)
- -- break;
- -- }
- -- spin_unlock_irqrestore(&domain->lock, flags);
- --
- -- return nid;
- --}
- --
/* Return the super pagesize bitmap if supported. */
static unsigned long domain_super_pgsize_bitmap(struct dmar_domain *domain)
{
return bitmap;
}
- --/* Some capabilities may be different across iommus */
- --void domain_update_iommu_cap(struct dmar_domain *domain)
- --{
- -- domain_update_iommu_coherency(domain);
- -- domain->iommu_superpage = domain_update_iommu_superpage(domain, NULL);
- --
- -- /*
- -- * If RHSA is missing, we should default to the device numa domain
- -- * as fall back.
- -- */
- -- if (domain->nid == NUMA_NO_NODE)
- -- domain->nid = domain_update_device_node(domain);
- --
- -- /*
- -- * First-level translation restricts the input-address to a
- -- * canonical address (i.e., address bits 63:N have the same
- -- * value as address bit [N-1], where N is 48-bits with 4-level
- -- * paging and 57-bits with 5-level paging). Hence, skip bit
- -- * [N-1].
- -- */
- -- if (domain->use_first_level)
- -- domain->domain.geometry.aperture_end = __DOMAIN_MAX_ADDR(domain->gaw - 1);
- -- else
- -- domain->domain.geometry.aperture_end = __DOMAIN_MAX_ADDR(domain->gaw);
- --
- -- domain->domain.pgsize_bitmap |= domain_super_pgsize_bitmap(domain);
- --}
- --
struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,
u8 devfn, int alloc)
{
while (1) {
offset = pfn_level_offset(pfn, level);
pte = &parent[offset];
- -- if (!pte || (dma_pte_superpage(pte) || !dma_pte_present(pte))) {
- -- pr_info("PTE not present at level %d\n", level);
- -- break;
- -- }
pr_info("pte level: %d, pte value: 0x%016llx\n", level, pte->val);
- -- if (level == 1)
+ ++ if (!dma_pte_present(pte)) {
+ ++ pr_info("page table not present at level %d\n", level - 1);
+ ++ break;
+ ++ }
+ ++
+ ++ if (level == 1 || dma_pte_superpage(pte))
break;
parent = phys_to_virt(dma_pte_addr(pte));
pr_info("Dump %s table entries for IOVA 0x%llx\n", iommu->name, addr);
/* root entry dump */
- -- rt_entry = &iommu->root_entry[bus];
- -- if (!rt_entry) {
- -- pr_info("root table entry is not present\n");
+ ++ if (!iommu->root_entry) {
+ ++ pr_info("root table is not present\n");
return;
}
+ ++ rt_entry = &iommu->root_entry[bus];
if (sm_supported(iommu))
pr_info("scalable mode root entry: hi 0x%016llx, low 0x%016llx\n",
/* context entry dump */
ctx_entry = iommu_context_addr(iommu, bus, devfn, 0);
if (!ctx_entry) {
- -- pr_info("context table entry is not present\n");
+ ++ pr_info("context table is not present\n");
return;
}
/* legacy mode does not require PASID entries */
if (!sm_supported(iommu)) {
+ ++ if (!context_present(ctx_entry)) {
+ ++ pr_info("legacy mode page table is not present\n");
+ ++ return;
+ ++ }
level = agaw_to_level(ctx_entry->hi & 7);
pgtable = phys_to_virt(ctx_entry->lo & VTD_PAGE_MASK);
goto pgtable_walk;
}
- -- /* get the pointer to pasid directory entry */
- -- dir = phys_to_virt(ctx_entry->lo & VTD_PAGE_MASK);
- -- if (!dir) {
- -- pr_info("pasid directory entry is not present\n");
+ ++ if (!context_present(ctx_entry)) {
+ ++ pr_info("pasid directory table is not present\n");
return;
}
+ ++
+ ++ /* get the pointer to pasid directory entry */
+ ++ dir = phys_to_virt(ctx_entry->lo & VTD_PAGE_MASK);
+ ++
/* For request-without-pasid, get the pasid from context entry */
if (intel_iommu_sm && pasid == IOMMU_PASID_INVALID)
pasid = IOMMU_NO_PASID;
/* get the pointer to the pasid table entry */
entries = get_pasid_table_from_pde(pde);
if (!entries) {
- -- pr_info("pasid table entry is not present\n");
+ ++ pr_info("pasid table is not present\n");
return;
}
index = pasid & PASID_PTE_MASK;
for (i = 0; i < ARRAY_SIZE(pte->val); i++)
pr_info("pasid table entry[%d]: 0x%016llx\n", i, pte->val[i]);
+ ++ if (!pasid_pte_is_present(pte)) {
+ ++ pr_info("scalable mode page table is not present\n");
+ ++ return;
+ ++ }
+ ++
if (pasid_pte_get_pgtt(pte) == PASID_ENTRY_PGTT_FL_ONLY) {
level = pte->val[2] & BIT_ULL(2) ? 5 : 4;
pgtable = phys_to_virt(pte->val[2] & VTD_PAGE_MASK);
/* free context mapping */
free_context_table(iommu);
- --#ifdef CONFIG_INTEL_IOMMU_SVM
- -- if (pasid_supported(iommu)) {
- -- if (ecap_prs(iommu->ecap))
- -- intel_svm_finish_prq(iommu);
- -- }
- --#endif
+ ++ if (ecap_prs(iommu->ecap))
+ ++ intel_iommu_finish_prq(iommu);
}
/*
* Check and return whether first level is used by default for
* DMA translation.
*/
- --static bool first_level_by_default(unsigned int type)
+ ++static bool first_level_by_default(struct intel_iommu *iommu)
{
/* Only SL is available in legacy mode */
- -- if (!scalable_mode_support())
+ ++ if (!sm_supported(iommu))
return false;
/* Only level (either FL or SL) is available, just use it */
- -- if (intel_cap_flts_sanity() ^ intel_cap_slts_sanity())
- -- return intel_cap_flts_sanity();
- --
- -- /* Both levels are available, decide it based on domain type */
- -- return type != IOMMU_DOMAIN_UNMANAGED;
- --}
+ ++ if (ecap_flts(iommu->ecap) ^ ecap_slts(iommu->ecap))
+ ++ return ecap_flts(iommu->ecap);
- --static struct dmar_domain *alloc_domain(unsigned int type)
- --{
- -- struct dmar_domain *domain;
- --
- -- domain = kzalloc(sizeof(*domain), GFP_KERNEL);
- -- if (!domain)
- -- return NULL;
- --
- -- domain->nid = NUMA_NO_NODE;
- -- if (first_level_by_default(type))
- -- domain->use_first_level = true;
- -- INIT_LIST_HEAD(&domain->devices);
- -- INIT_LIST_HEAD(&domain->dev_pasids);
- -- INIT_LIST_HEAD(&domain->cache_tags);
- -- spin_lock_init(&domain->lock);
- -- spin_lock_init(&domain->cache_lock);
- -- xa_init(&domain->iommu_array);
- --
- -- return domain;
+ ++ return true;
}
int domain_attach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu)
ret = xa_err(curr) ? : -EBUSY;
goto err_clear;
}
- -- domain_update_iommu_cap(domain);
spin_unlock(&iommu->lock);
return 0;
clear_bit(info->did, iommu->domain_ids);
xa_erase(&domain->iommu_array, iommu->seq_id);
domain->nid = NUMA_NO_NODE;
- -- domain_update_iommu_cap(domain);
kfree(info);
}
spin_unlock(&iommu->lock);
}
- --static int guestwidth_to_adjustwidth(int gaw)
- --{
- -- int agaw;
- -- int r = (gaw - 12) % 9;
- --
- -- if (r == 0)
- -- agaw = gaw;
- -- else
- -- agaw = gaw + 9 - r;
- -- if (agaw > 64)
- -- agaw = 64;
- -- return agaw;
- --}
- --
static void domain_exit(struct dmar_domain *domain)
{
if (domain->pgd) {
if (did_old < cap_ndoms(iommu->cap)) {
iommu->flush.flush_context(iommu, did_old,
- -- (((u16)bus) << 8) | devfn,
+ ++ PCI_DEVID(bus, devfn),
DMA_CCMD_MASK_NOBIT,
DMA_CCMD_DEVICE_INVL);
iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
{
if (cap_caching_mode(iommu->cap)) {
iommu->flush.flush_context(iommu, 0,
- -- (((u16)bus) << 8) | devfn,
+ ++ PCI_DEVID(bus, devfn),
DMA_CCMD_MASK_NOBIT,
DMA_CCMD_DEVICE_INVL);
iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
int translation = CONTEXT_TT_MULTI_LEVEL;
struct dma_pte *pgd = domain->pgd;
struct context_entry *context;
- -- int agaw, ret;
+ ++ int ret;
pr_debug("Set context mapping for %02x:%02x.%d\n",
bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
copied_context_tear_down(iommu, context, bus, devfn);
context_clear_entry(context);
- --
context_set_domain_id(context, did);
- -- /*
- -- * Skip top levels of page tables for iommu which has
- -- * less agaw than default. Unnecessary for PT mode.
- -- */
- -- for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
- -- ret = -ENOMEM;
- -- pgd = phys_to_virt(dma_pte_addr(pgd));
- -- if (!dma_pte_present(pgd))
- -- goto out_unlock;
- -- }
- --
if (info && info->ats_supported)
translation = CONTEXT_TT_DEV_IOTLB;
else
translation = CONTEXT_TT_MULTI_LEVEL;
context_set_address_root(context, virt_to_phys(pgd));
- -- context_set_address_width(context, agaw);
+ ++ context_set_address_width(context, domain->agaw);
context_set_translation_type(context, translation);
context_set_fault_enable(context);
context_set_present(context);
intel_context_flush_present(info, context, did, true);
}
+ ++int __domain_setup_first_level(struct intel_iommu *iommu,
+ ++ struct device *dev, ioasid_t pasid,
+ ++ u16 did, pgd_t *pgd, int flags,
+ ++ struct iommu_domain *old)
+ ++{
+ ++ if (!old)
+ ++ return intel_pasid_setup_first_level(iommu, dev, pgd,
+ ++ pasid, did, flags);
+ ++ return intel_pasid_replace_first_level(iommu, dev, pgd, pasid, did,
+ ++ iommu_domain_did(old, iommu),
+ ++ flags);
+ ++}
+ ++
+ ++static int domain_setup_second_level(struct intel_iommu *iommu,
+ ++ struct dmar_domain *domain,
+ ++ struct device *dev, ioasid_t pasid,
+ ++ struct iommu_domain *old)
+ ++{
+ ++ if (!old)
+ ++ return intel_pasid_setup_second_level(iommu, domain,
+ ++ dev, pasid);
+ ++ return intel_pasid_replace_second_level(iommu, domain, dev,
+ ++ iommu_domain_did(old, iommu),
+ ++ pasid);
+ ++}
+ ++
+ ++static int domain_setup_passthrough(struct intel_iommu *iommu,
+ ++ struct device *dev, ioasid_t pasid,
+ ++ struct iommu_domain *old)
+ ++{
+ ++ if (!old)
+ ++ return intel_pasid_setup_pass_through(iommu, dev, pasid);
+ ++ return intel_pasid_replace_pass_through(iommu, dev,
+ ++ iommu_domain_did(old, iommu),
+ ++ pasid);
+ ++}
+ ++
static int domain_setup_first_level(struct intel_iommu *iommu,
struct dmar_domain *domain,
struct device *dev,
- -- u32 pasid)
+ ++ u32 pasid, struct iommu_domain *old)
{
struct dma_pte *pgd = domain->pgd;
- -- int agaw, level;
- -- int flags = 0;
-
- /*
- * Skip top levels of page tables for iommu which has
- * less agaw than default. Unnecessary for PT mode.
- */
- for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
- pgd = phys_to_virt(dma_pte_addr(pgd));
- if (!dma_pte_present(pgd))
- return -ENOMEM;
- }
+ ++ int level, flags = 0;
-- /*
-- * Skip top levels of page tables for iommu which has
-- * less agaw than default. Unnecessary for PT mode.
-- */
-- for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
-- pgd = phys_to_virt(dma_pte_addr(pgd));
-- if (!dma_pte_present(pgd))
-- return -ENOMEM;
-- }
--
- -- level = agaw_to_level(agaw);
+ ++ level = agaw_to_level(domain->agaw);
if (level != 4 && level != 5)
return -EINVAL;
if (domain->force_snooping)
flags |= PASID_FLAG_PAGE_SNOOP;
- -- return intel_pasid_setup_first_level(iommu, dev, (pgd_t *)pgd, pasid,
- -- domain_id_iommu(domain, iommu),
- -- flags);
- --}
- --
- --static bool dev_is_real_dma_subdevice(struct device *dev)
- --{
- -- return dev && dev_is_pci(dev) &&
- -- pci_real_dma_dev(to_pci_dev(dev)) != to_pci_dev(dev);
+ ++ return __domain_setup_first_level(iommu, dev, pasid,
+ ++ domain_id_iommu(domain, iommu),
+ ++ (pgd_t *)pgd, flags, old);
}
static int dmar_domain_attach_device(struct dmar_domain *domain,
if (!sm_supported(iommu))
ret = domain_context_mapping(domain, dev);
else if (domain->use_first_level)
- -- ret = domain_setup_first_level(iommu, domain, dev, IOMMU_NO_PASID);
+ ++ ret = domain_setup_first_level(iommu, domain, dev,
+ ++ IOMMU_NO_PASID, NULL);
else
- -- ret = intel_pasid_setup_second_level(iommu, domain, dev, IOMMU_NO_PASID);
+ ++ ret = domain_setup_second_level(iommu, domain, dev,
+ ++ IOMMU_NO_PASID, NULL);
if (ret)
goto out_block_translation;
iommu_flush_write_buffer(iommu);
- --#ifdef CONFIG_INTEL_IOMMU_SVM
- -- if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) {
+ ++ if (ecap_prs(iommu->ecap)) {
/*
* Call dmar_alloc_hwirq() with dmar_global_lock held,
* could cause possible lock race condition.
*/
up_write(&dmar_global_lock);
- -- ret = intel_svm_enable_prq(iommu);
+ ++ ret = intel_iommu_enable_prq(iommu);
down_write(&dmar_global_lock);
if (ret)
goto free_iommu;
}
- --#endif
+ ++
ret = dmar_set_interrupt(iommu);
if (ret)
goto free_iommu;
static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
{
- -- int sp, ret;
struct intel_iommu *iommu = dmaru->iommu;
+ ++ int ret;
ret = intel_cap_audit(CAP_AUDIT_HOTPLUG_DMAR, iommu);
if (ret)
goto out;
- -- sp = domain_update_iommu_superpage(NULL, iommu) - 1;
- -- if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
- -- pr_warn("%s: Doesn't support large page.\n",
- -- iommu->name);
- -- return -ENXIO;
- -- }
- --
/*
* Disable translation if already enabled prior to OS handover.
*/
intel_iommu_init_qi(iommu);
iommu_flush_write_buffer(iommu);
- --#ifdef CONFIG_INTEL_IOMMU_SVM
- -- if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) {
- -- ret = intel_svm_enable_prq(iommu);
+ ++ if (ecap_prs(iommu->ecap)) {
+ ++ ret = intel_iommu_enable_prq(iommu);
if (ret)
goto disable_iommu;
}
- --#endif
+ ++
ret = dmar_set_interrupt(iommu);
if (ret)
goto disable_iommu;
* the virtual and physical IOMMU page-tables.
*/
if (cap_caching_mode(iommu->cap) &&
- -- !first_level_by_default(IOMMU_DOMAIN_DMA)) {
+ ++ !first_level_by_default(iommu)) {
pr_info_once("IOMMU batching disallowed due to virtualization\n");
iommu_set_dma_strict();
}
*/
static void domain_context_clear(struct device_domain_info *info)
{
--- if (!dev_is_pci(info->dev))
+++ if (!dev_is_pci(info->dev)) {
domain_context_clear_one(info, info->bus, info->devfn);
+++ return;
+++ }
pci_for_each_dma_alias(to_pci_dev(info->dev),
&domain_context_clear_one_cb, info);
info->domain = NULL;
}
- --static int md_domain_init(struct dmar_domain *domain, int guest_width)
- --{
- -- int adjust_width;
- --
- -- /* calculate AGAW */
- -- domain->gaw = guest_width;
- -- adjust_width = guestwidth_to_adjustwidth(guest_width);
- -- domain->agaw = width_to_agaw(adjust_width);
- --
- -- domain->iommu_coherency = false;
- -- domain->iommu_superpage = 0;
- -- domain->max_addr = 0;
- --
- -- /* always allocate the top pgd */
- -- domain->pgd = iommu_alloc_page_node(domain->nid, GFP_ATOMIC);
- -- if (!domain->pgd)
- -- return -ENOMEM;
- -- domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
- -- return 0;
- --}
- --
static int blocking_domain_attach_dev(struct iommu_domain *domain,
struct device *dev)
{
return domain;
}
- --static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
- --{
- -- struct dmar_domain *dmar_domain;
- -- struct iommu_domain *domain;
- --
- -- switch (type) {
- -- case IOMMU_DOMAIN_DMA:
- -- case IOMMU_DOMAIN_UNMANAGED:
- -- dmar_domain = alloc_domain(type);
- -- if (!dmar_domain) {
- -- pr_err("Can't allocate dmar_domain\n");
- -- return NULL;
- -- }
- -- if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
- -- pr_err("Domain initialization failed\n");
- -- domain_exit(dmar_domain);
- -- return NULL;
- -- }
- --
- -- domain = &dmar_domain->domain;
- -- domain->geometry.aperture_start = 0;
- -- domain->geometry.aperture_end =
- -- __DOMAIN_MAX_ADDR(dmar_domain->gaw);
- -- domain->geometry.force_aperture = true;
- --
- -- return domain;
- -- default:
- -- return NULL;
- -- }
- --
- -- return NULL;
- --}
- --
static struct iommu_domain *
intel_iommu_domain_alloc_user(struct device *dev, u32 flags,
struct iommu_domain *parent,
struct intel_iommu *iommu = info->iommu;
struct dmar_domain *dmar_domain;
struct iommu_domain *domain;
+ ++ bool first_stage;
/* Must be NESTING domain */
if (parent) {
}
if (flags &
- -- (~(IOMMU_HWPT_ALLOC_NEST_PARENT | IOMMU_HWPT_ALLOC_DIRTY_TRACKING)))
+ ++ (~(IOMMU_HWPT_ALLOC_NEST_PARENT | IOMMU_HWPT_ALLOC_DIRTY_TRACKING
+ ++ | IOMMU_HWPT_FAULT_ID_VALID)))
return ERR_PTR(-EOPNOTSUPP);
if (nested_parent && !nested_supported(iommu))
return ERR_PTR(-EOPNOTSUPP);
if (user_data || (dirty_tracking && !ssads_supported(iommu)))
return ERR_PTR(-EOPNOTSUPP);
- -- /* Do not use first stage for user domain translation. */
- -- dmar_domain = paging_domain_alloc(dev, false);
+ ++ /*
+ ++ * Always allocate the guest compatible page table unless
+ ++ * IOMMU_HWPT_ALLOC_NEST_PARENT or IOMMU_HWPT_ALLOC_DIRTY_TRACKING
+ ++ * is specified.
+ ++ */
+ ++ if (nested_parent || dirty_tracking) {
+ ++ if (!sm_supported(iommu) || !ecap_slts(iommu->ecap))
+ ++ return ERR_PTR(-EOPNOTSUPP);
+ ++ first_stage = false;
+ ++ } else {
+ ++ first_stage = first_level_by_default(iommu);
+ ++ }
+ ++
+ ++ dmar_domain = paging_domain_alloc(dev, first_stage);
if (IS_ERR(dmar_domain))
return ERR_CAST(dmar_domain);
domain = &dmar_domain->domain;
domain_exit(dmar_domain);
}
- --int prepare_domain_attach_device(struct iommu_domain *domain,
- -- struct device *dev)
+ ++int paging_domain_compatible(struct iommu_domain *domain, struct device *dev)
{
struct device_domain_info *info = dev_iommu_priv_get(dev);
struct dmar_domain *dmar_domain = to_dmar_domain(domain);
struct intel_iommu *iommu = info->iommu;
int addr_width;
+ ++ if (WARN_ON_ONCE(!(domain->type & __IOMMU_DOMAIN_PAGING)))
+ ++ return -EPERM;
+ ++
if (dmar_domain->force_snooping && !ecap_sc_support(iommu->ecap))
return -EINVAL;
if (domain->dirty_ops && !ssads_supported(iommu))
return -EINVAL;
+ ++ if (dmar_domain->iommu_coherency !=
+ ++ iommu_paging_structure_coherency(iommu))
+ ++ return -EINVAL;
+ ++
+ ++ if (dmar_domain->iommu_superpage !=
+ ++ iommu_superpage_capability(iommu, dmar_domain->use_first_level))
+ ++ return -EINVAL;
+ ++
+ ++ if (dmar_domain->use_first_level &&
+ ++ (!sm_supported(iommu) || !ecap_flts(iommu->ecap)))
+ ++ return -EINVAL;
+ ++
/* check if this iommu agaw is sufficient for max mapped address */
addr_width = agaw_to_width(iommu->agaw);
if (addr_width > cap_mgaw(iommu->cap))
addr_width = cap_mgaw(iommu->cap);
- -- if (dmar_domain->max_addr > (1LL << addr_width))
+ ++ if (dmar_domain->gaw > addr_width || dmar_domain->agaw > iommu->agaw)
return -EINVAL;
- -- dmar_domain->gaw = addr_width;
- --
- -- /*
- -- * Knock out extra levels of page tables if necessary
- -- */
- -- while (iommu->agaw < dmar_domain->agaw) {
- -- struct dma_pte *pte;
- --
- -- pte = dmar_domain->pgd;
- -- if (dma_pte_present(pte)) {
- -- dmar_domain->pgd = phys_to_virt(dma_pte_addr(pte));
- -- iommu_free_page(pte);
- -- }
- -- dmar_domain->agaw--;
- -- }
if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev) &&
context_copied(iommu, info->bus, info->devfn))
device_block_translation(dev);
- -- ret = prepare_domain_attach_device(domain, dev);
+ ++ ret = paging_domain_compatible(domain, dev);
if (ret)
return ret;
return 0;
}
- --static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid,
- -- struct iommu_domain *domain)
+ ++void domain_remove_dev_pasid(struct iommu_domain *domain,
+ ++ struct device *dev, ioasid_t pasid)
{
struct device_domain_info *info = dev_iommu_priv_get(dev);
struct dev_pasid_info *curr, *dev_pasid = NULL;
struct dmar_domain *dmar_domain;
unsigned long flags;
- -- if (domain->type == IOMMU_DOMAIN_IDENTITY) {
- -- intel_pasid_tear_down_entry(iommu, dev, pasid, false);
+ ++ if (!domain)
+ ++ return;
+ ++
+ ++ /* Identity domain has no meta data for pasid. */
+ ++ if (domain->type == IOMMU_DOMAIN_IDENTITY)
return;
- -- }
dmar_domain = to_dmar_domain(domain);
spin_lock_irqsave(&dmar_domain->lock, flags);
domain_detach_iommu(dmar_domain, iommu);
intel_iommu_debugfs_remove_dev_pasid(dev_pasid);
kfree(dev_pasid);
- -- intel_pasid_tear_down_entry(iommu, dev, pasid, false);
- -- intel_drain_pasid_prq(dev, pasid);
}
- --static int intel_iommu_set_dev_pasid(struct iommu_domain *domain,
- -- struct device *dev, ioasid_t pasid)
+ ++static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid,
+ ++ struct iommu_domain *domain)
+ ++{
+ ++ struct device_domain_info *info = dev_iommu_priv_get(dev);
+ ++
+ ++ intel_pasid_tear_down_entry(info->iommu, dev, pasid, false);
+ ++ domain_remove_dev_pasid(domain, dev, pasid);
+ ++}
+ ++
+ ++struct dev_pasid_info *
+ ++domain_add_dev_pasid(struct iommu_domain *domain,
+ ++ struct device *dev, ioasid_t pasid)
{
struct device_domain_info *info = dev_iommu_priv_get(dev);
struct dmar_domain *dmar_domain = to_dmar_domain(domain);
unsigned long flags;
int ret;
- -- if (!pasid_supported(iommu) || dev_is_real_dma_subdevice(dev))
- -- return -EOPNOTSUPP;
- --
- -- if (domain->dirty_ops)
- -- return -EINVAL;
- --
- -- if (context_copied(iommu, info->bus, info->devfn))
- -- return -EBUSY;
- --
- -- ret = prepare_domain_attach_device(domain, dev);
- -- if (ret)
- -- return ret;
- --
dev_pasid = kzalloc(sizeof(*dev_pasid), GFP_KERNEL);
if (!dev_pasid)
- -- return -ENOMEM;
+ ++ return ERR_PTR(-ENOMEM);
ret = domain_attach_iommu(dmar_domain, iommu);
if (ret)
if (ret)
goto out_detach_iommu;
- -- if (dmar_domain->use_first_level)
- -- ret = domain_setup_first_level(iommu, dmar_domain,
- -- dev, pasid);
- -- else
- -- ret = intel_pasid_setup_second_level(iommu, dmar_domain,
- -- dev, pasid);
- -- if (ret)
- -- goto out_unassign_tag;
- --
dev_pasid->dev = dev;
dev_pasid->pasid = pasid;
spin_lock_irqsave(&dmar_domain->lock, flags);
list_add(&dev_pasid->link_domain, &dmar_domain->dev_pasids);
spin_unlock_irqrestore(&dmar_domain->lock, flags);
- -- if (domain->type & __IOMMU_DOMAIN_PAGING)
- -- intel_iommu_debugfs_create_dev_pasid(dev_pasid);
- --
- -- return 0;
- --out_unassign_tag:
- -- cache_tag_unassign_domain(dmar_domain, dev, pasid);
+ ++ return dev_pasid;
out_detach_iommu:
domain_detach_iommu(dmar_domain, iommu);
out_free:
kfree(dev_pasid);
+ ++ return ERR_PTR(ret);
+ ++}
+ ++
+ ++static int intel_iommu_set_dev_pasid(struct iommu_domain *domain,
+ ++ struct device *dev, ioasid_t pasid,
+ ++ struct iommu_domain *old)
+ ++{
+ ++ struct device_domain_info *info = dev_iommu_priv_get(dev);
+ ++ struct dmar_domain *dmar_domain = to_dmar_domain(domain);
+ ++ struct intel_iommu *iommu = info->iommu;
+ ++ struct dev_pasid_info *dev_pasid;
+ ++ int ret;
+ ++
+ ++ if (WARN_ON_ONCE(!(domain->type & __IOMMU_DOMAIN_PAGING)))
+ ++ return -EINVAL;
+ ++
+ ++ if (!pasid_supported(iommu) || dev_is_real_dma_subdevice(dev))
+ ++ return -EOPNOTSUPP;
+ ++
+ ++ if (domain->dirty_ops)
+ ++ return -EINVAL;
+ ++
+ ++ if (context_copied(iommu, info->bus, info->devfn))
+ ++ return -EBUSY;
+ ++
+ ++ ret = paging_domain_compatible(domain, dev);
+ ++ if (ret)
+ ++ return ret;
+ ++
+ ++ dev_pasid = domain_add_dev_pasid(domain, dev, pasid);
+ ++ if (IS_ERR(dev_pasid))
+ ++ return PTR_ERR(dev_pasid);
+ ++
+ ++ if (dmar_domain->use_first_level)
+ ++ ret = domain_setup_first_level(iommu, dmar_domain,
+ ++ dev, pasid, old);
+ ++ else
+ ++ ret = domain_setup_second_level(iommu, dmar_domain,
+ ++ dev, pasid, old);
+ ++ if (ret)
+ ++ goto out_remove_dev_pasid;
+ ++
+ ++ domain_remove_dev_pasid(old, dev, pasid);
+ ++
+ ++ intel_iommu_debugfs_create_dev_pasid(dev_pasid);
+ ++
+ ++ return 0;
+ ++
+ ++out_remove_dev_pasid:
+ ++ domain_remove_dev_pasid(domain, dev, pasid);
return ret;
}
}
static int identity_domain_set_dev_pasid(struct iommu_domain *domain,
- -- struct device *dev, ioasid_t pasid)
+ ++ struct device *dev, ioasid_t pasid,
+ ++ struct iommu_domain *old)
{
struct device_domain_info *info = dev_iommu_priv_get(dev);
struct intel_iommu *iommu = info->iommu;
+ ++ int ret;
if (!pasid_supported(iommu) || dev_is_real_dma_subdevice(dev))
return -EOPNOTSUPP;
- -- return intel_pasid_setup_pass_through(iommu, dev, pasid);
+ ++ ret = domain_setup_passthrough(iommu, dev, pasid, old);
+ ++ if (ret)
+ ++ return ret;
+ ++
+ ++ domain_remove_dev_pasid(old, dev, pasid);
+ ++ return 0;
}
static struct iommu_domain identity_domain = {
},
};
+ ++static struct iommu_domain *intel_iommu_domain_alloc_paging(struct device *dev)
+ ++{
+ ++ struct device_domain_info *info = dev_iommu_priv_get(dev);
+ ++ struct intel_iommu *iommu = info->iommu;
+ ++ struct dmar_domain *dmar_domain;
+ ++ bool first_stage;
+ ++
+ ++ first_stage = first_level_by_default(iommu);
+ ++ dmar_domain = paging_domain_alloc(dev, first_stage);
+ ++ if (IS_ERR(dmar_domain))
+ ++ return ERR_CAST(dmar_domain);
+ ++
+ ++ return &dmar_domain->domain;
+ ++}
+ ++
const struct iommu_ops intel_iommu_ops = {
.blocked_domain = &blocking_domain,
.release_domain = &blocking_domain,
.identity_domain = &identity_domain,
.capable = intel_iommu_capable,
.hw_info = intel_iommu_hw_info,
- -- .domain_alloc = intel_iommu_domain_alloc,
.domain_alloc_user = intel_iommu_domain_alloc_user,
.domain_alloc_sva = intel_svm_domain_alloc,
+ ++ .domain_alloc_paging = intel_iommu_domain_alloc_paging,
.probe_device = intel_iommu_probe_device,
.release_device = intel_iommu_release_device,
.get_resv_regions = intel_iommu_get_resv_regions,
.def_domain_type = device_def_domain_type,
.remove_dev_pasid = intel_iommu_remove_dev_pasid,
.pgsize_bitmap = SZ_4K,
- --#ifdef CONFIG_INTEL_IOMMU_SVM
- -- .page_response = intel_svm_page_response,
- --#endif
+ ++ .page_response = intel_iommu_page_response,
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = intel_iommu_attach_device,
.set_dev_pasid = intel_iommu_set_dev_pasid,
#include <trace/events/iommu.h>
#include <linux/sched/mm.h>
#include <linux/msi.h>
+ +#include <uapi/linux/iommufd.h>
#include "dma-iommu.h"
#include "iommu-priv.h"
#define IOMMU_CMD_LINE_DMA_API BIT(0)
#define IOMMU_CMD_LINE_STRICT BIT(1)
+++static int bus_iommu_probe(const struct bus_type *bus);
static int iommu_bus_notifier(struct notifier_block *nb,
unsigned long action, void *data);
static void iommu_release_device(struct device *dev);
- -static struct iommu_domain *
- -__iommu_group_domain_alloc(struct iommu_group *group, unsigned int type);
static int __iommu_attach_device(struct iommu_domain *domain,
struct device *dev);
static int __iommu_attach_group(struct iommu_domain *domain,
struct iommu_group *group);
+ +static struct iommu_domain *__iommu_paging_domain_alloc_flags(struct device *dev,
+ + unsigned int type,
+ + unsigned int flags);
enum {
IOMMU_SET_DOMAIN_MUST_SUCCEED = 1 << 0,
struct device *dev);
static void __iommu_group_free_device(struct iommu_group *group,
struct group_device *grp_dev);
+ +static void iommu_domain_init(struct iommu_domain *domain, unsigned int type,
+ + const struct iommu_ops *ops);
#define IOMMU_GROUP_ATTR(_name, _mode, _show, _store) \
struct iommu_group_attribute iommu_group_attr_##_name = \
}
}
- -
- - if (!list_empty(&mappings) && iommu_is_dma_domain(domain))
- - iommu_flush_iotlb_all(domain);
- -
out:
iommu_put_resv_regions(dev, &mappings);
}
EXPORT_SYMBOL_GPL(fsl_mc_device_group);
+ +static struct iommu_domain *__iommu_alloc_identity_domain(struct device *dev)
+ +{
+ + const struct iommu_ops *ops = dev_iommu_ops(dev);
+ + struct iommu_domain *domain;
+ +
+ + if (ops->identity_domain)
+ + return ops->identity_domain;
+ +
+ + /* Older drivers create the identity domain via ops->domain_alloc() */
+ + if (!ops->domain_alloc)
+ + return ERR_PTR(-EOPNOTSUPP);
+ +
+ + domain = ops->domain_alloc(IOMMU_DOMAIN_IDENTITY);
+ + if (IS_ERR(domain))
+ + return domain;
+ + if (!domain)
+ + return ERR_PTR(-ENOMEM);
+ +
+ + iommu_domain_init(domain, IOMMU_DOMAIN_IDENTITY, ops);
+ + return domain;
+ +}
+ +
static struct iommu_domain *
__iommu_group_alloc_default_domain(struct iommu_group *group, int req_type)
{
+ + struct device *dev = iommu_group_first_dev(group);
+ + struct iommu_domain *dom;
+ +
if (group->default_domain && group->default_domain->type == req_type)
return group->default_domain;
- - return __iommu_group_domain_alloc(group, req_type);
+ +
+ + /*
+ + * When allocating the DMA API domain assume that the driver is going to
+ + * use PASID and make sure the RID's domain is PASID compatible.
+ + */
+ + if (req_type & __IOMMU_DOMAIN_PAGING) {
+ + dom = __iommu_paging_domain_alloc_flags(dev, req_type,
+ + dev->iommu->max_pasids ? IOMMU_HWPT_ALLOC_PASID : 0);
+ +
+ + /*
+ + * If driver does not support PASID feature then
+ + * try to allocate non-PASID domain
+ + */
+ + if (PTR_ERR(dom) == -EOPNOTSUPP)
+ + dom = __iommu_paging_domain_alloc_flags(dev, req_type, 0);
+ +
+ + return dom;
+ + }
+ +
+ + if (req_type == IOMMU_DOMAIN_IDENTITY)
+ + return __iommu_alloc_identity_domain(dev);
+ +
+ + return ERR_PTR(-EINVAL);
}
/*
ops->probe_finalize(dev);
}
---int bus_iommu_probe(const struct bus_type *bus)
+++static int bus_iommu_probe(const struct bus_type *bus)
{
struct iommu_group *group, *next;
LIST_HEAD(group_list);
return 0;
}
- -/**
- - * iommu_present() - make platform-specific assumptions about an IOMMU
- - * @bus: bus to check
- - *
- - * Do not use this function. You want device_iommu_mapped() instead.
- - *
- - * Return: true if some IOMMU is present and aware of devices on the given bus;
- - * in general it may not be the only IOMMU, and it may not have anything to do
- - * with whatever device you are ultimately interested in.
- - */
- -bool iommu_present(const struct bus_type *bus)
- -{
- - bool ret = false;
- -
- - for (int i = 0; i < ARRAY_SIZE(iommu_buses); i++) {
- - if (iommu_buses[i] == bus) {
- - spin_lock(&iommu_device_lock);
- - ret = !list_empty(&iommu_device_list);
- - spin_unlock(&iommu_device_lock);
- - }
- - }
- - return ret;
- -}
- -EXPORT_SYMBOL_GPL(iommu_present);
- -
/**
* device_iommu_capable() - check for a general IOMMU capability
* @dev: device to which the capability would be relevant, if available
}
EXPORT_SYMBOL_GPL(iommu_set_fault_handler);
- -static struct iommu_domain *__iommu_domain_alloc(const struct iommu_ops *ops,
- - struct device *dev,
- - unsigned int type)
+ +static void iommu_domain_init(struct iommu_domain *domain, unsigned int type,
+ + const struct iommu_ops *ops)
{
- - struct iommu_domain *domain;
- - unsigned int alloc_type = type & IOMMU_DOMAIN_ALLOC_FLAGS;
- -
- - if (alloc_type == IOMMU_DOMAIN_IDENTITY && ops->identity_domain)
- - return ops->identity_domain;
- - else if (alloc_type == IOMMU_DOMAIN_BLOCKED && ops->blocked_domain)
- - return ops->blocked_domain;
- - else if (type & __IOMMU_DOMAIN_PAGING && ops->domain_alloc_paging)
- - domain = ops->domain_alloc_paging(dev);
- - else if (ops->domain_alloc)
- - domain = ops->domain_alloc(alloc_type);
- - else
- - return ERR_PTR(-EOPNOTSUPP);
- -
- - /*
- - * Many domain_alloc ops now return ERR_PTR, make things easier for the
- - * driver by accepting ERR_PTR from all domain_alloc ops instead of
- - * having two rules.
- - */
- - if (IS_ERR(domain))
- - return domain;
- - if (!domain)
- - return ERR_PTR(-ENOMEM);
- -
domain->type = type;
domain->owner = ops;
+ + if (!domain->ops)
+ + domain->ops = ops->default_domain_ops;
+ +
/*
* If not already set, assume all sizes by default; the driver
* may override this later
*/
if (!domain->pgsize_bitmap)
domain->pgsize_bitmap = ops->pgsize_bitmap;
- -
- - if (!domain->ops)
- - domain->ops = ops->default_domain_ops;
- -
- - if (iommu_is_dma_domain(domain)) {
- - int rc;
- -
- - rc = iommu_get_dma_cookie(domain);
- - if (rc) {
- - iommu_domain_free(domain);
- - return ERR_PTR(rc);
- - }
- - }
- - return domain;
}
static struct iommu_domain *
- -__iommu_group_domain_alloc(struct iommu_group *group, unsigned int type)
- -{
- - struct device *dev = iommu_group_first_dev(group);
- -
- - return __iommu_domain_alloc(dev_iommu_ops(dev), dev, type);
- -}
- -
- -static int __iommu_domain_alloc_dev(struct device *dev, void *data)
+ +__iommu_paging_domain_alloc_flags(struct device *dev, unsigned int type,
+ + unsigned int flags)
{
- - const struct iommu_ops **ops = data;
+ + const struct iommu_ops *ops;
+ + struct iommu_domain *domain;
if (!dev_has_iommu(dev))
- - return 0;
- -
- - if (WARN_ONCE(*ops && *ops != dev_iommu_ops(dev),
- - "Multiple IOMMU drivers present for bus %s, which the public IOMMU API can't fully support yet. You will still need to disable one or more for this to work, sorry!\n",
- - dev_bus_name(dev)))
- - return -EBUSY;
- -
- - *ops = dev_iommu_ops(dev);
- - return 0;
- -}
+ + return ERR_PTR(-ENODEV);
- -/*
- - * The iommu ops in bus has been retired. Do not use this interface in
- - * new drivers.
- - */
- -struct iommu_domain *iommu_domain_alloc(const struct bus_type *bus)
- -{
- - const struct iommu_ops *ops = NULL;
- - int err = bus_for_each_dev(bus, NULL, &ops, __iommu_domain_alloc_dev);
- - struct iommu_domain *domain;
+ + ops = dev_iommu_ops(dev);
- - if (err || !ops)
- - return NULL;
+ + if (ops->domain_alloc_paging && !flags)
+ + domain = ops->domain_alloc_paging(dev);
+ + else if (ops->domain_alloc_user)
+ + domain = ops->domain_alloc_user(dev, flags, NULL, NULL);
+ + else if (ops->domain_alloc && !flags)
+ + domain = ops->domain_alloc(IOMMU_DOMAIN_UNMANAGED);
+ + else
+ + return ERR_PTR(-EOPNOTSUPP);
- - domain = __iommu_domain_alloc(ops, NULL, IOMMU_DOMAIN_UNMANAGED);
if (IS_ERR(domain))
- - return NULL;
+ + return domain;
+ + if (!domain)
+ + return ERR_PTR(-ENOMEM);
+ +
+ + iommu_domain_init(domain, type, ops);
return domain;
}
- -EXPORT_SYMBOL_GPL(iommu_domain_alloc);
/**
- - * iommu_paging_domain_alloc() - Allocate a paging domain
+ + * iommu_paging_domain_alloc_flags() - Allocate a paging domain
* @dev: device for which the domain is allocated
+ + * @flags: Bitmap of iommufd_hwpt_alloc_flags
*
* Allocate a paging domain which will be managed by a kernel driver. Return
- - * allocated domain if successful, or a ERR pointer for failure.
+ + * allocated domain if successful, or an ERR pointer for failure.
*/
- -struct iommu_domain *iommu_paging_domain_alloc(struct device *dev)
+ +struct iommu_domain *iommu_paging_domain_alloc_flags(struct device *dev,
+ + unsigned int flags)
{
- - if (!dev_has_iommu(dev))
- - return ERR_PTR(-ENODEV);
- -
- - return __iommu_domain_alloc(dev_iommu_ops(dev), dev, IOMMU_DOMAIN_UNMANAGED);
+ + return __iommu_paging_domain_alloc_flags(dev,
+ + IOMMU_DOMAIN_UNMANAGED, flags);
}
- -EXPORT_SYMBOL_GPL(iommu_paging_domain_alloc);
+ +EXPORT_SYMBOL_GPL(iommu_paging_domain_alloc_flags);
void iommu_domain_free(struct iommu_domain *domain)
{
/**
* iommu_group_replace_domain - replace the domain that a group is attached to
- - * @new_domain: new IOMMU domain to replace with
* @group: IOMMU group that will be attached to the new domain
+ + * @new_domain: new IOMMU domain to replace with
*
* This API allows the group to switch domains without being forced to go to
* the blocking domain in-between.
return unmapped;
}
+++/**
+++ * iommu_unmap() - Remove mappings from a range of IOVA
+++ * @domain: Domain to manipulate
+++ * @iova: IO virtual address to start
+++ * @size: Length of the range starting from @iova
+++ *
+++ * iommu_unmap() will remove a translation created by iommu_map(). It cannot
+++ * subdivide a mapping created by iommu_map(), so it should be called with IOVA
+++ * ranges that match what was passed to iommu_map(). The range can aggregate
+++ * contiguous iommu_map() calls so long as no individual range is split.
+++ *
+++ * Returns: Number of bytes of IOVA unmapped. iova + res will be the point
+++ * unmapping stopped.
+++ */
size_t iommu_unmap(struct iommu_domain *domain,
unsigned long iova, size_t size)
{
}
core_initcall(iommu_init);
--- int iommu_enable_nesting(struct iommu_domain *domain)
--- {
--- if (domain->type != IOMMU_DOMAIN_UNMANAGED)
--- return -EINVAL;
--- if (!domain->ops->enable_nesting)
--- return -EINVAL;
--- return domain->ops->enable_nesting(domain);
--- }
--- EXPORT_SYMBOL_GPL(iommu_enable_nesting);
---
int iommu_set_pgtable_quirks(struct iommu_domain *domain,
unsigned long quirk)
{
if (group->default_domain == dom)
return 0;
+ + if (iommu_is_dma_domain(dom)) {
+ + ret = iommu_get_dma_cookie(dom);
+ + if (ret) {
+ + iommu_domain_free(dom);
+ + return ret;
+ + }
+ + }
+ +
/*
* IOMMU_RESV_DIRECT and IOMMU_RESV_DIRECT_RELAXABLE regions must be
* mapped before their device is attached, in order to guarantee
static int __iommu_group_alloc_blocking_domain(struct iommu_group *group)
{
+ + struct device *dev = iommu_group_first_dev(group);
+ + const struct iommu_ops *ops = dev_iommu_ops(dev);
struct iommu_domain *domain;
if (group->blocking_domain)
return 0;
- - domain = __iommu_group_domain_alloc(group, IOMMU_DOMAIN_BLOCKED);
- - if (IS_ERR(domain)) {
- - /*
- - * For drivers that do not yet understand IOMMU_DOMAIN_BLOCKED
- - * create an empty domain instead.
- - */
- - domain = __iommu_group_domain_alloc(group,
- - IOMMU_DOMAIN_UNMANAGED);
- - if (IS_ERR(domain))
- - return PTR_ERR(domain);
+ + if (ops->blocked_domain) {
+ + group->blocking_domain = ops->blocked_domain;
+ + return 0;
}
+ +
+ + /*
+ + * For drivers that do not yet understand IOMMU_DOMAIN_BLOCKED create an
+ + * empty PAGING domain instead.
+ + */
+ + domain = iommu_paging_domain_alloc(dev);
+ + if (IS_ERR(domain))
+ + return PTR_ERR(domain);
group->blocking_domain = domain;
return 0;
}
int ret;
for_each_group_device(group, device) {
- -- ret = domain->ops->set_dev_pasid(domain, device->dev, pasid);
+ ++ ret = domain->ops->set_dev_pasid(domain, device->dev,
+ ++ pasid, NULL);
if (ret)
goto err_revert;
}
* the caller iommu_domain_alloc() returns.
* @domain_alloc_user: Allocate an iommu domain corresponding to the input
* parameters as defined in include/uapi/linux/iommufd.h.
- - * Unlike @domain_alloc, it is called only by IOMMUFD and
- - * must fully initialize the new domain before return.
* Upon success, if the @user_data is valid and the @parent
* points to a kernel-managed domain, the new domain must be
* IOMMU_DOMAIN_NESTED type; otherwise, the @parent must be
* * EBUSY - device is attached to a domain and cannot be changed
* * ENODEV - device specific errors, not able to be attached
* * <others> - treated as ENODEV by the caller. Use is discouraged
- -- * @set_dev_pasid: set an iommu domain to a pasid of device
+ ++ * @set_dev_pasid: set or replace an iommu domain to a pasid of device. The pasid of
+ ++ * the device should be left in the old config in error case.
* @map_pages: map a physically contiguous set of pages of the same size to
* an iommu domain.
* @unmap_pages: unmap a number of pages of the same size from an iommu domain
* @enforce_cache_coherency: Prevent any kind of DMA from bypassing IOMMU_CACHE,
* including no-snoop TLPs on PCIe or other platform
* specific mechanisms.
--- * @enable_nesting: Enable nesting
* @set_pgtable_quirks: Set io page table quirks (IO_PGTABLE_QUIRK_*)
* @free: Release the domain after use.
*/
struct iommu_domain_ops {
int (*attach_dev)(struct iommu_domain *domain, struct device *dev);
int (*set_dev_pasid)(struct iommu_domain *domain, struct device *dev,
- -- ioasid_t pasid);
+ ++ ioasid_t pasid, struct iommu_domain *old);
int (*map_pages)(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t pgsize, size_t pgcount,
dma_addr_t iova);
bool (*enforce_cache_coherency)(struct iommu_domain *domain);
--- int (*enable_nesting)(struct iommu_domain *domain);
int (*set_pgtable_quirks)(struct iommu_domain *domain,
unsigned long quirks);
};
}
---extern int bus_iommu_probe(const struct bus_type *bus);
+ extern bool iommu_present(const struct bus_type *bus);
extern bool device_iommu_capable(struct device *dev, enum iommu_cap cap);
extern bool iommu_group_has_isolated_msi(struct iommu_group *group);
- -extern struct iommu_domain *iommu_domain_alloc(const struct bus_type *bus);
- -struct iommu_domain *iommu_paging_domain_alloc(struct device *dev);
+ +struct iommu_domain *iommu_paging_domain_alloc_flags(struct device *dev, unsigned int flags);
+ +static inline struct iommu_domain *iommu_paging_domain_alloc(struct device *dev)
+ +{
+ + return iommu_paging_domain_alloc_flags(dev, 0);
+ +}
extern void iommu_domain_free(struct iommu_domain *domain);
extern int iommu_attach_device(struct iommu_domain *domain,
struct device *dev);
extern int iommu_group_id(struct iommu_group *group);
extern struct iommu_domain *iommu_group_default_domain(struct iommu_group *);
--- int iommu_enable_nesting(struct iommu_domain *domain);
int iommu_set_pgtable_quirks(struct iommu_domain *domain,
unsigned long quirks);
/* ATS is supported */
#define IOMMU_FWSPEC_PCI_RC_ATS (1 << 0)
+++ /* CANWBS is supported */
+++ #define IOMMU_FWSPEC_PCI_RC_CANWBS (1 << 1)
/*
* An iommu attach handle represents a relationship between an iommu domain
struct iommu_dirty_bitmap {};
struct iommu_dirty_ops {};
- -static inline bool iommu_present(const struct bus_type *bus)
- -{
- - return false;
- -}
- -
static inline bool device_iommu_capable(struct device *dev, enum iommu_cap cap)
{
return false;
}
- -static inline struct iommu_domain *iommu_domain_alloc(const struct bus_type *bus)
+ +static inline struct iommu_domain *iommu_paging_domain_alloc_flags(struct device *dev,
+ + unsigned int flags)
{
- - return NULL;
+ + return ERR_PTR(-ENODEV);
}
static inline struct iommu_domain *iommu_paging_domain_alloc(struct device *dev)
* enforced on device attachment
* @IOMMU_HWPT_FAULT_ID_VALID: The fault_id field of hwpt allocation data is
* valid.
+ + * @IOMMU_HWPT_ALLOC_PASID: Requests a domain that can be used with PASID. The
+ + * domain can be attached to any PASID on the device.
+ + * Any domain attached to the non-PASID part of the
+ + * device must also be flaged, otherwise attaching a
+ + * PASID will blocked.
+ + * If IOMMU does not support PASID it will return
+ + * error (-EOPNOTSUPP).
*/
enum iommufd_hwpt_alloc_flags {
IOMMU_HWPT_ALLOC_NEST_PARENT = 1 << 0,
IOMMU_HWPT_ALLOC_DIRTY_TRACKING = 1 << 1,
IOMMU_HWPT_FAULT_ID_VALID = 1 << 2,
+ + IOMMU_HWPT_ALLOC_PASID = 1 << 3,
};
/**
__aligned_u64 ecap_reg;
};
+++ /**
+++ * struct iommu_hw_info_arm_smmuv3 - ARM SMMUv3 hardware information
+++ * (IOMMU_HW_INFO_TYPE_ARM_SMMUV3)
+++ *
+++ * @flags: Must be set to 0
+++ * @__reserved: Must be 0
+++ * @idr: Implemented features for ARM SMMU Non-secure programming interface
+++ * @iidr: Information about the implementation and implementer of ARM SMMU,
+++ * and architecture version supported
+++ * @aidr: ARM SMMU architecture version
+++ *
+++ * For the details of @idr, @iidr and @aidr, please refer to the chapters
+++ * from 6.3.1 to 6.3.6 in the SMMUv3 Spec.
+++ *
+++ * User space should read the underlying ARM SMMUv3 hardware information for
+++ * the list of supported features.
+++ *
+++ * Note that these values reflect the raw HW capability, without any insight if
+++ * any required kernel driver support is present. Bits may be set indicating the
+++ * HW has functionality that is lacking kernel software support, such as BTM. If
+++ * a VMM is using this information to construct emulated copies of these
+++ * registers it should only forward bits that it knows it can support.
+++ *
+++ * In future, presence of required kernel support will be indicated in flags.
+++ */
+++ struct iommu_hw_info_arm_smmuv3 {
+++ __u32 flags;
+++ __u32 __reserved;
+++ __u32 idr[6];
+++ __u32 iidr;
+++ __u32 aidr;
+++ };
+++
/**
* enum iommu_hw_info_type - IOMMU Hardware Info Types
* @IOMMU_HW_INFO_TYPE_NONE: Used by the drivers that do not report hardware
* info
* @IOMMU_HW_INFO_TYPE_INTEL_VTD: Intel VT-d iommu info type
+++ * @IOMMU_HW_INFO_TYPE_ARM_SMMUV3: ARM SMMUv3 iommu info type
*/
enum iommu_hw_info_type {
IOMMU_HW_INFO_TYPE_NONE = 0,
IOMMU_HW_INFO_TYPE_INTEL_VTD = 1,
+++ IOMMU_HW_INFO_TYPE_ARM_SMMUV3 = 2,
};
/**
projects / linux.git / commitdiff