select ARCH_HAS_CACHE_LINE_SIZE
select ARCH_HAS_CPU_CACHE_INVALIDATE_MEMREGION
select ARCH_HAS_CPU_FINALIZE_INIT
+++++++ select ARCH_HAS_CPU_PASID if IOMMU_SVA
select ARCH_HAS_CURRENT_STACK_POINTER
select ARCH_HAS_DEBUG_VIRTUAL
select ARCH_HAS_DEBUG_VM_PGTABLE if !X86_PAE
def_bool y
config ARCH_SUPPORTS_KEXEC_FILE
------- def_bool X86_64 && CRYPTO && CRYPTO_SHA256
+++++++ def_bool X86_64
config ARCH_SELECTS_KEXEC_FILE
def_bool y
select HAVE_IMA_KEXEC if IMA
config ARCH_SUPPORTS_KEXEC_PURGATORY
------- def_bool KEXEC_FILE
+++++++ def_bool y
config ARCH_SUPPORTS_KEXEC_SIG
def_bool y
LIST_HEAD(acpihid_map);
const struct iommu_ops amd_iommu_ops;
------ -const struct iommu_dirty_ops amd_dirty_ops;
++++++ +static const struct iommu_dirty_ops amd_dirty_ops;
int amd_iommu_max_glx_val = -1;
*
****************************************************************************/
++++++ +static inline bool pdom_is_v2_pgtbl_mode(struct protection_domain *pdom)
++++++ +{
++++++ + return (pdom && (pdom->flags & PD_IOMMUV2_MASK));
++++++ +}
++++++ +
static inline int get_acpihid_device_id(struct device *dev,
struct acpihid_map_entry **entry)
{
if (dev_data->domain)
detach_device(dev);
------- dev_iommu_priv_set(dev, NULL);
-------
/*
* We keep dev_data around for unplugged devices and reuse it when the
* device is re-plugged - not doing so would introduce a ton of races.
}
static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
------ - size_t size, u16 domid, int pde)
++++++ + size_t size, u16 domid,
++++++ + ioasid_t pasid, bool gn)
{
u64 inv_address = build_inv_address(address, size);
memset(cmd, 0, sizeof(*cmd));
++++++ +
cmd->data[1] |= domid;
cmd->data[2] = lower_32_bits(inv_address);
cmd->data[3] = upper_32_bits(inv_address);
++++++ + /* PDE bit - we want to flush everything, not only the PTEs */
++++++ + cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
++++++ + if (gn) {
++++++ + cmd->data[0] |= pasid;
++++++ + cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
++++++ + }
CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
------ - if (pde) /* PDE bit - we want to flush everything, not only the PTEs */
------ - cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
}
static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep,
------ - u64 address, size_t size)
++++++ + u64 address, size_t size,
++++++ + ioasid_t pasid, bool gn)
{
u64 inv_address = build_inv_address(address, size);
memset(cmd, 0, sizeof(*cmd));
++++++ +
cmd->data[0] = devid;
cmd->data[0] |= (qdep & 0xff) << 24;
cmd->data[1] = devid;
cmd->data[2] = lower_32_bits(inv_address);
cmd->data[3] = upper_32_bits(inv_address);
------ - CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
------ -}
------ -
------ -static void build_inv_iommu_pasid(struct iommu_cmd *cmd, u16 domid, u32 pasid,
------ - u64 address, bool size)
------ -{
------ - memset(cmd, 0, sizeof(*cmd));
------ -
------ - address &= ~(0xfffULL);
------ -
------ - cmd->data[0] = pasid;
------ - cmd->data[1] = domid;
------ - cmd->data[2] = lower_32_bits(address);
------ - cmd->data[3] = upper_32_bits(address);
------ - cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
------ - cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
------ - if (size)
------ - cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
------ - CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
------ -}
------ -
------ -static void build_inv_iotlb_pasid(struct iommu_cmd *cmd, u16 devid, u32 pasid,
------ - int qdep, u64 address, bool size)
------ -{
------ - memset(cmd, 0, sizeof(*cmd));
------ -
------ - address &= ~(0xfffULL);
++++++ + if (gn) {
++++++ + cmd->data[0] |= ((pasid >> 8) & 0xff) << 16;
++++++ + cmd->data[1] |= (pasid & 0xff) << 16;
++++++ + cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
++++++ + }
------ - cmd->data[0] = devid;
------ - cmd->data[0] |= ((pasid >> 8) & 0xff) << 16;
------ - cmd->data[0] |= (qdep & 0xff) << 24;
------ - cmd->data[1] = devid;
------ - cmd->data[1] |= (pasid & 0xff) << 16;
------ - cmd->data[2] = lower_32_bits(address);
------ - cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
------ - cmd->data[3] = upper_32_bits(address);
------ - if (size)
------ - cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
}
for (dom_id = 0; dom_id <= last_bdf; ++dom_id) {
struct iommu_cmd cmd;
build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
------ - dom_id, 1);
++++++ + dom_id, IOMMU_NO_PASID, false);
iommu_queue_command(iommu, &cmd);
}
struct iommu_cmd cmd;
build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
------ - dom_id, 1);
++++++ + dom_id, IOMMU_NO_PASID, false);
iommu_queue_command(iommu, &cmd);
iommu_completion_wait(iommu);
iommu_completion_wait(iommu);
}
------ -void iommu_flush_all_caches(struct amd_iommu *iommu)
++++++ +void amd_iommu_flush_all_caches(struct amd_iommu *iommu)
{
if (check_feature(FEATURE_IA)) {
amd_iommu_flush_all(iommu);
/*
* Command send function for flushing on-device TLB
*/
------ -static int device_flush_iotlb(struct iommu_dev_data *dev_data,
------ - u64 address, size_t size)
++++++ +static int device_flush_iotlb(struct iommu_dev_data *dev_data, u64 address,
++++++ + size_t size, ioasid_t pasid, bool gn)
{
struct amd_iommu *iommu;
struct iommu_cmd cmd;
if (!iommu)
return -EINVAL;
------ - build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size);
++++++ + build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address,
++++++ + size, pasid, gn);
return iommu_queue_command(iommu, &cmd);
}
return ret;
}
------ - if (dev_data->ats_enabled)
------ - ret = device_flush_iotlb(dev_data, 0, ~0UL);
++++++ + if (dev_data->ats_enabled) {
++++++ + /* Invalidate the entire contents of an IOTLB */
++++++ + ret = device_flush_iotlb(dev_data, 0, ~0UL,
++++++ + IOMMU_NO_PASID, false);
++++++ + }
return ret;
}
* page. Otherwise it flushes the whole TLB of the IOMMU.
*/
static void __domain_flush_pages(struct protection_domain *domain,
------ - u64 address, size_t size, int pde)
++++++ + u64 address, size_t size)
{
struct iommu_dev_data *dev_data;
struct iommu_cmd cmd;
int ret = 0, i;
++++++ + ioasid_t pasid = IOMMU_NO_PASID;
++++++ + bool gn = false;
++++++ +
++++++ + if (pdom_is_v2_pgtbl_mode(domain))
++++++ + gn = true;
------ - build_inv_iommu_pages(&cmd, address, size, domain->id, pde);
++++++ + build_inv_iommu_pages(&cmd, address, size, domain->id, pasid, gn);
for (i = 0; i < amd_iommu_get_num_iommus(); ++i) {
if (!domain->dev_iommu[i])
if (!dev_data->ats_enabled)
continue;
------ - ret |= device_flush_iotlb(dev_data, address, size);
++++++ + ret |= device_flush_iotlb(dev_data, address, size, pasid, gn);
}
WARN_ON(ret);
}
------ -static void domain_flush_pages(struct protection_domain *domain,
------ - u64 address, size_t size, int pde)
++++++ +void amd_iommu_domain_flush_pages(struct protection_domain *domain,
++++++ + u64 address, size_t size)
{
if (likely(!amd_iommu_np_cache)) {
------ - __domain_flush_pages(domain, address, size, pde);
++++++ + __domain_flush_pages(domain, address, size);
++++++ +
++++++ + /* Wait until IOMMU TLB and all device IOTLB flushes are complete */
++++++ + amd_iommu_domain_flush_complete(domain);
++++++ +
return;
}
flush_size = 1ul << min_alignment;
------ - __domain_flush_pages(domain, address, flush_size, pde);
++++++ + __domain_flush_pages(domain, address, flush_size);
address += flush_size;
size -= flush_size;
}
++++++ +
++++++ + /* Wait until IOMMU TLB and all device IOTLB flushes are complete */
++++++ + amd_iommu_domain_flush_complete(domain);
}
/* Flush the whole IO/TLB for a given protection domain - including PDE */
------ -void amd_iommu_domain_flush_tlb_pde(struct protection_domain *domain)
++++++ +static void amd_iommu_domain_flush_all(struct protection_domain *domain)
{
------ - domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1);
++++++ + amd_iommu_domain_flush_pages(domain, 0,
++++++ + CMD_INV_IOMMU_ALL_PAGES_ADDRESS);
}
void amd_iommu_domain_flush_complete(struct protection_domain *domain)
unsigned long flags;
spin_lock_irqsave(&domain->lock, flags);
------ - domain_flush_pages(domain, iova, size, 1);
------ - amd_iommu_domain_flush_complete(domain);
++++++ + amd_iommu_domain_flush_pages(domain, iova, size);
spin_unlock_irqrestore(&domain->lock, flags);
}
}
/* Flush the DTE entry */
device_flush_dte(dev_data);
------ - /* Flush IOTLB */
------ - amd_iommu_domain_flush_tlb_pde(domain);
------ -
------ - /* Wait for the flushes to finish */
------ - amd_iommu_domain_flush_complete(domain);
++++++ + /* Flush IOTLB and wait for the flushes to finish */
++++++ + amd_iommu_domain_flush_all(domain);
/* decrease reference counters - needs to happen after the flushes */
domain->dev_iommu[iommu->index] -= 1;
do_attach(dev_data, domain);
------ - /*
------ - * We might boot into a crash-kernel here. The crashed kernel
------ - * left the caches in the IOMMU dirty. So we have to flush
------ - * here to evict all dirty stuff.
------ - */
------ - amd_iommu_domain_flush_tlb_pde(domain);
------ -
------ - amd_iommu_domain_flush_complete(domain);
------ -
out:
spin_unlock(&dev_data->lock);
amd_iommu_update_and_flush_device_table(domain);
/* Flush domain TLB(s) and wait for completion */
------ - amd_iommu_domain_flush_tlb_pde(domain);
------ - amd_iommu_domain_flush_complete(domain);
++++++ + amd_iommu_domain_flush_all(domain);
}
/*****************************************************************************
}
/* Flush IOTLB to mark IOPTE dirty on the next translation(s) */
------ - if (domain_flush) {
------ - amd_iommu_domain_flush_tlb_pde(pdomain);
------ - amd_iommu_domain_flush_complete(pdomain);
------ - }
++++++ + if (domain_flush)
++++++ + amd_iommu_domain_flush_all(pdomain);
++++++ +
pdomain->dirty_tracking = enable;
spin_unlock_irqrestore(&pdomain->lock, flags);
unsigned long flags;
spin_lock_irqsave(&dom->lock, flags);
------ - amd_iommu_domain_flush_tlb_pde(dom);
------ - amd_iommu_domain_flush_complete(dom);
++++++ + amd_iommu_domain_flush_all(dom);
spin_unlock_irqrestore(&dom->lock, flags);
}
unsigned long flags;
spin_lock_irqsave(&dom->lock, flags);
------ - domain_flush_pages(dom, gather->start, gather->end - gather->start + 1, 1);
------ - amd_iommu_domain_flush_complete(dom);
++++++ + amd_iommu_domain_flush_pages(dom, gather->start,
++++++ + gather->end - gather->start + 1);
spin_unlock_irqrestore(&dom->lock, flags);
}
return true;
}
------ -const struct iommu_dirty_ops amd_dirty_ops = {
++++++ +static const struct iommu_dirty_ops amd_dirty_ops = {
.set_dirty_tracking = amd_iommu_set_dirty_tracking,
.read_and_clear_dirty = amd_iommu_read_and_clear_dirty,
};
};
static int __flush_pasid(struct protection_domain *domain, u32 pasid,
------ - u64 address, bool size)
++++++ + u64 address, size_t size)
{
struct iommu_dev_data *dev_data;
struct iommu_cmd cmd;
if (!(domain->flags & PD_IOMMUV2_MASK))
return -EINVAL;
------ - build_inv_iommu_pasid(&cmd, domain->id, pasid, address, size);
++++++ + build_inv_iommu_pages(&cmd, address, size, domain->id, pasid, true);
/*
* IOMMU TLB needs to be flushed before Device TLB to
iommu = rlookup_amd_iommu(dev_data->dev);
if (!iommu)
continue;
------ - build_inv_iotlb_pasid(&cmd, dev_data->devid, pasid,
------ - qdep, address, size);
++++++ + build_inv_iotlb_pages(&cmd, dev_data->devid, qdep,
++++++ + address, size, pasid, true);
ret = iommu_queue_command(iommu, &cmd);
if (ret != 0)
static int __amd_iommu_flush_page(struct protection_domain *domain, u32 pasid,
u64 address)
{
------ - return __flush_pasid(domain, pasid, address, false);
++++++ + return __flush_pasid(domain, pasid, address, PAGE_SIZE);
}
int amd_iommu_flush_page(struct iommu_domain *dom, u32 pasid,
static int __amd_iommu_flush_tlb(struct protection_domain *domain, u32 pasid)
{
------ - return __flush_pasid(domain, pasid, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
------ - true);
++++++ + return __flush_pasid(domain, pasid, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS);
}
int amd_iommu_flush_tlb(struct iommu_domain *dom, u32 pasid)
return index;
}
------ -static int modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index,
------ - struct irte_ga *irte)
++++++ +static int __modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index,
++++++ + struct irte_ga *irte)
{
struct irq_remap_table *table;
struct irte_ga *entry;
raw_spin_unlock_irqrestore(&table->lock, flags);
++++++ + return 0;
++++++ +}
++++++ +
++++++ +static int modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index,
++++++ + struct irte_ga *irte)
++++++ +{
++++++ + bool ret;
++++++ +
++++++ + ret = __modify_irte_ga(iommu, devid, index, irte);
++++++ + if (ret)
++++++ + return ret;
++++++ +
iommu_flush_irt_and_complete(iommu, devid);
return 0;
}
entry->lo.fields_vapic.is_run = is_run;
------ - return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid,
------ - ir_data->irq_2_irte.index, entry);
++++++ + return __modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid,
++++++ + ir_data->irq_2_irte.index, entry);
}
EXPORT_SYMBOL(amd_iommu_update_ga);
#endif
#define DART_T8020_TCR_BYPASS_DAPF BIT(12)
#define DART_T8020_TTBR 0x200
+ ++++++#define DART_T8020_USB4_TTBR 0x400
#define DART_T8020_TTBR_VALID BIT(31)
#define DART_T8020_TTBR_ADDR_FIELD_SHIFT 0
#define DART_T8020_TTBR_SHIFT 12
u32 command)
{
unsigned long flags;
- ------ int ret;
+ ++++++ int ret, i;
u32 command_reg;
spin_lock_irqsave(&stream_map->dart->lock, flags);
- ------ writel(stream_map->sidmap[0], stream_map->dart->regs + DART_T8020_STREAM_SELECT);
+ ++++++ for (i = 0; i < BITS_TO_U32(stream_map->dart->num_streams); i++)
+ ++++++ writel(stream_map->sidmap[i],
+ ++++++ stream_map->dart->regs + DART_T8020_STREAM_SELECT + 4 * i);
writel(command, stream_map->dart->regs + DART_T8020_STREAM_COMMAND);
ret = readl_poll_timeout_atomic(
{
struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
------- dev_iommu_priv_set(dev, NULL);
kfree(cfg);
}
ret = apple_dart_merge_master_cfg(group_master_cfg, cfg);
if (ret) {
- ------ dev_err(dev, "Failed to merge DART IOMMU grups.\n");
+ ++++++ dev_err(dev, "Failed to merge DART IOMMU groups.\n");
iommu_group_put(group);
res = ERR_PTR(ret);
goto out;
.ttbr_shift = DART_T8020_TTBR_SHIFT,
.ttbr_count = 4,
};
+ ++++++
+ ++++++static const struct apple_dart_hw apple_dart_hw_t8103_usb4 = {
+ ++++++ .type = DART_T8020,
+ ++++++ .irq_handler = apple_dart_t8020_irq,
+ ++++++ .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb,
+ ++++++ .oas = 36,
+ ++++++ .fmt = APPLE_DART,
+ ++++++ .max_sid_count = 64,
+ ++++++
+ ++++++ .enable_streams = DART_T8020_STREAMS_ENABLE,
+ ++++++ .lock = DART_T8020_CONFIG,
+ ++++++ .lock_bit = DART_T8020_CONFIG_LOCK,
+ ++++++
+ ++++++ .error = DART_T8020_ERROR,
+ ++++++
+ ++++++ .tcr = DART_T8020_TCR,
+ ++++++ .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE,
+ ++++++ .tcr_disabled = 0,
+ ++++++ .tcr_bypass = 0,
+ ++++++
+ ++++++ .ttbr = DART_T8020_USB4_TTBR,
+ ++++++ .ttbr_valid = DART_T8020_TTBR_VALID,
+ ++++++ .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT,
+ ++++++ .ttbr_shift = DART_T8020_TTBR_SHIFT,
+ ++++++ .ttbr_count = 4,
+ ++++++};
+ ++++++
static const struct apple_dart_hw apple_dart_hw_t6000 = {
.type = DART_T6000,
.irq_handler = apple_dart_t8020_irq,
unsigned int sid, idx;
for (sid = 0; sid < dart->num_streams; sid++) {
- ------ dart->save_tcr[sid] = readl_relaxed(dart->regs + DART_TCR(dart, sid));
+ ++++++ dart->save_tcr[sid] = readl(dart->regs + DART_TCR(dart, sid));
for (idx = 0; idx < dart->hw->ttbr_count; idx++)
dart->save_ttbr[sid][idx] =
readl(dart->regs + DART_TTBR(dart, sid, idx));
static const struct of_device_id apple_dart_of_match[] = {
{ .compatible = "apple,t8103-dart", .data = &apple_dart_hw_t8103 },
+ ++++++ { .compatible = "apple,t8103-usb4-dart", .data = &apple_dart_hw_t8103_usb4 },
{ .compatible = "apple,t8110-dart", .data = &apple_dart_hw_t8110 },
{ .compatible = "apple,t6000-dart", .data = &apple_dart_hw_t6000 },
{},
bool cd_live;
__le64 *cdptr;
struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
+++ ++++ struct arm_smmu_device *smmu = master->smmu;
if (WARN_ON(ssid >= (1 << cd_table->s1cdmax)))
return -E2BIG;
if (!cd) { /* (5) */
val = 0;
} else if (cd == &quiet_cd) { /* (4) */
+++ ++++ if (!(smmu->features & ARM_SMMU_FEAT_STALL_FORCE))
+++ ++++ val &= ~(CTXDESC_CD_0_S | CTXDESC_CD_0_R);
val |= CTXDESC_CD_0_TCR_EPD0;
} else if (cd_live) { /* (3) */
val &= ~CTXDESC_CD_0_ASID;
}
static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
--- ---- __le64 *dst)
+++ ++++ struct arm_smmu_ste *dst)
{
/*
* This is hideously complicated, but we only really care about
* 2. Write everything apart from dword 0, sync, write dword 0, sync
* 3. Update Config, sync
*/
--- ---- u64 val = le64_to_cpu(dst[0]);
+++ ++++ u64 val = le64_to_cpu(dst->data[0]);
bool ste_live = false;
--- ---- struct arm_smmu_device *smmu = NULL;
+++ ++++ struct arm_smmu_device *smmu = master->smmu;
struct arm_smmu_ctx_desc_cfg *cd_table = NULL;
struct arm_smmu_s2_cfg *s2_cfg = NULL;
--- ---- struct arm_smmu_domain *smmu_domain = NULL;
+++ ++++ struct arm_smmu_domain *smmu_domain = master->domain;
struct arm_smmu_cmdq_ent prefetch_cmd = {
.opcode = CMDQ_OP_PREFETCH_CFG,
.prefetch = {
},
};
--- ---- if (master) {
--- ---- smmu_domain = master->domain;
--- ---- smmu = master->smmu;
--- ---- }
--- ----
if (smmu_domain) {
switch (smmu_domain->stage) {
case ARM_SMMU_DOMAIN_S1:
cd_table = &master->cd_table;
break;
case ARM_SMMU_DOMAIN_S2:
--- ---- case ARM_SMMU_DOMAIN_NESTED:
s2_cfg = &smmu_domain->s2_cfg;
break;
default:
else
val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS);
--- ---- dst[0] = cpu_to_le64(val);
--- ---- dst[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
+++ ++++ dst->data[0] = cpu_to_le64(val);
+++ ++++ dst->data[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
STRTAB_STE_1_SHCFG_INCOMING));
--- ---- dst[2] = 0; /* Nuke the VMID */
+++ ++++ dst->data[2] = 0; /* Nuke the VMID */
/*
* The SMMU can perform negative caching, so we must sync
* the STE regardless of whether the old value was live.
STRTAB_STE_1_STRW_EL2 : STRTAB_STE_1_STRW_NSEL1;
BUG_ON(ste_live);
--- ---- dst[1] = cpu_to_le64(
+++ ++++ dst->data[1] = cpu_to_le64(
FIELD_PREP(STRTAB_STE_1_S1DSS, STRTAB_STE_1_S1DSS_SSID0) |
FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) |
FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) |
if (smmu->features & ARM_SMMU_FEAT_STALLS &&
!master->stall_enabled)
--- ---- dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD);
+++ ++++ dst->data[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD);
val |= (cd_table->cdtab_dma & STRTAB_STE_0_S1CTXPTR_MASK) |
FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS) |
if (s2_cfg) {
BUG_ON(ste_live);
--- ---- dst[2] = cpu_to_le64(
+++ ++++ dst->data[2] = cpu_to_le64(
FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) |
FIELD_PREP(STRTAB_STE_2_VTCR, s2_cfg->vtcr) |
#ifdef __BIG_ENDIAN
STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 |
STRTAB_STE_2_S2R);
--- ---- dst[3] = cpu_to_le64(s2_cfg->vttbr & STRTAB_STE_3_S2TTB_MASK);
+++ ++++ dst->data[3] = cpu_to_le64(s2_cfg->vttbr & STRTAB_STE_3_S2TTB_MASK);
val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS);
}
if (master->ats_enabled)
--- ---- dst[1] |= cpu_to_le64(FIELD_PREP(STRTAB_STE_1_EATS,
+++ ++++ dst->data[1] |= cpu_to_le64(FIELD_PREP(STRTAB_STE_1_EATS,
STRTAB_STE_1_EATS_TRANS));
arm_smmu_sync_ste_for_sid(smmu, sid);
/* See comment in arm_smmu_write_ctx_desc() */
--- ---- WRITE_ONCE(dst[0], cpu_to_le64(val));
+++ ++++ WRITE_ONCE(dst->data[0], cpu_to_le64(val));
arm_smmu_sync_ste_for_sid(smmu, sid);
/* It's likely that we'll want to use the new STE soon */
arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
}
--- ----static void arm_smmu_init_bypass_stes(__le64 *strtab, unsigned int nent, bool force)
+++ ++++static void arm_smmu_init_bypass_stes(struct arm_smmu_ste *strtab,
+++ ++++ unsigned int nent, bool force)
{
unsigned int i;
u64 val = STRTAB_STE_0_V;
val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS);
for (i = 0; i < nent; ++i) {
--- ---- strtab[0] = cpu_to_le64(val);
--- ---- strtab[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
--- ---- STRTAB_STE_1_SHCFG_INCOMING));
--- ---- strtab[2] = 0;
--- ---- strtab += STRTAB_STE_DWORDS;
+++ ++++ strtab->data[0] = cpu_to_le64(val);
+++ ++++ strtab->data[1] = cpu_to_le64(FIELD_PREP(
+++ ++++ STRTAB_STE_1_SHCFG, STRTAB_STE_1_SHCFG_INCOMING));
+++ ++++ strtab->data[2] = 0;
+++ ++++ strtab++;
}
}
fmt = ARM_64_LPAE_S1;
finalise_stage_fn = arm_smmu_domain_finalise_s1;
break;
--- ---- case ARM_SMMU_DOMAIN_NESTED:
case ARM_SMMU_DOMAIN_S2:
ias = smmu->ias;
oas = smmu->oas;
return 0;
}
--- ----static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
+++ ++++static struct arm_smmu_ste *
+++ ++++arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
{
--- ---- __le64 *step;
struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
--- ---- struct arm_smmu_strtab_l1_desc *l1_desc;
--- ---- int idx;
+++ ++++ unsigned int idx1, idx2;
/* Two-level walk */
--- ---- idx = (sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS;
--- ---- l1_desc = &cfg->l1_desc[idx];
--- ---- idx = (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS;
--- ---- step = &l1_desc->l2ptr[idx];
+++ ++++ idx1 = (sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS;
+++ ++++ idx2 = sid & ((1 << STRTAB_SPLIT) - 1);
+++ ++++ return &cfg->l1_desc[idx1].l2ptr[idx2];
} else {
/* Simple linear lookup */
--- ---- step = &cfg->strtab[sid * STRTAB_STE_DWORDS];
+++ ++++ return (struct arm_smmu_ste *)&cfg
+++ ++++ ->strtab[sid * STRTAB_STE_DWORDS];
}
--- ----
--- ---- return step;
}
static void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master)
for (i = 0; i < master->num_streams; ++i) {
u32 sid = master->streams[i].id;
--- ---- __le64 *step = arm_smmu_get_step_for_sid(smmu, sid);
+++ ++++ struct arm_smmu_ste *step =
+++ ++++ arm_smmu_get_step_for_sid(smmu, sid);
/* Bridged PCI devices may end up with duplicated IDs */
for (j = 0; j < i; j++)
struct arm_smmu_master *master;
struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
------- if (!fwspec || fwspec->ops != &arm_smmu_ops)
------- return ERR_PTR(-ENODEV);
-------
if (WARN_ON_ONCE(dev_iommu_priv_get(dev)))
return ERR_PTR(-EBUSY);
err_free_master:
kfree(master);
------- dev_iommu_priv_set(dev, NULL);
return ERR_PTR(ret);
}
if (smmu_domain->smmu)
ret = -EPERM;
else
--- ---- smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
+++ ++++ smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
mutex_unlock(&smmu_domain->init_mutex);
return ret;
iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
list_for_each_entry(e, &rmr_list, list) {
--- ---- __le64 *step;
+++ ++++ struct arm_smmu_ste *step;
struct iommu_iort_rmr_data *rmr;
int ret, i;
pm_runtime_put_autosuspend(smmu->dev);
}
+++ ++++static void arm_smmu_rpm_use_autosuspend(struct arm_smmu_device *smmu)
+++ ++++{
+++ ++++ /*
+++ ++++ * Setup an autosuspend delay to avoid bouncing runpm state.
+++ ++++ * Otherwise, if a driver for a suspended consumer device
+++ ++++ * unmaps buffers, it will runpm resume/suspend for each one.
+++ ++++ *
+++ ++++ * For example, when used by a GPU device, when an application
+++ ++++ * or game exits, it can trigger unmapping 100s or 1000s of
+++ ++++ * buffers. With a runpm cycle for each buffer, that adds up
+++ ++++ * to 5-10sec worth of reprogramming the context bank, while
+++ ++++ * the system appears to be locked up to the user.
+++ ++++ */
+++ ++++ pm_runtime_set_autosuspend_delay(smmu->dev, 20);
+++ ++++ pm_runtime_use_autosuspend(smmu->dev);
+++ ++++}
+++ ++++
static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
{
return container_of(dom, struct arm_smmu_domain, domain);
{
u32 fsr, fsynr, cbfrsynra;
unsigned long iova;
--- ---- struct iommu_domain *domain = dev;
--- ---- struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+++ ++++ struct arm_smmu_domain *smmu_domain = dev;
struct arm_smmu_device *smmu = smmu_domain->smmu;
int idx = smmu_domain->cfg.cbndx;
int ret;
iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
--- ---- ret = report_iommu_fault(domain, NULL, iova,
+++ ++++ ret = report_iommu_fault(&smmu_domain->domain, NULL, iova,
fsynr & ARM_SMMU_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ);
if (ret == -ENOSYS)
return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks);
}
--- ----static int arm_smmu_init_domain_context(struct iommu_domain *domain,
+++ ++++static int arm_smmu_init_domain_context(struct arm_smmu_domain *smmu_domain,
struct arm_smmu_device *smmu,
struct device *dev)
{
struct io_pgtable_ops *pgtbl_ops;
struct io_pgtable_cfg pgtbl_cfg;
enum io_pgtable_fmt fmt;
--- ---- struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+++ ++++ struct iommu_domain *domain = &smmu_domain->domain;
struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
irqreturn_t (*context_fault)(int irq, void *dev);
if (smmu_domain->smmu)
goto out_unlock;
--- ---- if (domain->type == IOMMU_DOMAIN_IDENTITY) {
--- ---- smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
--- ---- smmu_domain->smmu = smmu;
--- ---- goto out_unlock;
--- ---- }
--- ----
/*
* Mapping the requested stage onto what we support is surprisingly
* complicated, mainly because the spec allows S1+S2 SMMUs without
else
context_fault = arm_smmu_context_fault;
--- ---- ret = devm_request_irq(smmu->dev, irq, context_fault,
--- ---- IRQF_SHARED, "arm-smmu-context-fault", domain);
+++ ++++ ret = devm_request_irq(smmu->dev, irq, context_fault, IRQF_SHARED,
+++ ++++ "arm-smmu-context-fault", smmu_domain);
if (ret < 0) {
dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
cfg->irptndx, irq);
return ret;
}
--- ----static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
+++ ++++static void arm_smmu_destroy_domain_context(struct arm_smmu_domain *smmu_domain)
{
--- ---- struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
struct arm_smmu_device *smmu = smmu_domain->smmu;
struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
int ret, irq;
--- ---- if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
+++ ++++ if (!smmu)
return;
ret = arm_smmu_rpm_get(smmu);
if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
irq = smmu->irqs[cfg->irptndx];
--- ---- devm_free_irq(smmu->dev, irq, domain);
+++ ++++ devm_free_irq(smmu->dev, irq, smmu_domain);
}
free_io_pgtable_ops(smmu_domain->pgtbl_ops);
arm_smmu_rpm_put(smmu);
}
--- ----static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
+++ ++++static struct iommu_domain *arm_smmu_domain_alloc_paging(struct device *dev)
{
struct arm_smmu_domain *smmu_domain;
--- ---- if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_IDENTITY) {
--- ---- if (using_legacy_binding || type != IOMMU_DOMAIN_DMA)
--- ---- return NULL;
--- ---- }
/*
* Allocate the domain and initialise some of its data structures.
* We can't really do anything meaningful until we've added a
mutex_init(&smmu_domain->init_mutex);
spin_lock_init(&smmu_domain->cb_lock);
+++ ++++ if (dev) {
+++ ++++ struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
+++ ++++
+++ ++++ if (arm_smmu_init_domain_context(smmu_domain, cfg->smmu, dev)) {
+++ ++++ kfree(smmu_domain);
+++ ++++ return NULL;
+++ ++++ }
+++ ++++ }
+++ ++++
return &smmu_domain->domain;
}
* Free the domain resources. We assume that all devices have
* already been detached.
*/
--- ---- arm_smmu_destroy_domain_context(domain);
+++ ++++ arm_smmu_destroy_domain_context(smmu_domain);
kfree(smmu_domain);
}
mutex_unlock(&smmu->stream_map_mutex);
}
--- ----static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
--- ---- struct arm_smmu_master_cfg *cfg,
--- ---- struct iommu_fwspec *fwspec)
+++ ++++static void arm_smmu_master_install_s2crs(struct arm_smmu_master_cfg *cfg,
+++ ++++ enum arm_smmu_s2cr_type type,
+++ ++++ u8 cbndx, struct iommu_fwspec *fwspec)
{
--- ---- struct arm_smmu_device *smmu = smmu_domain->smmu;
+++ ++++ struct arm_smmu_device *smmu = cfg->smmu;
struct arm_smmu_s2cr *s2cr = smmu->s2crs;
--- ---- u8 cbndx = smmu_domain->cfg.cbndx;
--- ---- enum arm_smmu_s2cr_type type;
int i, idx;
--- ---- if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
--- ---- type = S2CR_TYPE_BYPASS;
--- ---- else
--- ---- type = S2CR_TYPE_TRANS;
--- ----
for_each_cfg_sme(cfg, fwspec, i, idx) {
if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
continue;
s2cr[idx].cbndx = cbndx;
arm_smmu_write_s2cr(smmu, idx);
}
--- ---- return 0;
}
static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
struct arm_smmu_device *smmu;
int ret;
------- if (!fwspec || fwspec->ops != &arm_smmu_ops) {
------- dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
------- return -ENXIO;
------- }
-------
/*
* FIXME: The arch/arm DMA API code tries to attach devices to its own
* domains between of_xlate() and probe_device() - we have no way to cope
return ret;
/* Ensure that the domain is finalised */
--- ---- ret = arm_smmu_init_domain_context(domain, smmu, dev);
+++ ++++ ret = arm_smmu_init_domain_context(smmu_domain, smmu, dev);
if (ret < 0)
goto rpm_put;
}
/* Looks ok, so add the device to the domain */
--- ---- ret = arm_smmu_domain_add_master(smmu_domain, cfg, fwspec);
--- ----
--- ---- /*
--- ---- * Setup an autosuspend delay to avoid bouncing runpm state.
--- ---- * Otherwise, if a driver for a suspended consumer device
--- ---- * unmaps buffers, it will runpm resume/suspend for each one.
--- ---- *
--- ---- * For example, when used by a GPU device, when an application
--- ---- * or game exits, it can trigger unmapping 100s or 1000s of
--- ---- * buffers. With a runpm cycle for each buffer, that adds up
--- ---- * to 5-10sec worth of reprogramming the context bank, while
--- ---- * the system appears to be locked up to the user.
--- ---- */
--- ---- pm_runtime_set_autosuspend_delay(smmu->dev, 20);
--- ---- pm_runtime_use_autosuspend(smmu->dev);
--- ----
+++ ++++ arm_smmu_master_install_s2crs(cfg, S2CR_TYPE_TRANS,
+++ ++++ smmu_domain->cfg.cbndx, fwspec);
+++ ++++ arm_smmu_rpm_use_autosuspend(smmu);
rpm_put:
arm_smmu_rpm_put(smmu);
return ret;
}
+++ ++++static int arm_smmu_attach_dev_type(struct device *dev,
+++ ++++ enum arm_smmu_s2cr_type type)
+++ ++++{
+++ ++++ struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
+++ ++++ struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+++ ++++ struct arm_smmu_device *smmu;
+++ ++++ int ret;
+++ ++++
+++ ++++ if (!cfg)
+++ ++++ return -ENODEV;
+++ ++++ smmu = cfg->smmu;
+++ ++++
+++ ++++ ret = arm_smmu_rpm_get(smmu);
+++ ++++ if (ret < 0)
+++ ++++ return ret;
+++ ++++
+++ ++++ arm_smmu_master_install_s2crs(cfg, type, 0, fwspec);
+++ ++++ arm_smmu_rpm_use_autosuspend(smmu);
+++ ++++ arm_smmu_rpm_put(smmu);
+++ ++++ return 0;
+++ ++++}
+++ ++++
+++ ++++static int arm_smmu_attach_dev_identity(struct iommu_domain *domain,
+++ ++++ struct device *dev)
+++ ++++{
+++ ++++ return arm_smmu_attach_dev_type(dev, S2CR_TYPE_BYPASS);
+++ ++++}
+++ ++++
+++ ++++static const struct iommu_domain_ops arm_smmu_identity_ops = {
+++ ++++ .attach_dev = arm_smmu_attach_dev_identity,
+++ ++++};
+++ ++++
+++ ++++static struct iommu_domain arm_smmu_identity_domain = {
+++ ++++ .type = IOMMU_DOMAIN_IDENTITY,
+++ ++++ .ops = &arm_smmu_identity_ops,
+++ ++++};
+++ ++++
+++ ++++static int arm_smmu_attach_dev_blocked(struct iommu_domain *domain,
+++ ++++ struct device *dev)
+++ ++++{
+++ ++++ return arm_smmu_attach_dev_type(dev, S2CR_TYPE_FAULT);
+++ ++++}
+++ ++++
+++ ++++static const struct iommu_domain_ops arm_smmu_blocked_ops = {
+++ ++++ .attach_dev = arm_smmu_attach_dev_blocked,
+++ ++++};
+++ ++++
+++ ++++static struct iommu_domain arm_smmu_blocked_domain = {
+++ ++++ .type = IOMMU_DOMAIN_BLOCKED,
+++ ++++ .ops = &arm_smmu_blocked_ops,
+++ ++++};
+++ ++++
static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t pgsize, size_t pgcount,
int prot, gfp_t gfp, size_t *mapped)
fwspec = dev_iommu_fwspec_get(dev);
if (ret)
goto out_free;
------- } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
------- smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
} else {
------- return ERR_PTR(-ENODEV);
+++++++ smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
}
ret = -EINVAL;
arm_smmu_rpm_put(cfg->smmu);
------- dev_iommu_priv_set(dev, NULL);
kfree(cfg);
}
}
static struct iommu_ops arm_smmu_ops = {
+++ ++++ .identity_domain = &arm_smmu_identity_domain,
+++ ++++ .blocked_domain = &arm_smmu_blocked_domain,
.capable = arm_smmu_capable,
--- ---- .domain_alloc = arm_smmu_domain_alloc,
+++ ++++ .domain_alloc_paging = arm_smmu_domain_alloc_paging,
.probe_device = arm_smmu_probe_device,
.release_device = arm_smmu_release_device,
.probe_finalize = arm_smmu_probe_finalize,
return err;
}
------- err = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev);
+++++++ err = iommu_device_register(&smmu->iommu, &arm_smmu_ops,
+++++++ using_legacy_binding ? NULL : dev);
if (err) {
dev_err(dev, "Failed to register iommu\n");
iommu_device_sysfs_remove(&smmu->iommu);
static const struct iommu_ops qcom_iommu_ops;
------- static struct qcom_iommu_dev * to_iommu(struct device *dev)
------- {
------- struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
-------
------- if (!fwspec || fwspec->ops != &qcom_iommu_ops)
------- return NULL;
-------
------- return dev_iommu_priv_get(dev);
------- }
-------
static struct qcom_iommu_ctx * to_ctx(struct qcom_iommu_domain *d, unsigned asid)
{
struct qcom_iommu_dev *qcom_iommu = d->iommu;
static int qcom_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
{
------- struct qcom_iommu_dev *qcom_iommu = to_iommu(dev);
+++++++ struct qcom_iommu_dev *qcom_iommu = dev_iommu_priv_get(dev);
struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
int ret;
struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
struct qcom_iommu_domain *qcom_domain;
struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
------- struct qcom_iommu_dev *qcom_iommu = to_iommu(dev);
+++++++ struct qcom_iommu_dev *qcom_iommu = dev_iommu_priv_get(dev);
unsigned int i;
if (domain == identity_domain || !domain)
static struct iommu_device *qcom_iommu_probe_device(struct device *dev)
{
------- struct qcom_iommu_dev *qcom_iommu = to_iommu(dev);
+++++++ struct qcom_iommu_dev *qcom_iommu = dev_iommu_priv_get(dev);
struct device_link *link;
if (!qcom_iommu)
static int __maybe_unused qcom_iommu_resume(struct device *dev)
{
struct qcom_iommu_dev *qcom_iommu = dev_get_drvdata(dev);
+++ ++++ int ret;
+++ ++++
+++ ++++ ret = clk_bulk_prepare_enable(CLK_NUM, qcom_iommu->clks);
+++ ++++ if (ret < 0)
+++ ++++ return ret;
+++ ++++
+++ ++++ if (dev->pm_domain)
+++ ++++ return qcom_scm_restore_sec_cfg(qcom_iommu->sec_id, 0);
--- ---- return clk_bulk_prepare_enable(CLK_NUM, qcom_iommu->clks);
+++ ++++ return ret;
}
static int __maybe_unused qcom_iommu_suspend(struct device *dev)
#define DEFAULT_DOMAIN_ADDRESS_WIDTH 57
----- --#define MAX_AGAW_WIDTH 64
----- --#define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
----- --
#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << ((gaw) - VTD_PAGE_SHIFT)) - 1)
#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << (gaw)) - 1)
#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
----- --/* page table handling */
----- --#define LEVEL_STRIDE (9)
----- --#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
----- --
----- --static inline int agaw_to_level(int agaw)
----- --{
----- -- return agaw + 2;
----- --}
----- --
----- --static inline int agaw_to_width(int agaw)
----- --{
----- -- return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
----- --}
----- --
----- --static inline int width_to_agaw(int width)
----- --{
----- -- return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
----- --}
----- --
----- --static inline unsigned int level_to_offset_bits(int level)
----- --{
----- -- return (level - 1) * LEVEL_STRIDE;
----- --}
----- --
----- --static inline int pfn_level_offset(u64 pfn, int level)
----- --{
----- -- return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
----- --}
----- --
----- --static inline u64 level_mask(int level)
----- --{
----- -- return -1ULL << level_to_offset_bits(level);
----- --}
----- --
----- --static inline u64 level_size(int level)
----- --{
----- -- return 1ULL << level_to_offset_bits(level);
----- --}
----- --
----- --static inline u64 align_to_level(u64 pfn, int level)
----- --{
----- -- return (pfn + level_size(level) - 1) & level_mask(level);
----- --}
----- --
----- --static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
----- --{
----- -- return 1UL << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
----- --}
----- --
----- --/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
----- -- are never going to work. */
----- --static inline unsigned long mm_to_dma_pfn_start(unsigned long mm_pfn)
----- --{
----- -- return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
----- --}
----- --static inline unsigned long mm_to_dma_pfn_end(unsigned long mm_pfn)
----- --{
----- -- return ((mm_pfn + 1) << (PAGE_SHIFT - VTD_PAGE_SHIFT)) - 1;
----- --}
----- --static inline unsigned long page_to_dma_pfn(struct page *pg)
----- --{
----- -- return mm_to_dma_pfn_start(page_to_pfn(pg));
----- --}
----- --static inline unsigned long virt_to_dma_pfn(void *p)
----- --{
----- -- return page_to_dma_pfn(virt_to_page(p));
----- --}
----- --
static void __init check_tylersburg_isoch(void);
static int rwbf_quirk;
return re->hi & VTD_PAGE_MASK;
}
----- --static inline void context_set_present(struct context_entry *context)
----- --{
----- -- context->lo |= 1;
----- --}
----- --
----- --static inline void context_set_fault_enable(struct context_entry *context)
----- --{
----- -- context->lo &= (((u64)-1) << 2) | 1;
----- --}
----- --
----- --static inline void context_set_translation_type(struct context_entry *context,
----- -- unsigned long value)
----- --{
----- -- context->lo &= (((u64)-1) << 4) | 3;
----- -- context->lo |= (value & 3) << 2;
----- --}
----- --
----- --static inline void context_set_address_root(struct context_entry *context,
----- -- unsigned long value)
----- --{
----- -- context->lo &= ~VTD_PAGE_MASK;
----- -- context->lo |= value & VTD_PAGE_MASK;
----- --}
----- --
----- --static inline void context_set_address_width(struct context_entry *context,
----- -- unsigned long value)
----- --{
----- -- context->hi |= value & 7;
----- --}
----- --
----- --static inline void context_set_domain_id(struct context_entry *context,
----- -- unsigned long value)
----- --{
----- -- context->hi |= (value & ((1 << 16) - 1)) << 8;
----- --}
----- --
----- --static inline void context_set_pasid(struct context_entry *context)
----- --{
----- -- context->lo |= CONTEXT_PASIDE;
----- --}
----- --
----- --static inline int context_domain_id(struct context_entry *c)
----- --{
----- -- return((c->hi >> 8) & 0xffff);
----- --}
----- --
----- --static inline void context_clear_entry(struct context_entry *context)
----- --{
----- -- context->lo = 0;
----- -- context->hi = 0;
----- --}
----- --
----- --static inline bool context_copied(struct intel_iommu *iommu, u8 bus, u8 devfn)
----- --{
----- -- if (!iommu->copied_tables)
----- -- return false;
----- --
----- -- return test_bit(((long)bus << 8) | devfn, iommu->copied_tables);
----- --}
----- --
----- --static inline void
----- --set_context_copied(struct intel_iommu *iommu, u8 bus, u8 devfn)
----- --{
----- -- set_bit(((long)bus << 8) | devfn, iommu->copied_tables);
----- --}
----- --
----- --static inline void
----- --clear_context_copied(struct intel_iommu *iommu, u8 bus, u8 devfn)
----- --{
----- -- clear_bit(((long)bus << 8) | devfn, iommu->copied_tables);
----- --}
----- --
/*
* This domain is a statically identity mapping domain.
* 1. This domain creats a static 1:1 mapping to all usable memory.
#define IDENTMAP_AZALIA 4
const struct iommu_ops intel_iommu_ops;
------ const struct iommu_dirty_ops intel_dirty_ops;
++++++ static const struct iommu_dirty_ops intel_dirty_ops;
static bool translation_pre_enabled(struct intel_iommu *iommu)
{
free_page((unsigned long)vaddr);
}
----- --static inline int domain_type_is_si(struct dmar_domain *domain)
+++++ ++static int domain_type_is_si(struct dmar_domain *domain)
{
return domain->domain.type == IOMMU_DOMAIN_IDENTITY;
}
----- --static inline int domain_pfn_supported(struct dmar_domain *domain,
----- -- unsigned long pfn)
+++++ ++static int domain_pfn_supported(struct dmar_domain *domain, unsigned long pfn)
{
int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
}
----- --static inline bool iommu_paging_structure_coherency(struct intel_iommu *iommu)
+++++ ++static bool iommu_paging_structure_coherency(struct intel_iommu *iommu)
{
return sm_supported(iommu) ?
ecap_smpwc(iommu->ecap) : ecap_coherent(iommu->ecap);
return false;
}
----- --struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
+++++ ++static struct intel_iommu *device_lookup_iommu(struct device *dev, u8 *bus, u8 *devfn)
{
struct dmar_drhd_unit *drhd = NULL;
struct pci_dev *pdev = NULL;
}
/* Notification for newly created mappings */
----- --static inline void __mapping_notify_one(struct intel_iommu *iommu,
----- -- struct dmar_domain *domain,
----- -- unsigned long pfn, unsigned int pages)
+++++ ++static void __mapping_notify_one(struct intel_iommu *iommu, struct dmar_domain *domain,
+++++ ++ unsigned long pfn, unsigned int pages)
{
/*
* It's a non-present to present mapping. Only flush if caching mode
spin_unlock(&iommu->lock);
}
----- --static inline int guestwidth_to_adjustwidth(int gaw)
+++++ ++static int guestwidth_to_adjustwidth(int gaw)
{
int agaw;
int r = (gaw - 12) % 9;
* Value of X in the PDTS field of a scalable mode context entry
* indicates PASID directory with 2^(X + 7) entries.
*/
----- --static inline unsigned long context_get_sm_pds(struct pasid_table *table)
+++++ ++static unsigned long context_get_sm_pds(struct pasid_table *table)
{
unsigned long pds, max_pde;
return pds - 7;
}
----- --/*
----- -- * Set the RID_PASID field of a scalable mode context entry. The
----- -- * IOMMU hardware will use the PASID value set in this field for
----- -- * DMA translations of DMA requests without PASID.
----- -- */
----- --static inline void
----- --context_set_sm_rid2pasid(struct context_entry *context, unsigned long pasid)
----- --{
----- -- context->hi |= pasid & ((1 << 20) - 1);
----- --}
----- --
----- --/*
----- -- * Set the DTE(Device-TLB Enable) field of a scalable mode context
----- -- * entry.
----- -- */
----- --static inline void context_set_sm_dte(struct context_entry *context)
----- --{
----- -- context->lo |= BIT_ULL(2);
----- --}
----- --
----- --/*
----- -- * Set the PRE(Page Request Enable) field of a scalable mode context
----- -- * entry.
----- -- */
----- --static inline void context_set_sm_pre(struct context_entry *context)
----- --{
----- -- context->lo |= BIT_ULL(4);
----- --}
----- --
----- --/* Convert value to context PASID directory size field coding. */
----- --#define context_pdts(pds) (((pds) & 0x7) << 9)
----- --
static int domain_context_mapping_one(struct dmar_domain *domain,
struct intel_iommu *iommu,
struct pasid_table *table,
static int
domain_context_mapping(struct dmar_domain *domain, struct device *dev)
{
+++++ ++ struct device_domain_info *info = dev_iommu_priv_get(dev);
struct domain_context_mapping_data data;
+++++ ++ struct intel_iommu *iommu = info->iommu;
+++++ ++ u8 bus = info->bus, devfn = info->devfn;
struct pasid_table *table;
----- -- struct intel_iommu *iommu;
----- -- u8 bus, devfn;
----- --
----- -- iommu = device_to_iommu(dev, &bus, &devfn);
----- -- if (!iommu)
----- -- return -ENODEV;
table = intel_pasid_get_table(dev);
}
/* Returns a number of VTD pages, but aligned to MM page size */
----- --static inline unsigned long aligned_nrpages(unsigned long host_addr,
----- -- size_t size)
+++++ ++static unsigned long aligned_nrpages(unsigned long host_addr, size_t size)
{
host_addr &= ~PAGE_MASK;
return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
}
/* Return largest possible superpage level for a given mapping */
----- --static inline int hardware_largepage_caps(struct dmar_domain *domain,
----- -- unsigned long iov_pfn,
----- -- unsigned long phy_pfn,
----- -- unsigned long pages)
+++++ ++static int hardware_largepage_caps(struct dmar_domain *domain, unsigned long iov_pfn,
+++++ ++ unsigned long phy_pfn, unsigned long pages)
{
int support, level = 1;
unsigned long pfnmerge;
attr |= DMA_FL_PTE_DIRTY;
}
++++++ domain->has_mappings = true;
++++++
pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | attr;
while (nr_pages > 0) {
struct device *dev)
{
struct device_domain_info *info = dev_iommu_priv_get(dev);
----- -- struct intel_iommu *iommu;
+++++ ++ struct intel_iommu *iommu = info->iommu;
unsigned long flags;
----- -- u8 bus, devfn;
int ret;
----- -- iommu = device_to_iommu(dev, &bus, &devfn);
----- -- if (!iommu)
----- -- return -ENODEV;
----- --
ret = domain_attach_iommu(domain, iommu);
if (ret)
return ret;
if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev)) {
/* Setup the PASID entry for requests without PASID: */
if (hw_pass_through && domain_type_is_si(domain))
----- -- ret = intel_pasid_setup_pass_through(iommu, domain,
+++++ ++ ret = intel_pasid_setup_pass_through(iommu,
dev, IOMMU_NO_PASID);
else if (domain->use_first_level)
ret = domain_setup_first_level(iommu, domain, dev,
return ret;
}
------ iommu_enable_pci_caps(info);
++++++ if (sm_supported(info->iommu) || !domain_type_is_si(info->domain))
++++++ iommu_enable_pci_caps(info);
return 0;
}
up_write(&dmar_global_lock);
}
----- --static inline struct intel_iommu *dev_to_intel_iommu(struct device *dev)
+++++ ++static struct intel_iommu *dev_to_intel_iommu(struct device *dev)
{
struct iommu_device *iommu_dev = dev_to_iommu_device(dev);
NULL,
};
----- --static inline bool has_external_pci(void)
+++++ ++static bool has_external_pci(void)
{
struct pci_dev *pdev = NULL;
*/
static void domain_context_clear(struct device_domain_info *info)
{
------ if (!info->iommu || !info->dev || !dev_is_pci(info->dev))
------ return;
++++++ if (!dev_is_pci(info->dev))
++++++ domain_context_clear_one(info, info->bus, info->devfn);
pci_for_each_dma_alias(to_pci_dev(info->dev),
&domain_context_clear_one_cb, info);
int prepare_domain_attach_device(struct iommu_domain *domain,
struct device *dev)
{
+++++ ++ struct device_domain_info *info = dev_iommu_priv_get(dev);
struct dmar_domain *dmar_domain = to_dmar_domain(domain);
----- -- struct intel_iommu *iommu;
+++++ ++ struct intel_iommu *iommu = info->iommu;
int addr_width;
----- -- iommu = device_to_iommu(dev, NULL, NULL);
----- -- if (!iommu)
----- -- return -ENODEV;
----- --
if (dmar_domain->force_snooping && !ecap_sc_support(iommu->ecap))
return -EINVAL;
return true;
spin_lock_irqsave(&dmar_domain->lock, flags);
------ if (!domain_support_force_snooping(dmar_domain)) {
++++++ if (!domain_support_force_snooping(dmar_domain) ||
++++++ (!dmar_domain->use_first_level && dmar_domain->has_mappings)) {
spin_unlock_irqrestore(&dmar_domain->lock, flags);
return false;
}
u8 bus, devfn;
int ret;
----- -- iommu = device_to_iommu(dev, &bus, &devfn);
+++++ ++ iommu = device_lookup_iommu(dev, &bus, &devfn);
if (!iommu || !iommu->iommu.ops)
return ERR_PTR(-ENODEV);
ret = intel_pasid_alloc_table(dev);
if (ret) {
dev_err(dev, "PASID table allocation failed\n");
------- dev_iommu_priv_set(dev, NULL);
kfree(info);
return ERR_PTR(ret);
}
dmar_remove_one_dev_info(dev);
intel_pasid_free_table(dev);
intel_iommu_debugfs_remove_dev(info);
------- dev_iommu_priv_set(dev, NULL);
kfree(info);
set_dma_ops(dev, NULL);
}
static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid)
{
----- -- struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL);
+++++ ++ struct device_domain_info *info = dev_iommu_priv_get(dev);
struct dev_pasid_info *curr, *dev_pasid = NULL;
+++++ ++ struct intel_iommu *iommu = info->iommu;
struct dmar_domain *dmar_domain;
struct iommu_domain *domain;
unsigned long flags;
goto out_free;
if (domain_type_is_si(dmar_domain))
----- -- ret = intel_pasid_setup_pass_through(iommu, dmar_domain,
----- -- dev, pasid);
+++++ ++ ret = intel_pasid_setup_pass_through(iommu, dev, pasid);
else if (dmar_domain->use_first_level)
ret = domain_setup_first_level(iommu, dmar_domain,
dev, pasid);
return 0;
}
------ const struct iommu_dirty_ops intel_dirty_ops = {
++++++ static const struct iommu_dirty_ops intel_dirty_ops = {
.set_dirty_tracking = intel_iommu_set_dirty_tracking,
.read_and_clear_dirty = intel_iommu_read_and_clear_dirty,
};
ver = (dev->device >> 8) & 0xff;
if (ver != 0x45 && ver != 0x46 && ver != 0x4c &&
ver != 0x4e && ver != 0x8a && ver != 0x98 &&
------ ver != 0x9a && ver != 0xa7)
++++++ ver != 0x9a && ver != 0xa7 && ver != 0x7d)
return;
if (risky_device(dev))
#define DMAR_ECEO_REG 0x408
#define DMAR_ECRSP_REG 0x410
#define DMAR_ECCAP_REG 0x430
----- --#define DMAR_VCCAP_REG 0xe30 /* Virtual command capability register */
----- --#define DMAR_VCMD_REG 0xe00 /* Virtual command register */
----- --#define DMAR_VCRSP_REG 0xe10 /* Virtual command response register */
#define DMAR_IQER_REG_IQEI(reg) FIELD_GET(GENMASK_ULL(3, 0), reg)
#define DMAR_IQER_REG_ITESID(reg) FIELD_GET(GENMASK_ULL(47, 32), reg)
*/
u8 dirty_tracking:1; /* Dirty tracking is enabled */
u8 nested_parent:1; /* Has other domains nested on it */
++++++ u8 has_mappings:1; /* Has mappings configured through
++++++ * iommu_map() interface.
++++++ */
spinlock_t lock; /* Protect device tracking lists */
struct list_head devices; /* all devices' list */
return (context->lo & 1);
}
+++++ ++#define LEVEL_STRIDE (9)
+++++ ++#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
+++++ ++#define MAX_AGAW_WIDTH (64)
+++++ ++#define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
+++++ ++
+++++ ++static inline int agaw_to_level(int agaw)
+++++ ++{
+++++ ++ return agaw + 2;
+++++ ++}
+++++ ++
+++++ ++static inline int agaw_to_width(int agaw)
+++++ ++{
+++++ ++ return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
+++++ ++}
+++++ ++
+++++ ++static inline int width_to_agaw(int width)
+++++ ++{
+++++ ++ return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
+++++ ++}
+++++ ++
+++++ ++static inline unsigned int level_to_offset_bits(int level)
+++++ ++{
+++++ ++ return (level - 1) * LEVEL_STRIDE;
+++++ ++}
+++++ ++
+++++ ++static inline int pfn_level_offset(u64 pfn, int level)
+++++ ++{
+++++ ++ return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
+++++ ++}
+++++ ++
+++++ ++static inline u64 level_mask(int level)
+++++ ++{
+++++ ++ return -1ULL << level_to_offset_bits(level);
+++++ ++}
+++++ ++
+++++ ++static inline u64 level_size(int level)
+++++ ++{
+++++ ++ return 1ULL << level_to_offset_bits(level);
+++++ ++}
+++++ ++
+++++ ++static inline u64 align_to_level(u64 pfn, int level)
+++++ ++{
+++++ ++ return (pfn + level_size(level) - 1) & level_mask(level);
+++++ ++}
+++++ ++
+++++ ++static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
+++++ ++{
+++++ ++ return 1UL << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
+++++ ++}
+++++ ++
+++++ ++/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
+++++ ++ are never going to work. */
+++++ ++static inline unsigned long mm_to_dma_pfn_start(unsigned long mm_pfn)
+++++ ++{
+++++ ++ return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
+++++ ++}
+++++ ++static inline unsigned long mm_to_dma_pfn_end(unsigned long mm_pfn)
+++++ ++{
+++++ ++ return ((mm_pfn + 1) << (PAGE_SHIFT - VTD_PAGE_SHIFT)) - 1;
+++++ ++}
+++++ ++static inline unsigned long page_to_dma_pfn(struct page *pg)
+++++ ++{
+++++ ++ return mm_to_dma_pfn_start(page_to_pfn(pg));
+++++ ++}
+++++ ++static inline unsigned long virt_to_dma_pfn(void *p)
+++++ ++{
+++++ ++ return page_to_dma_pfn(virt_to_page(p));
+++++ ++}
+++++ ++
+++++ ++static inline void context_set_present(struct context_entry *context)
+++++ ++{
+++++ ++ context->lo |= 1;
+++++ ++}
+++++ ++
+++++ ++static inline void context_set_fault_enable(struct context_entry *context)
+++++ ++{
+++++ ++ context->lo &= (((u64)-1) << 2) | 1;
+++++ ++}
+++++ ++
+++++ ++static inline void context_set_translation_type(struct context_entry *context,
+++++ ++ unsigned long value)
+++++ ++{
+++++ ++ context->lo &= (((u64)-1) << 4) | 3;
+++++ ++ context->lo |= (value & 3) << 2;
+++++ ++}
+++++ ++
+++++ ++static inline void context_set_address_root(struct context_entry *context,
+++++ ++ unsigned long value)
+++++ ++{
+++++ ++ context->lo &= ~VTD_PAGE_MASK;
+++++ ++ context->lo |= value & VTD_PAGE_MASK;
+++++ ++}
+++++ ++
+++++ ++static inline void context_set_address_width(struct context_entry *context,
+++++ ++ unsigned long value)
+++++ ++{
+++++ ++ context->hi |= value & 7;
+++++ ++}
+++++ ++
+++++ ++static inline void context_set_domain_id(struct context_entry *context,
+++++ ++ unsigned long value)
+++++ ++{
+++++ ++ context->hi |= (value & ((1 << 16) - 1)) << 8;
+++++ ++}
+++++ ++
+++++ ++static inline void context_set_pasid(struct context_entry *context)
+++++ ++{
+++++ ++ context->lo |= CONTEXT_PASIDE;
+++++ ++}
+++++ ++
+++++ ++static inline int context_domain_id(struct context_entry *c)
+++++ ++{
+++++ ++ return((c->hi >> 8) & 0xffff);
+++++ ++}
+++++ ++
+++++ ++static inline void context_clear_entry(struct context_entry *context)
+++++ ++{
+++++ ++ context->lo = 0;
+++++ ++ context->hi = 0;
+++++ ++}
+++++ ++
+++++ ++#ifdef CONFIG_INTEL_IOMMU
+++++ ++static inline bool context_copied(struct intel_iommu *iommu, u8 bus, u8 devfn)
+++++ ++{
+++++ ++ if (!iommu->copied_tables)
+++++ ++ return false;
+++++ ++
+++++ ++ return test_bit(((long)bus << 8) | devfn, iommu->copied_tables);
+++++ ++}
+++++ ++
+++++ ++static inline void
+++++ ++set_context_copied(struct intel_iommu *iommu, u8 bus, u8 devfn)
+++++ ++{
+++++ ++ set_bit(((long)bus << 8) | devfn, iommu->copied_tables);
+++++ ++}
+++++ ++
+++++ ++static inline void
+++++ ++clear_context_copied(struct intel_iommu *iommu, u8 bus, u8 devfn)
+++++ ++{
+++++ ++ clear_bit(((long)bus << 8) | devfn, iommu->copied_tables);
+++++ ++}
+++++ ++#endif /* CONFIG_INTEL_IOMMU */
+++++ ++
+++++ ++/*
+++++ ++ * Set the RID_PASID field of a scalable mode context entry. The
+++++ ++ * IOMMU hardware will use the PASID value set in this field for
+++++ ++ * DMA translations of DMA requests without PASID.
+++++ ++ */
+++++ ++static inline void
+++++ ++context_set_sm_rid2pasid(struct context_entry *context, unsigned long pasid)
+++++ ++{
+++++ ++ context->hi |= pasid & ((1 << 20) - 1);
+++++ ++}
+++++ ++
+++++ ++/*
+++++ ++ * Set the DTE(Device-TLB Enable) field of a scalable mode context
+++++ ++ * entry.
+++++ ++ */
+++++ ++static inline void context_set_sm_dte(struct context_entry *context)
+++++ ++{
+++++ ++ context->lo |= BIT_ULL(2);
+++++ ++}
+++++ ++
+++++ ++/*
+++++ ++ * Set the PRE(Page Request Enable) field of a scalable mode context
+++++ ++ * entry.
+++++ ++ */
+++++ ++static inline void context_set_sm_pre(struct context_entry *context)
+++++ ++{
+++++ ++ context->lo |= BIT_ULL(4);
+++++ ++}
+++++ ++
+++++ ++/* Convert value to context PASID directory size field coding. */
+++++ ++#define context_pdts(pds) (((pds) & 0x7) << 9)
+++++ ++
struct dmar_drhd_unit *dmar_find_matched_drhd_unit(struct pci_dev *dev);
int dmar_enable_qi(struct intel_iommu *iommu);
void *alloc_pgtable_page(int node, gfp_t gfp);
void free_pgtable_page(void *vaddr);
void iommu_flush_write_buffer(struct intel_iommu *iommu);
----- --struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn);
struct iommu_domain *intel_nested_domain_alloc(struct iommu_domain *parent,
const struct iommu_user_data *user_data);
rcu_read_unlock();
}
++++++ static void intel_flush_svm_all(struct intel_svm *svm)
++++++ {
++++++ struct device_domain_info *info;
++++++ struct intel_svm_dev *sdev;
++++++
++++++ rcu_read_lock();
++++++ list_for_each_entry_rcu(sdev, &svm->devs, list) {
++++++ info = dev_iommu_priv_get(sdev->dev);
++++++
++++++ qi_flush_piotlb(sdev->iommu, sdev->did, svm->pasid, 0, -1UL, 0);
++++++ if (info->ats_enabled) {
++++++ qi_flush_dev_iotlb_pasid(sdev->iommu, sdev->sid, info->pfsid,
++++++ svm->pasid, sdev->qdep,
++++++ 0, 64 - VTD_PAGE_SHIFT);
++++++ quirk_extra_dev_tlb_flush(info, 0, 64 - VTD_PAGE_SHIFT,
++++++ svm->pasid, sdev->qdep);
++++++ }
++++++ }
++++++ rcu_read_unlock();
++++++ }
++++++
/* Pages have been freed at this point */
static void intel_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn,
struct mm_struct *mm,
{
struct intel_svm *svm = container_of(mn, struct intel_svm, notifier);
++++++ if (start == 0 && end == -1UL) {
++++++ intel_flush_svm_all(svm);
++++++ return;
++++++ }
++++++
intel_flush_svm_range(svm, start,
(end - start + PAGE_SIZE - 1) >> VTD_PAGE_SHIFT, 0);
}
}
static int intel_svm_bind_mm(struct intel_iommu *iommu, struct device *dev,
------- struct mm_struct *mm)
+++++++ struct iommu_domain *domain, ioasid_t pasid)
{
struct device_domain_info *info = dev_iommu_priv_get(dev);
+++++++ struct mm_struct *mm = domain->mm;
struct intel_svm_dev *sdev;
struct intel_svm *svm;
unsigned long sflags;
int ret = 0;
------- svm = pasid_private_find(mm->pasid);
+++++++ svm = pasid_private_find(pasid);
if (!svm) {
svm = kzalloc(sizeof(*svm), GFP_KERNEL);
if (!svm)
return -ENOMEM;
------- svm->pasid = mm->pasid;
+++++++ svm->pasid = pasid;
svm->mm = mm;
INIT_LIST_HEAD_RCU(&svm->devs);
/* Setup the pasid table: */
sflags = cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0;
------- ret = intel_pasid_setup_first_level(iommu, dev, mm->pgd, mm->pasid,
+++++++ ret = intel_pasid_setup_first_level(iommu, dev, mm->pgd, pasid,
FLPT_DEFAULT_DID, sflags);
if (ret)
goto free_sdev;
free_svm:
if (list_empty(&svm->devs)) {
mmu_notifier_unregister(&svm->notifier, mm);
------- pasid_private_remove(mm->pasid);
+++++++ pasid_private_remove(pasid);
kfree(svm);
}
void intel_svm_remove_dev_pasid(struct device *dev, u32 pasid)
{
struct intel_svm_dev *sdev;
----- -- struct intel_iommu *iommu;
struct intel_svm *svm;
struct mm_struct *mm;
----- -- iommu = device_to_iommu(dev, NULL, NULL);
----- -- if (!iommu)
----- -- return;
----- --
if (pasid_to_svm_sdev(dev, pasid, &svm, &sdev))
return;
mm = svm->mm;
struct iommu_fault_event *evt,
struct iommu_page_response *msg)
{
+++++ ++ struct device_domain_info *info = dev_iommu_priv_get(dev);
+++++ ++ struct intel_iommu *iommu = info->iommu;
+++++ ++ u8 bus = info->bus, devfn = info->devfn;
struct iommu_fault_page_request *prm;
----- -- struct intel_iommu *iommu;
bool private_present;
bool pasid_present;
bool last_page;
----- -- u8 bus, devfn;
int ret = 0;
u16 sid;
----- -- if (!dev || !dev_is_pci(dev))
----- -- return -ENODEV;
----- --
----- -- iommu = device_to_iommu(dev, &bus, &devfn);
----- -- if (!iommu)
----- -- return -ENODEV;
----- --
----- -- if (!msg || !evt)
----- -- return -EINVAL;
----- --
prm = &evt->fault.prm;
sid = PCI_DEVID(bus, devfn);
pasid_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID;
{
struct device_domain_info *info = dev_iommu_priv_get(dev);
struct intel_iommu *iommu = info->iommu;
------- struct mm_struct *mm = domain->mm;
------- return intel_svm_bind_mm(iommu, dev, mm);
+++++++ return intel_svm_bind_mm(iommu, dev, domain, pasid);
}
static void intel_svm_domain_free(struct iommu_domain *domain)
hwpt->domain = NULL;
goto out_abort;
}
+++++++ hwpt->domain->owner = ops;
} else {
hwpt->domain = iommu_domain_alloc(idev->dev->bus);
if (!hwpt->domain) {
hwpt->domain = NULL;
goto out_abort;
}
+++++++ hwpt->domain->owner = ops;
if (WARN_ON_ONCE(hwpt->domain->type != IOMMU_DOMAIN_NESTED)) {
rc = -EINVAL;
if (ioas)
mutex_unlock(&ioas->mutex);
out_put_pt:
------- iommufd_put_object(pt_obj);
+++++++ iommufd_put_object(ucmd->ictx, pt_obj);
out_put_idev:
------- iommufd_put_object(&idev->obj);
+++++++ iommufd_put_object(ucmd->ictx, &idev->obj);
return rc;
}
rc = iopt_set_dirty_tracking(&ioas->iopt, hwpt_paging->common.domain,
enable);
------- iommufd_put_object(&hwpt_paging->common.obj);
+++++++ iommufd_put_object(ucmd->ictx, &hwpt_paging->common.obj);
return rc;
}
rc = iopt_read_and_clear_dirty_data(
&ioas->iopt, hwpt_paging->common.domain, cmd->flags, cmd);
------- iommufd_put_object(&hwpt_paging->common.obj);
+++++++ iommufd_put_object(ucmd->ictx, &hwpt_paging->common.obj);
return rc;
}
#include <linux/slab.h>
#include <linux/fsl/mc.h>
------- #define NO_IOMMU 1
-------
static int of_iommu_xlate(struct device *dev,
struct of_phandle_args *iommu_spec)
{
ops = iommu_ops_from_fwnode(fwnode);
if ((ops && !ops->of_xlate) ||
!of_device_is_available(iommu_spec->np))
------- return NO_IOMMU;
+++++++ return -ENODEV;
ret = iommu_fwspec_init(dev, &iommu_spec->np->fwnode, ops);
if (ret)
"iommu-map-mask", &iommu_spec.np,
iommu_spec.args);
if (err)
------- return err == -ENODEV ? NO_IOMMU : err;
+++++++ return err;
err = of_iommu_xlate(dev, &iommu_spec);
of_node_put(iommu_spec.np);
struct device *dev)
{
struct of_phandle_args iommu_spec;
------- int err = NO_IOMMU, idx = 0;
+++++++ int err = -ENODEV, idx = 0;
while (!of_parse_phandle_with_args(master_np, "iommus",
"#iommu-cells",
of_iommu_configure_dev(master_np, dev);
}
------- const struct iommu_ops *of_iommu_configure(struct device *dev,
------- struct device_node *master_np,
------- const u32 *id)
+++++++ /*
+++++++ * Returns:
+++++++ * 0 on success, an iommu was configured
+++++++ * -ENODEV if the device does not have any IOMMU
+++++++ * -EPROBEDEFER if probing should be tried again
+++++++ * -errno fatal errors
+++++++ */
+++++++ int of_iommu_configure(struct device *dev, struct device_node *master_np,
+++++++ const u32 *id)
{
------- const struct iommu_ops *ops = NULL;
------ struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
------ int err = NO_IOMMU;
++++++ struct iommu_fwspec *fwspec;
- int err = NO_IOMMU;
+++++++ int err;
if (!master_np)
------- return NULL;
+++++++ return -ENODEV;
++++++ /* Serialise to make dev->iommu stable under our potential fwspec */
++++++ mutex_lock(&iommu_probe_device_lock);
++++++ fwspec = dev_iommu_fwspec_get(dev);
if (fwspec) {
------ if (fwspec->ops)
------ return fwspec->ops;
------
++++++ if (fwspec->ops) {
++++++ mutex_unlock(&iommu_probe_device_lock);
- return fwspec->ops;
+++++++ return 0;
++++++ }
/* In the deferred case, start again from scratch */
iommu_fwspec_free(dev);
}
} else {
err = of_iommu_configure_device(master_np, dev, id);
}
-
- /*
- * Two success conditions can be represented by non-negative err here:
- * >0 : there is no IOMMU, or one was unavailable for non-fatal reasons
- * 0 : we found an IOMMU, and dev->fwspec is initialised appropriately
- * <0 : any actual error
- */
- if (!err) {
- /* The fwspec pointer changed, read it again */
- fwspec = dev_iommu_fwspec_get(dev);
- ops = fwspec->ops;
- }
++++++ mutex_unlock(&iommu_probe_device_lock);
------ /*
------ * Two success conditions can be represented by non-negative err here:
------ * >0 : there is no IOMMU, or one was unavailable for non-fatal reasons
------ * 0 : we found an IOMMU, and dev->fwspec is initialised appropriately
------ * <0 : any actual error
------ */
------ if (!err) {
------ /* The fwspec pointer changed, read it again */
------ fwspec = dev_iommu_fwspec_get(dev);
------ ops = fwspec->ops;
------ }
------- /*
------- * If we have reason to believe the IOMMU driver missed the initial
------- * probe for dev, replay it to get things in order.
------- */
------- if (!err && dev->bus)
------- err = iommu_probe_device(dev);
-------
------- /* Ignore all other errors apart from EPROBE_DEFER */
------- if (err == -EPROBE_DEFER) {
------- ops = ERR_PTR(err);
------- } else if (err < 0) {
------- dev_dbg(dev, "Adding to IOMMU failed: %d\n", err);
------- ops = NULL;
------- }
+++++++ if (err == -ENODEV || err == -EPROBE_DEFER)
+++++++ return err;
+++++++ if (err)
+++++++ goto err_log;
------- return ops;
+++++++ err = iommu_probe_device(dev);
+++++++ if (err)
+++++++ goto err_log;
+++++++ return 0;
+++++++
+++++++ err_log:
+++++++ dev_dbg(dev, "Adding to IOMMU failed: %pe\n", ERR_PTR(err));
+++++++ return err;
}
static enum iommu_resv_type __maybe_unused
if (start == phys->start && end == phys->end)
return IOMMU_RESV_DIRECT;
------- dev_warn(dev, "treating non-direct mapping [%pr] -> [%pap-%pap] as reservation\n", &phys,
+++++++ dev_warn(dev, "treating non-direct mapping [%pr] -> [%pap-%pap] as reservation\n", phys,
&start, &end);
return IOMMU_RESV_RESERVED;
}
phys_addr_t iova;
size_t length;
+++++++ if (of_dma_is_coherent(dev->of_node))
+++++++ prot |= IOMMU_CACHE;
+++++++
maps = of_translate_dma_region(np, maps, &iova, &length);
+++++++ if (length == 0) {
+++++++ dev_warn(dev, "Cannot reserve IOVA region of 0 size\n");
+++++++ continue;
+++++++ }
type = iommu_resv_region_get_type(dev, &phys, iova, length);
region = iommu_alloc_resv_region(iova, length, prot, type,
.flags = cpu_to_le32(flags),
};
---- --- ret = viommu_send_req_sync(vdomain->viommu, &map, sizeof(map));
++++ +++ ret = viommu_add_req(vdomain->viommu, &map, sizeof(map));
if (ret) {
viommu_del_mappings(vdomain, iova, end);
return ret;
viommu_sync_req(vdomain->viommu);
}
++++ +++static int viommu_iotlb_sync_map(struct iommu_domain *domain,
++++ +++ unsigned long iova, size_t size)
++++ +++{
++++ +++ struct viommu_domain *vdomain = to_viommu_domain(domain);
++++ +++
++++ +++ /*
++++ +++ * May be called before the viommu is initialized including
++++ +++ * while creating direct mapping
++++ +++ */
++++ +++ if (!vdomain->nr_endpoints)
++++ +++ return 0;
++++ +++ return viommu_sync_req(vdomain->viommu);
++++ +++}
++++ +++
++++ +++static void viommu_flush_iotlb_all(struct iommu_domain *domain)
++++ +++{
++++ +++ struct viommu_domain *vdomain = to_viommu_domain(domain);
++++ +++
++++ +++ /*
++++ +++ * May be called before the viommu is initialized including
++++ +++ * while creating direct mapping
++++ +++ */
++++ +++ if (!vdomain->nr_endpoints)
++++ +++ return;
++++ +++ viommu_sync_req(vdomain->viommu);
++++ +++}
++++ +++
static void viommu_get_resv_regions(struct device *dev, struct list_head *head)
{
struct iommu_resv_region *entry, *new_entry, *msi = NULL;
struct viommu_dev *viommu = NULL;
struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
------- if (!fwspec || fwspec->ops != &viommu_ops)
------- return ERR_PTR(-ENODEV);
-------
viommu = viommu_get_by_fwnode(fwspec->iommu_fwnode);
if (!viommu)
return ERR_PTR(-ENODEV);
switch (cap) {
case IOMMU_CAP_CACHE_COHERENCY:
return true;
++++ +++ case IOMMU_CAP_DEFERRED_FLUSH:
++++ +++ return true;
default:
return false;
}
.map_pages = viommu_map_pages,
.unmap_pages = viommu_unmap_pages,
.iova_to_phys = viommu_iova_to_phys,
++++ +++ .flush_iotlb_all = viommu_flush_iotlb_all,
.iotlb_sync = viommu_iotlb_sync,
++++ +++ .iotlb_sync_map = viommu_iotlb_sync_map,
.free = viommu_domain_free,
}
};
struct subsys_private;
struct device_node;
struct fwnode_handle;
------- struct iommu_ops;
struct iommu_group;
struct dev_pin_info;
struct dev_iommu;
{
mutex_unlock(&dev->mutex);
}
+++++++
+++++++DEFINE_GUARD(device, struct device *, device_lock(_T), device_unlock(_T))
static inline void device_lock_assert(struct device *dev)
{
area from being merged with adjacent virtual memory areas due to the
difference in their name.
-------config USERFAULTFD
------- bool "Enable userfaultfd() system call"
------- depends on MMU
------- help
------- Enable the userfaultfd() system call that allows to intercept and
------- handle page faults in userland.
-------
config HAVE_ARCH_USERFAULTFD_WP
bool
help
help
Arch has userfaultfd minor fault support
+++++++menuconfig USERFAULTFD
+++++++ bool "Enable userfaultfd() system call"
+++++++ depends on MMU
+++++++ help
+++++++ Enable the userfaultfd() system call that allows to intercept and
+++++++ handle page faults in userland.
+++++++
+++++++if USERFAULTFD
config PTE_MARKER_UFFD_WP
bool "Userfaultfd write protection support for shmem/hugetlbfs"
default y
Allows to create marker PTEs for userfaultfd write protection
purposes. It is required to enable userfaultfd write protection on
file-backed memory types like shmem and hugetlbfs.
+++++++endif # USERFAULTFD
# multi-gen LRU {
config LRU_GEN
bool
depends on !STACK_GROWSUP
+++++++ config IOMMU_MM_DATA
+++++++ bool
+++++++
source "mm/damon/Kconfig"
endmenu
projects / J-linux.git / commitdiff