diff options
Diffstat (limited to 'drivers/iommu')
-rw-r--r-- | drivers/iommu/Kconfig | 6 | ||||
-rw-r--r-- | drivers/iommu/amd_iommu.c | 30 | ||||
-rw-r--r-- | drivers/iommu/amd_iommu_init.c | 5 | ||||
-rw-r--r-- | drivers/iommu/amd_iommu_v2.c | 4 | ||||
-rw-r--r-- | drivers/iommu/arm-smmu-v3.c | 87 | ||||
-rw-r--r-- | drivers/iommu/arm-smmu.c | 45 | ||||
-rw-r--r-- | drivers/iommu/dmar.c | 2 | ||||
-rw-r--r-- | drivers/iommu/fsl_pamu.c | 26 | ||||
-rw-r--r-- | drivers/iommu/intel-iommu.c | 757 | ||||
-rw-r--r-- | drivers/iommu/intel_irq_remapping.c | 6 | ||||
-rw-r--r-- | drivers/iommu/io-pgtable-arm.c | 145 | ||||
-rw-r--r-- | drivers/iommu/io-pgtable.c | 5 | ||||
-rw-r--r-- | drivers/iommu/io-pgtable.h | 14 | ||||
-rw-r--r-- | drivers/iommu/iommu.c | 2 | ||||
-rw-r--r-- | drivers/iommu/iova.c | 120 | ||||
-rw-r--r-- | drivers/iommu/ipmmu-vmsa.c | 19 | ||||
-rw-r--r-- | drivers/iommu/irq_remapping.c | 2 | ||||
-rw-r--r-- | drivers/iommu/msm_iommu.c | 4 | ||||
-rw-r--r-- | drivers/iommu/of_iommu.c | 8 | ||||
-rw-r--r-- | drivers/iommu/omap-iommu-debug.c | 132 | ||||
-rw-r--r-- | drivers/iommu/omap-iommu.c | 198 | ||||
-rw-r--r-- | drivers/iommu/omap-iommu.h | 79 | ||||
-rw-r--r-- | drivers/iommu/omap-iopgtable.h | 27 | ||||
-rw-r--r-- | drivers/iommu/tegra-smmu.c | 306 |
24 files changed, 1052 insertions, 977 deletions
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index f1fb1d3ccc56..cbe6a890a93a 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -23,7 +23,7 @@ config IOMMU_IO_PGTABLE config IOMMU_IO_PGTABLE_LPAE bool "ARMv7/v8 Long Descriptor Format" select IOMMU_IO_PGTABLE - depends on ARM || ARM64 || COMPILE_TEST + depends on HAS_DMA && (ARM || ARM64 || COMPILE_TEST) help Enable support for the ARM long descriptor pagetable format. This allocator supports 4K/2M/1G, 16K/32M and 64K/512M page @@ -42,7 +42,7 @@ config IOMMU_IO_PGTABLE_LPAE_SELFTEST endmenu config IOMMU_IOVA - bool + tristate config OF_IOMMU def_bool y @@ -222,7 +222,7 @@ config TEGRA_IOMMU_SMMU select IOMMU_API help This driver supports the IOMMU hardware (SMMU) found on NVIDIA Tegra - SoCs (Tegra30 up to Tegra132). + SoCs (Tegra30 up to Tegra210). config EXYNOS_IOMMU bool "Exynos IOMMU Support" diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 658ee39e6569..08d2775887f7 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -1835,8 +1835,8 @@ static void free_gcr3_table(struct protection_domain *domain) free_gcr3_tbl_level2(domain->gcr3_tbl); else if (domain->glx == 1) free_gcr3_tbl_level1(domain->gcr3_tbl); - else if (domain->glx != 0) - BUG(); + else + BUG_ON(domain->glx != 0); free_page((unsigned long)domain->gcr3_tbl); } @@ -2006,6 +2006,15 @@ static void do_detach(struct iommu_dev_data *dev_data) { struct amd_iommu *iommu; + /* + * First check if the device is still attached. It might already + * be detached from its domain because the generic + * iommu_detach_group code detached it and we try again here in + * our alias handling. + */ + if (!dev_data->domain) + return; + iommu = amd_iommu_rlookup_table[dev_data->devid]; /* decrease reference counters */ @@ -3947,11 +3956,6 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, if (ret < 0) return ret; - ret = -ENOMEM; - data = kzalloc(sizeof(*data), GFP_KERNEL); - if (!data) - goto out_free_parent; - if (info->type == X86_IRQ_ALLOC_TYPE_IOAPIC) { if (get_irq_table(devid, true)) index = info->ioapic_pin; @@ -3962,7 +3966,6 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, } if (index < 0) { pr_warn("Failed to allocate IRTE\n"); - kfree(data); goto out_free_parent; } @@ -3974,17 +3977,18 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, goto out_free_data; } - if (i > 0) { - data = kzalloc(sizeof(*data), GFP_KERNEL); - if (!data) - goto out_free_data; - } + ret = -ENOMEM; + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) + goto out_free_data; + irq_data->hwirq = (devid << 16) + i; irq_data->chip_data = data; irq_data->chip = &amd_ir_chip; irq_remapping_prepare_irte(data, cfg, info, devid, index, i); irq_set_status_flags(virq + i, IRQ_MOVE_PCNTXT); } + return 0; out_free_data: diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index a24495eb4e26..1b066e7d144d 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -154,7 +154,7 @@ bool amd_iommu_iotlb_sup __read_mostly = true; u32 amd_iommu_max_pasid __read_mostly = ~0; bool amd_iommu_v2_present __read_mostly; -bool amd_iommu_pc_present __read_mostly; +static bool amd_iommu_pc_present __read_mostly; bool amd_iommu_force_isolation __read_mostly; @@ -1256,6 +1256,9 @@ static int iommu_init_pci(struct amd_iommu *iommu) if (!iommu->dev) return -ENODEV; + /* Prevent binding other PCI device drivers to IOMMU devices */ + iommu->dev->match_driver = false; + pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET, &iommu->cap); pci_read_config_dword(iommu->dev, cap_ptr + MMIO_RANGE_OFFSET, diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c index f7b875bb70d4..1131664b918b 100644 --- a/drivers/iommu/amd_iommu_v2.c +++ b/drivers/iommu/amd_iommu_v2.c @@ -356,8 +356,8 @@ static void free_pasid_states(struct device_state *dev_state) free_pasid_states_level2(dev_state->states); else if (dev_state->pasid_levels == 1) free_pasid_states_level1(dev_state->states); - else if (dev_state->pasid_levels != 0) - BUG(); + else + BUG_ON(dev_state->pasid_levels != 0); free_page((unsigned long)dev_state->states); } diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index da902baaa794..286e890e7d64 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -56,6 +56,7 @@ #define IDR0_TTF_SHIFT 2 #define IDR0_TTF_MASK 0x3 #define IDR0_TTF_AARCH64 (2 << IDR0_TTF_SHIFT) +#define IDR0_TTF_AARCH32_64 (3 << IDR0_TTF_SHIFT) #define IDR0_S1P (1 << 1) #define IDR0_S2P (1 << 0) @@ -118,6 +119,7 @@ #define ARM_SMMU_IRQ_CTRL 0x50 #define IRQ_CTRL_EVTQ_IRQEN (1 << 2) +#define IRQ_CTRL_PRIQ_IRQEN (1 << 1) #define IRQ_CTRL_GERROR_IRQEN (1 << 0) #define ARM_SMMU_IRQ_CTRLACK 0x54 @@ -173,14 +175,14 @@ #define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc /* Common MSI config fields */ -#define MSI_CFG0_SH_SHIFT 60 -#define MSI_CFG0_SH_NSH (0UL << MSI_CFG0_SH_SHIFT) -#define MSI_CFG0_SH_OSH (2UL << MSI_CFG0_SH_SHIFT) -#define MSI_CFG0_SH_ISH (3UL << MSI_CFG0_SH_SHIFT) -#define MSI_CFG0_MEMATTR_SHIFT 56 -#define MSI_CFG0_MEMATTR_DEVICE_nGnRE (0x1 << MSI_CFG0_MEMATTR_SHIFT) #define MSI_CFG0_ADDR_SHIFT 2 #define MSI_CFG0_ADDR_MASK 0x3fffffffffffUL +#define MSI_CFG2_SH_SHIFT 4 +#define MSI_CFG2_SH_NSH (0UL << MSI_CFG2_SH_SHIFT) +#define MSI_CFG2_SH_OSH (2UL << MSI_CFG2_SH_SHIFT) +#define MSI_CFG2_SH_ISH (3UL << MSI_CFG2_SH_SHIFT) +#define MSI_CFG2_MEMATTR_SHIFT 0 +#define MSI_CFG2_MEMATTR_DEVICE_nGnRE (0x1 << MSI_CFG2_MEMATTR_SHIFT) #define Q_IDX(q, p) ((p) & ((1 << (q)->max_n_shift) - 1)) #define Q_WRP(q, p) ((p) & (1 << (q)->max_n_shift)) @@ -341,7 +343,8 @@ #define CMDQ_TLBI_0_VMID_SHIFT 32 #define CMDQ_TLBI_0_ASID_SHIFT 48 #define CMDQ_TLBI_1_LEAF (1UL << 0) -#define CMDQ_TLBI_1_ADDR_MASK ~0xfffUL +#define CMDQ_TLBI_1_VA_MASK ~0xfffUL +#define CMDQ_TLBI_1_IPA_MASK 0xfffffffff000UL #define CMDQ_PRI_0_SSID_SHIFT 12 #define CMDQ_PRI_0_SSID_MASK 0xfffffUL @@ -769,11 +772,13 @@ static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent) break; case CMDQ_OP_TLBI_NH_VA: cmd[0] |= (u64)ent->tlbi.asid << CMDQ_TLBI_0_ASID_SHIFT; - /* Fallthrough */ + cmd[1] |= ent->tlbi.leaf ? CMDQ_TLBI_1_LEAF : 0; + cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK; + break; case CMDQ_OP_TLBI_S2_IPA: cmd[0] |= (u64)ent->tlbi.vmid << CMDQ_TLBI_0_VMID_SHIFT; cmd[1] |= ent->tlbi.leaf ? CMDQ_TLBI_1_LEAF : 0; - cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_ADDR_MASK; + cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK; break; case CMDQ_OP_TLBI_NH_ASID: cmd[0] |= (u64)ent->tlbi.asid << CMDQ_TLBI_0_ASID_SHIFT; @@ -1330,33 +1335,10 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, arm_smmu_cmdq_issue_cmd(smmu, &cmd); } -static void arm_smmu_flush_pgtable(void *addr, size_t size, void *cookie) -{ - struct arm_smmu_domain *smmu_domain = cookie; - struct arm_smmu_device *smmu = smmu_domain->smmu; - unsigned long offset = (unsigned long)addr & ~PAGE_MASK; - - if (smmu->features & ARM_SMMU_FEAT_COHERENCY) { - dsb(ishst); - } else { - dma_addr_t dma_addr; - struct device *dev = smmu->dev; - - dma_addr = dma_map_page(dev, virt_to_page(addr), offset, size, - DMA_TO_DEVICE); - - if (dma_mapping_error(dev, dma_addr)) - dev_err(dev, "failed to flush pgtable at %p\n", addr); - else - dma_unmap_page(dev, dma_addr, size, DMA_TO_DEVICE); - } -} - static struct iommu_gather_ops arm_smmu_gather_ops = { .tlb_flush_all = arm_smmu_tlb_inv_context, .tlb_add_flush = arm_smmu_tlb_inv_range_nosync, .tlb_sync = arm_smmu_tlb_sync, - .flush_pgtable = arm_smmu_flush_pgtable, }; /* IOMMU API */ @@ -1531,6 +1513,7 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain) .ias = ias, .oas = oas, .tlb = &arm_smmu_gather_ops, + .iommu_dev = smmu->dev, }; pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain); @@ -2053,9 +2036,17 @@ static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu) int ret; struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; - /* Calculate the L1 size, capped to the SIDSIZE */ - size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3); - size = min(size, smmu->sid_bits - STRTAB_SPLIT); + /* + * If we can resolve everything with a single L2 table, then we + * just need a single L1 descriptor. Otherwise, calculate the L1 + * size, capped to the SIDSIZE. + */ + if (smmu->sid_bits < STRTAB_SPLIT) { + size = 0; + } else { + size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3); + size = min(size, smmu->sid_bits - STRTAB_SPLIT); + } cfg->num_l1_ents = 1 << size; size += STRTAB_SPLIT; @@ -2198,6 +2189,7 @@ static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val, static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu) { int ret, irq; + u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN; /* Disable IRQs first */ ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL, @@ -2252,13 +2244,13 @@ static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu) if (IS_ERR_VALUE(ret)) dev_warn(smmu->dev, "failed to enable priq irq\n"); + else + irqen_flags |= IRQ_CTRL_PRIQ_IRQEN; } } /* Enable interrupt generation on the SMMU */ - ret = arm_smmu_write_reg_sync(smmu, - IRQ_CTRL_EVTQ_IRQEN | - IRQ_CTRL_GERROR_IRQEN, + ret = arm_smmu_write_reg_sync(smmu, irqen_flags, ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK); if (ret) dev_warn(smmu->dev, "failed to enable irqs\n"); @@ -2472,7 +2464,13 @@ static int arm_smmu_device_probe(struct arm_smmu_device *smmu) } /* We only support the AArch64 table format at present */ - if ((reg & IDR0_TTF_MASK << IDR0_TTF_SHIFT) < IDR0_TTF_AARCH64) { + switch (reg & IDR0_TTF_MASK << IDR0_TTF_SHIFT) { + case IDR0_TTF_AARCH32_64: + smmu->ias = 40; + /* Fallthrough */ + case IDR0_TTF_AARCH64: + break; + default: dev_err(smmu->dev, "AArch64 table format not supported!\n"); return -ENXIO; } @@ -2540,12 +2538,12 @@ static int arm_smmu_device_probe(struct arm_smmu_device *smmu) case IDR5_OAS_44_BIT: smmu->oas = 44; break; + default: + dev_info(smmu->dev, + "unknown output address size. Truncating to 48-bit\n"); + /* Fallthrough */ case IDR5_OAS_48_BIT: smmu->oas = 48; - break; - default: - dev_err(smmu->dev, "unknown output address size!\n"); - return -ENXIO; } /* Set the DMA mask for our table walker */ @@ -2553,8 +2551,7 @@ static int arm_smmu_device_probe(struct arm_smmu_device *smmu) dev_warn(smmu->dev, "failed to set DMA mask for table walker\n"); - if (!smmu->ias) - smmu->ias = smmu->oas; + smmu->ias = max(smmu->ias, smmu->oas); dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n", smmu->ias, smmu->oas, smmu->features); diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 4cd0c29cb585..48a39dfa9777 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -37,6 +37,7 @@ #include <linux/iopoll.h> #include <linux/module.h> #include <linux/of.h> +#include <linux/of_address.h> #include <linux/pci.h> #include <linux/platform_device.h> #include <linux/slab.h> @@ -607,34 +608,10 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, } } -static void arm_smmu_flush_pgtable(void *addr, size_t size, void *cookie) -{ - struct arm_smmu_domain *smmu_domain = cookie; - struct arm_smmu_device *smmu = smmu_domain->smmu; - unsigned long offset = (unsigned long)addr & ~PAGE_MASK; - - - /* Ensure new page tables are visible to the hardware walker */ - if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) { - dsb(ishst); - } else { - /* - * If the SMMU can't walk tables in the CPU caches, treat them - * like non-coherent DMA since we need to flush the new entries - * all the way out to memory. There's no possibility of - * recursion here as the SMMU table walker will not be wired - * through another SMMU. - */ - dma_map_page(smmu->dev, virt_to_page(addr), offset, size, - DMA_TO_DEVICE); - } -} - static struct iommu_gather_ops arm_smmu_gather_ops = { .tlb_flush_all = arm_smmu_tlb_inv_context, .tlb_add_flush = arm_smmu_tlb_inv_range_nosync, .tlb_sync = arm_smmu_tlb_sync, - .flush_pgtable = arm_smmu_flush_pgtable, }; static irqreturn_t arm_smmu_context_fault(int irq, void *dev) @@ -898,6 +875,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, .ias = ias, .oas = oas, .tlb = &arm_smmu_gather_ops, + .iommu_dev = smmu->dev, }; smmu_domain->smmu = smmu; @@ -1532,6 +1510,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) unsigned long size; void __iomem *gr0_base = ARM_SMMU_GR0(smmu); u32 id; + bool cttw_dt, cttw_reg; dev_notice(smmu->dev, "probing hardware configuration...\n"); dev_notice(smmu->dev, "SMMUv%d with:\n", smmu->version); @@ -1571,10 +1550,22 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) dev_notice(smmu->dev, "\taddress translation ops\n"); } - if (id & ID0_CTTW) { + /* + * In order for DMA API calls to work properly, we must defer to what + * the DT says about coherency, regardless of what the hardware claims. + * Fortunately, this also opens up a workaround for systems where the + * ID register value has ended up configured incorrectly. + */ + cttw_dt = of_dma_is_coherent(smmu->dev->of_node); + cttw_reg = !!(id & ID0_CTTW); + if (cttw_dt) smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK; - dev_notice(smmu->dev, "\tcoherent table walk\n"); - } + if (cttw_dt || cttw_reg) + dev_notice(smmu->dev, "\t%scoherent table walk\n", + cttw_dt ? "" : "non-"); + if (cttw_dt != cttw_reg) + dev_notice(smmu->dev, + "\t(IDR0.CTTW overridden by dma-coherent property)\n"); if (id & ID0_SMS) { u32 smr, sid, mask; diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index c9db04d4ef39..8757f8dfc4e5 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -1068,7 +1068,7 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd) if (intel_iommu_enabled) iommu->iommu_dev = iommu_device_create(NULL, iommu, intel_iommu_groups, - iommu->name); + "%s", iommu->name); return 0; diff --git a/drivers/iommu/fsl_pamu.c b/drivers/iommu/fsl_pamu.c index abeedc9a78c2..2570f2a25dc4 100644 --- a/drivers/iommu/fsl_pamu.c +++ b/drivers/iommu/fsl_pamu.c @@ -41,7 +41,6 @@ struct pamu_isr_data { static struct paace *ppaact; static struct paace *spaact; -static struct ome *omt __initdata; /* * Table for matching compatible strings, for device tree @@ -50,7 +49,7 @@ static struct ome *omt __initdata; * SOCs. For the older SOCs "fsl,qoriq-device-config-1.0" * string would be used. */ -static const struct of_device_id guts_device_ids[] __initconst = { +static const struct of_device_id guts_device_ids[] = { { .compatible = "fsl,qoriq-device-config-1.0", }, { .compatible = "fsl,qoriq-device-config-2.0", }, {} @@ -599,7 +598,7 @@ found_cpu_node: * Memory accesses to QMAN and BMAN private memory need not be coherent, so * clear the PAACE entry coherency attribute for them. */ -static void __init setup_qbman_paace(struct paace *ppaace, int paace_type) +static void setup_qbman_paace(struct paace *ppaace, int paace_type) { switch (paace_type) { case QMAN_PAACE: @@ -629,7 +628,7 @@ static void __init setup_qbman_paace(struct paace *ppaace, int paace_type) * this table to translate device transaction to appropriate corenet * transaction. */ -static void __init setup_omt(struct ome *omt) +static void setup_omt(struct ome *omt) { struct ome *ome; @@ -666,7 +665,7 @@ static void __init setup_omt(struct ome *omt) * Get the maximum number of PAACT table entries * and subwindows supported by PAMU */ -static void __init get_pamu_cap_values(unsigned long pamu_reg_base) +static void get_pamu_cap_values(unsigned long pamu_reg_base) { u32 pc_val; @@ -676,9 +675,9 @@ static void __init get_pamu_cap_values(unsigned long pamu_reg_base) } /* Setup PAMU registers pointing to PAACT, SPAACT and OMT */ -static int __init setup_one_pamu(unsigned long pamu_reg_base, unsigned long pamu_reg_size, - phys_addr_t ppaact_phys, phys_addr_t spaact_phys, - phys_addr_t omt_phys) +static int setup_one_pamu(unsigned long pamu_reg_base, unsigned long pamu_reg_size, + phys_addr_t ppaact_phys, phys_addr_t spaact_phys, + phys_addr_t omt_phys) { u32 *pc; struct pamu_mmap_regs *pamu_regs; @@ -720,7 +719,7 @@ static int __init setup_one_pamu(unsigned long pamu_reg_base, unsigned long pamu } /* Enable all device LIODNS */ -static void __init setup_liodns(void) +static void setup_liodns(void) { int i, len; struct paace *ppaace; @@ -846,7 +845,7 @@ struct ccsr_law { /* * Create a coherence subdomain for a given memory block. */ -static int __init create_csd(phys_addr_t phys, size_t size, u32 csd_port_id) +static int create_csd(phys_addr_t phys, size_t size, u32 csd_port_id) { struct device_node *np; const __be32 *iprop; @@ -988,7 +987,7 @@ error: static const struct { u32 svr; u32 port_id; -} port_id_map[] __initconst = { +} port_id_map[] = { {(SVR_P2040 << 8) | 0x10, 0xFF000000}, /* P2040 1.0 */ {(SVR_P2040 << 8) | 0x11, 0xFF000000}, /* P2040 1.1 */ {(SVR_P2041 << 8) | 0x10, 0xFF000000}, /* P2041 1.0 */ @@ -1006,7 +1005,7 @@ static const struct { #define SVR_SECURITY 0x80000 /* The Security (E) bit */ -static int __init fsl_pamu_probe(struct platform_device *pdev) +static int fsl_pamu_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; void __iomem *pamu_regs = NULL; @@ -1022,6 +1021,7 @@ static int __init fsl_pamu_probe(struct platform_device *pdev) int irq; phys_addr_t ppaact_phys; phys_addr_t spaact_phys; + struct ome *omt; phys_addr_t omt_phys; size_t mem_size = 0; unsigned int order = 0; @@ -1200,7 +1200,7 @@ error: return ret; } -static struct platform_driver fsl_of_pamu_driver __initdata = { +static struct platform_driver fsl_of_pamu_driver = { .driver = { .name = "fsl-of-pamu", }, diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 0649b94f5958..35365f046923 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -364,7 +364,8 @@ static inline int first_pte_in_page(struct dma_pte *pte) static struct dmar_domain *si_domain; static int hw_pass_through = 1; -/* domain represents a virtual machine, more than one devices +/* + * Domain represents a virtual machine, more than one devices * across iommus may be owned in one domain, e.g. kvm guest. */ #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0) @@ -372,11 +373,21 @@ static int hw_pass_through = 1; /* si_domain contains mulitple devices */ #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1) +#define for_each_domain_iommu(idx, domain) \ + for (idx = 0; idx < g_num_of_iommus; idx++) \ + if (domain->iommu_refcnt[idx]) + struct dmar_domain { - int id; /* domain id */ int nid; /* node id */ - DECLARE_BITMAP(iommu_bmp, DMAR_UNITS_SUPPORTED); - /* bitmap of iommus this domain uses*/ + + unsigned iommu_refcnt[DMAR_UNITS_SUPPORTED]; + /* Refcount of devices per iommu */ + + + u16 iommu_did[DMAR_UNITS_SUPPORTED]; + /* Domain ids per IOMMU. Use u16 since + * domain ids are 16 bit wide according + * to VT-d spec, section 9.3 */ struct list_head devices; /* all devices' list */ struct iova_domain iovad; /* iova's that belong to this domain */ @@ -395,7 +406,6 @@ struct dmar_domain { int iommu_superpage;/* Level of superpages supported: 0 == 4KiB (no superpages), 1 == 2MiB, 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */ - spinlock_t iommu_lock; /* protect iommu set in domain */ u64 max_addr; /* maximum mapped address */ struct iommu_domain domain; /* generic domain data structure for @@ -408,6 +418,10 @@ struct device_domain_info { struct list_head global; /* link to global list */ u8 bus; /* PCI bus number */ u8 devfn; /* PCI devfn number */ + struct { + u8 enabled:1; + u8 qdep; + } ats; /* ATS state */ struct device *dev; /* it's NULL for PCIe-to-PCI bridge */ struct intel_iommu *iommu; /* IOMMU used by this device */ struct dmar_domain *domain; /* pointer to domain */ @@ -461,10 +475,11 @@ static long list_size; static void domain_exit(struct dmar_domain *domain); static void domain_remove_dev_info(struct dmar_domain *domain); -static void domain_remove_one_dev_info(struct dmar_domain *domain, - struct device *dev); -static void iommu_detach_dependent_devices(struct intel_iommu *iommu, - struct device *dev); +static void dmar_remove_one_dev_info(struct dmar_domain *domain, + struct device *dev); +static void __dmar_remove_one_dev_info(struct device_domain_info *info); +static void domain_context_clear(struct intel_iommu *iommu, + struct device *dev); static int domain_detach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu); @@ -564,6 +579,36 @@ __setup("intel_iommu=", intel_iommu_setup); static struct kmem_cache *iommu_domain_cache; static struct kmem_cache *iommu_devinfo_cache; +static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did) +{ + struct dmar_domain **domains; + int idx = did >> 8; + + domains = iommu->domains[idx]; + if (!domains) + return NULL; + + return domains[did & 0xff]; +} + +static void set_iommu_domain(struct intel_iommu *iommu, u16 did, + struct dmar_domain *domain) +{ + struct dmar_domain **domains; + int idx = did >> 8; + + if (!iommu->domains[idx]) { + size_t size = 256 * sizeof(struct dmar_domain *); + iommu->domains[idx] = kzalloc(size, GFP_ATOMIC); + } + + domains = iommu->domains[idx]; + if (WARN_ON(!domains)) + return; + else + domains[did & 0xff] = domain; +} + static inline void *alloc_pgtable_page(int node) { struct page *page; @@ -605,6 +650,11 @@ static inline int domain_type_is_vm(struct dmar_domain *domain) return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE; } +static inline int domain_type_is_si(struct dmar_domain *domain) +{ + return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY; +} + static inline int domain_type_is_vm_or_si(struct dmar_domain *domain) { return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE | @@ -659,7 +709,9 @@ static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain) /* si_domain and vm domain should not get here. */ BUG_ON(domain_type_is_vm_or_si(domain)); - iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus); + for_each_domain_iommu(iommu_id, domain) + break; + if (iommu_id < 0 || iommu_id >= g_num_of_iommus) return NULL; @@ -675,7 +727,7 @@ static void domain_update_iommu_coherency(struct dmar_domain *domain) domain->iommu_coherency = 1; - for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) { + for_each_domain_iommu(i, domain) { found = true; if (!ecap_coherent(g_iommus[i]->ecap)) { domain->iommu_coherency = 0; @@ -755,6 +807,7 @@ static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu struct context_entry *context; u64 *entry; + entry = &root->lo; if (ecs_enabled(iommu)) { if (devfn >= 0x80) { devfn -= 0x80; @@ -762,7 +815,6 @@ static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu } devfn *= 2; } - entry = &root->lo; if (*entry & 1) context = phys_to_virt(*entry & VTD_PAGE_MASK); else { @@ -1162,9 +1214,9 @@ next: /* We can't just free the pages because the IOMMU may still be walking the page tables, and may have cached the intermediate levels. The pages can only be freed after the IOTLB flush has been done. */ -struct page *domain_unmap(struct dmar_domain *domain, - unsigned long start_pfn, - unsigned long last_pfn) +static struct page *domain_unmap(struct dmar_domain *domain, + unsigned long start_pfn, + unsigned long last_pfn) { struct page *freelist = NULL; @@ -1188,7 +1240,7 @@ struct page *domain_unmap(struct dmar_domain *domain, return freelist; } -void dma_free_pagelist(struct page *freelist) +static void dma_free_pagelist(struct page *freelist) { struct page *pg; @@ -1356,24 +1408,23 @@ iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu, u8 bus, u8 devfn) { bool found = false; - unsigned long flags; struct device_domain_info *info; struct pci_dev *pdev; + assert_spin_locked(&device_domain_lock); + if (!ecap_dev_iotlb_support(iommu->ecap)) return NULL; if (!iommu->qi) return NULL; - spin_lock_irqsave(&device_domain_lock, flags); list_for_each_entry(info, &domain->devices, link) if (info->iommu == iommu && info->bus == bus && info->devfn == devfn) { found = true; break; } - spin_unlock_irqrestore(&device_domain_lock, flags); if (!found || !info->dev || !dev_is_pci(info->dev)) return NULL; @@ -1391,19 +1442,26 @@ iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu, static void iommu_enable_dev_iotlb(struct device_domain_info *info) { + struct pci_dev *pdev; + if (!info || !dev_is_pci(info->dev)) return; - pci_enable_ats(to_pci_dev(info->dev), VTD_PAGE_SHIFT); + pdev = to_pci_dev(info->dev); + if (pci_enable_ats(pdev, VTD_PAGE_SHIFT)) + return; + + info->ats.enabled = 1; + info->ats.qdep = pci_ats_queue_depth(pdev); } static void iommu_disable_dev_iotlb(struct device_domain_info *info) { - if (!info->dev || !dev_is_pci(info->dev) || - !pci_ats_enabled(to_pci_dev(info->dev))) + if (!info->ats.enabled) return; pci_disable_ats(to_pci_dev(info->dev)); + info->ats.enabled = 0; } static void iommu_flush_dev_iotlb(struct dmar_domain *domain, @@ -1415,26 +1473,24 @@ static void iommu_flush_dev_iotlb(struct dmar_domain *domain, spin_lock_irqsave(&device_domain_lock, flags); list_for_each_entry(info, &domain->devices, link) { - struct pci_dev *pdev; - if (!info->dev || !dev_is_pci(info->dev)) - continue; - - pdev = to_pci_dev(info->dev); - if (!pci_ats_enabled(pdev)) + if (!info->ats.enabled) continue; sid = info->bus << 8 | info->devfn; - qdep = pci_ats_queue_depth(pdev); + qdep = info->ats.qdep; qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask); } spin_unlock_irqrestore(&device_domain_lock, flags); } -static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did, - unsigned long pfn, unsigned int pages, int ih, int map) +static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, + struct dmar_domain *domain, + unsigned long pfn, unsigned int pages, + int ih, int map) { unsigned int mask = ilog2(__roundup_pow_of_two(pages)); uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT; + u16 did = domain->iommu_did[iommu->seq_id]; BUG_ON(pages == 0); @@ -1458,7 +1514,8 @@ static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did, * flush. However, device IOTLB doesn't need to be flushed in this case. */ if (!cap_caching_mode(iommu->cap) || !map) - iommu_flush_dev_iotlb(iommu->domains[did], addr, mask); + iommu_flush_dev_iotlb(get_iommu_domain(iommu, did), + addr, mask); } static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu) @@ -1513,65 +1570,80 @@ static void iommu_disable_translation(struct intel_iommu *iommu) static int iommu_init_domains(struct intel_iommu *iommu) { - unsigned long ndomains; - unsigned long nlongs; + u32 ndomains, nlongs; + size_t size; ndomains = cap_ndoms(iommu->cap); - pr_debug("%s: Number of Domains supported <%ld>\n", + pr_debug("%s: Number of Domains supported <%d>\n", iommu->name, ndomains); nlongs = BITS_TO_LONGS(ndomains); spin_lock_init(&iommu->lock); - /* TBD: there might be 64K domains, - * consider other allocation for future chip - */ iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL); if (!iommu->domain_ids) { pr_err("%s: Allocating domain id array failed\n", iommu->name); return -ENOMEM; } - iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *), - GFP_KERNEL); - if (!iommu->domains) { + + size = ((ndomains >> 8) + 1) * sizeof(struct dmar_domain **); + iommu->domains = kzalloc(size, GFP_KERNEL); + + if (iommu->domains) { + size = 256 * sizeof(struct dmar_domain *); + iommu->domains[0] = kzalloc(size, GFP_KERNEL); + } + + if (!iommu->domains || !iommu->domains[0]) { pr_err("%s: Allocating domain array failed\n", iommu->name); kfree(iommu->domain_ids); + kfree(iommu->domains); iommu->domain_ids = NULL; + iommu->domains = NULL; return -ENOMEM; } + + /* - * if Caching mode is set, then invalid translations are tagged - * with domainid 0. Hence we need t |