diff options
50 files changed, 3345 insertions, 1927 deletions
diff --git a/Documentation/devicetree/bindings/iommu/allwinner,sun50i-h6-iommu.yaml b/Documentation/devicetree/bindings/iommu/allwinner,sun50i-h6-iommu.yaml new file mode 100644 index 000000000000..5e125cf2a88b --- /dev/null +++ b/Documentation/devicetree/bindings/iommu/allwinner,sun50i-h6-iommu.yaml @@ -0,0 +1,61 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/iommu/allwinner,sun50i-h6-iommu.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Allwinner H6 IOMMU Device Tree Bindings + +maintainers: + - Chen-Yu Tsai <wens@csie.org> + - Maxime Ripard <mripard@kernel.org> + +properties: + "#iommu-cells": + const: 1 + description: + The content of the cell is the master ID. + + compatible: + const: allwinner,sun50i-h6-iommu + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + + resets: + maxItems: 1 + +required: + - "#iommu-cells" + - compatible + - reg + - interrupts + - clocks + - resets + +additionalProperties: false + +examples: + - | + #include <dt-bindings/interrupt-controller/arm-gic.h> + #include <dt-bindings/interrupt-controller/irq.h> + + #include <dt-bindings/clock/sun50i-h6-ccu.h> + #include <dt-bindings/reset/sun50i-h6-ccu.h> + + iommu: iommu@30f0000 { + compatible = "allwinner,sun50i-h6-iommu"; + reg = <0x030f0000 0x10000>; + interrupts = <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&ccu CLK_BUS_IOMMU>; + resets = <&ccu RST_BUS_IOMMU>; + #iommu-cells = <1>; + }; + +... diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml index 218f7ecb4703..d7ceb4c34423 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml @@ -42,7 +42,9 @@ properties: - const: arm,mmu-500 - const: arm,smmu-v2 - items: - - const: arm,mmu-401 + - enum: + - arm,mmu-400 + - arm,mmu-401 - const: arm,smmu-v1 - enum: - arm,smmu-v1 diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,q6v5.txt b/Documentation/devicetree/bindings/remoteproc/qcom,q6v5.txt index 88dfa3fc15f7..130e50aab741 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,q6v5.txt +++ b/Documentation/devicetree/bindings/remoteproc/qcom,q6v5.txt @@ -184,6 +184,9 @@ For the compatible strings below the following phandle references are required: followed by the offset within syscon for conn_box_spare0 register. +The Hexagon node must contain iommus property as described in ../iommu/iommu.txt +on platforms which do not have TrustZone. + = SUBNODES: The Hexagon node must contain two subnodes, named "mba" and "mpss" representing the memory regions used by the Hexagon firmware. Each sub-node must contain: diff --git a/arch/arm64/boot/dts/qcom/sdm845-cheza.dtsi b/arch/arm64/boot/dts/qcom/sdm845-cheza.dtsi index 5938f8b2aa2f..70466cc4b405 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-cheza.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm845-cheza.dtsi @@ -633,6 +633,11 @@ ap_ts_i2c: &i2c14 { status = "okay"; }; +&mss_pil { + iommus = <&apps_smmu 0x780 0x1>, + <&apps_smmu 0x724 0x3>; +}; + &pm8998_pwrkey { status = "disabled"; }; diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 2ab07ce17abb..aca76383f201 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -303,6 +303,15 @@ config ROCKCHIP_IOMMU Say Y here if you are using a Rockchip SoC that includes an IOMMU device. +config SUN50I_IOMMU + bool "Allwinner H6 IOMMU Support" + depends on ARCH_SUNXI || COMPILE_TEST + select ARM_DMA_USE_IOMMU + select IOMMU_API + select IOMMU_DMA + help + Support for the IOMMU introduced in the Allwinner H6 SoCs. + config TEGRA_IOMMU_GART bool "Tegra GART IOMMU Support" depends on ARCH_TEGRA_2x_SOC diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 9f33fdb3bb05..57cf4ba5e27c 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -29,6 +29,7 @@ obj-$(CONFIG_MTK_IOMMU_V1) += mtk_iommu_v1.o obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o obj-$(CONFIG_ROCKCHIP_IOMMU) += rockchip-iommu.o +obj-$(CONFIG_SUN50I_IOMMU) += sun50i-iommu.o obj-$(CONFIG_TEGRA_IOMMU_GART) += tegra-gart.o obj-$(CONFIG_TEGRA_IOMMU_SMMU) += tegra-smmu.o obj-$(CONFIG_EXYNOS_IOMMU) += exynos-iommu.o diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 2883ac389abb..311ef7105c6d 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -22,7 +22,6 @@ #include <linux/dma-direct.h> #include <linux/dma-iommu.h> #include <linux/iommu-helper.h> -#include <linux/iommu.h> #include <linux/delay.h> #include <linux/amd-iommu.h> #include <linux/notifier.h> @@ -43,8 +42,7 @@ #include <asm/gart.h> #include <asm/dma.h> -#include "amd_iommu_proto.h" -#include "amd_iommu_types.h" +#include "amd_iommu.h" #include "irq_remapping.h" #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28)) @@ -71,6 +69,8 @@ */ #define AMD_IOMMU_PGSIZES ((~0xFFFUL) & ~(2ULL << 38)) +#define DEFAULT_PGTABLE_LEVEL PAGE_MODE_3_LEVEL + static DEFINE_SPINLOCK(pd_bitmap_lock); /* List of all available dev_data structures */ @@ -99,7 +99,6 @@ struct iommu_cmd { struct kmem_cache *amd_iommu_irq_cache; static void update_domain(struct protection_domain *domain); -static int protection_domain_init(struct protection_domain *domain); static void detach_device(struct device *dev); static void update_and_flush_device_table(struct protection_domain *domain, struct domain_pgtable *pgtable); @@ -280,12 +279,6 @@ static struct iommu_dev_data *find_dev_data(u16 devid) return dev_data; } -struct iommu_dev_data *get_dev_data(struct device *dev) -{ - return dev->archdata.iommu; -} -EXPORT_SYMBOL(get_dev_data); - /* * Find or create an IOMMU group for a acpihid device. */ @@ -314,16 +307,15 @@ static struct iommu_group *acpihid_device_group(struct device *dev) static bool pci_iommuv2_capable(struct pci_dev *pdev) { static const int caps[] = { - PCI_EXT_CAP_ID_ATS, PCI_EXT_CAP_ID_PRI, PCI_EXT_CAP_ID_PASID, }; int i, pos; - if (pci_ats_disabled()) + if (!pci_ats_supported(pdev)) return false; - for (i = 0; i < 3; ++i) { + for (i = 0; i < 2; ++i) { pos = pci_find_ext_capability(pdev, caps[i]); if (pos == 0) return false; @@ -336,7 +328,7 @@ static bool pdev_pri_erratum(struct pci_dev *pdev, u32 erratum) { struct iommu_dev_data *dev_data; - dev_data = get_dev_data(&pdev->dev); + dev_data = dev_iommu_priv_get(&pdev->dev); return dev_data->errata & (1 << erratum) ? true : false; } @@ -349,7 +341,7 @@ static bool check_device(struct device *dev) { int devid; - if (!dev || !dev->dma_mask) + if (!dev) return false; devid = get_device_id(dev); @@ -366,32 +358,18 @@ static bool check_device(struct device *dev) return true; } -static void init_iommu_group(struct device *dev) -{ - struct iommu_group *group; - - group = iommu_group_get_for_dev(dev); - if (IS_ERR(group)) - return; - - iommu_group_put(group); -} - static int iommu_init_device(struct device *dev) { struct iommu_dev_data *dev_data; - struct amd_iommu *iommu; int devid; - if (dev->archdata.iommu) + if (dev_iommu_priv_get(dev)) return 0; devid = get_device_id(dev); if (devid < 0) return devid; - iommu = amd_iommu_rlookup_table[devid]; - dev_data = find_dev_data(devid); if (!dev_data) return -ENOMEM; @@ -412,9 +390,7 @@ static int iommu_init_device(struct device *dev) dev_data->iommu_v2 = iommu->is_iommu_v2; } - dev->archdata.iommu = dev_data; - - iommu_device_link(&iommu->iommu, dev); + dev_iommu_priv_set(dev, dev_data); return 0; } @@ -433,31 +409,18 @@ static void iommu_ignore_device(struct device *dev) setup_aliases(dev); } -static void iommu_uninit_device(struct device *dev) +static void amd_iommu_uninit_device(struct device *dev) { struct iommu_dev_data *dev_data; - struct amd_iommu *iommu; - int devid; - devid = get_device_id(dev); - if (devid < 0) - return; - - iommu = amd_iommu_rlookup_table[devid]; - - dev_data = search_dev_data(devid); + dev_data = dev_iommu_priv_get(dev); if (!dev_data) return; if (dev_data->domain) detach_device(dev); - iommu_device_unlink(&iommu->iommu, dev); - - iommu_group_remove_device(dev); - - /* Remove dma-ops */ - dev->dma_ops = NULL; + dev_iommu_priv_set(dev, NULL); /* * We keep dev_data around for unplugged devices and reuse it when the @@ -521,7 +484,7 @@ static void amd_iommu_report_page_fault(u16 devid, u16 domain_id, pdev = pci_get_domain_bus_and_slot(0, PCI_BUS_NUM(devid), devid & 0xff); if (pdev) - dev_data = get_dev_data(&pdev->dev); + dev_data = dev_iommu_priv_get(&pdev->dev); if (dev_data && __ratelimit(&dev_data->rs)) { pci_err(pdev, "Event logged [IO_PAGE_FAULT domain=0x%04x address=0x%llx flags=0x%04x]\n", @@ -1418,20 +1381,19 @@ static struct page *free_sub_pt(unsigned long root, int mode, return freelist; } -static void free_pagetable(struct protection_domain *domain) +static void free_pagetable(struct domain_pgtable *pgtable) { - struct domain_pgtable pgtable; struct page *freelist = NULL; unsigned long root; - amd_iommu_domain_get_pgtable(domain, &pgtable); - atomic64_set(&domain->pt_root, 0); + if (pgtable->mode == PAGE_MODE_NONE) + return; - BUG_ON(pgtable.mode < PAGE_MODE_NONE || - pgtable.mode > PAGE_MODE_6_LEVEL); + BUG_ON(pgtable->mode < PAGE_MODE_NONE || + pgtable->mode > PAGE_MODE_6_LEVEL); - root = (unsigned long)pgtable.root; - freelist = free_sub_pt(root, pgtable.mode, freelist); + root = (unsigned long)pgtable->root; + freelist = free_sub_pt(root, pgtable->mode, freelist); free_page_list(freelist); } @@ -1844,70 +1806,6 @@ static void free_gcr3_table(struct protection_domain *domain) free_page((unsigned long)domain->gcr3_tbl); } -/* - * Free a domain, only used if something went wrong in the - * allocation path and we need to free an already allocated page table - */ -static void dma_ops_domain_free(struct protection_domain *domain) -{ - if (!domain) - return; - - iommu_put_dma_cookie(&domain->domain); - - free_pagetable(domain); - - if (domain->id) - domain_id_free(domain->id); - - kfree(domain); -} - -/* - * Allocates a new protection domain usable for the dma_ops functions. - * It also initializes the page table and the address allocator data - * structures required for the dma_ops interface - */ -static struct protection_domain *dma_ops_domain_alloc(void) -{ - struct protection_domain *domain; - u64 *pt_root, root; - - domain = kzalloc(sizeof(struct protection_domain), GFP_KERNEL); - if (!domain) - return NULL; - - if (protection_domain_init(domain)) - goto free_domain; - - pt_root = (void *)get_zeroed_page(GFP_KERNEL); - if (!pt_root) - goto free_domain; - - root = amd_iommu_domain_encode_pgtable(pt_root, PAGE_MODE_3_LEVEL); - atomic64_set(&domain->pt_root, root); - domain->flags = PD_DMA_OPS_MASK; - - if (iommu_get_dma_cookie(&domain->domain) == -ENOMEM) - goto free_domain; - - return domain; - -free_domain: - dma_ops_domain_free(domain); - - return NULL; -} - -/* - * little helper function to check whether a given protection domain is a - * dma_ops domain - */ -static bool dma_ops_domain(struct protection_domain *domain) -{ - return domain->flags & PD_DMA_OPS_MASK; -} - static void set_dte_entry(u16 devid, struct protection_domain *domain, struct domain_pgtable *pgtable, bool ats, bool ppr) @@ -2119,14 +2017,14 @@ out_err: static int attach_device(struct device *dev, struct protection_domain *domain) { - struct pci_dev *pdev; struct iommu_dev_data *dev_data; + struct pci_dev *pdev; unsigned long flags; int ret; spin_lock_irqsave(&domain->lock, flags); - dev_data = get_dev_data(dev); + dev_data = dev_iommu_priv_get(dev); spin_lock(&dev_data->lock); @@ -2139,8 +2037,10 @@ static int attach_device(struct device *dev, pdev = to_pci_dev(dev); if (domain->flags & PD_IOMMUV2_MASK) { + struct iommu_domain *def_domain = iommu_get_dma_domain(dev); + ret = -EINVAL; - if (!dev_data->passthrough) + if (def_domain->type != IOMMU_DOMAIN_IDENTITY) goto out; if (dev_data->iommu_v2) { @@ -2188,7 +2088,7 @@ static void detach_device(struct device *dev) struct iommu_dev_data *dev_data; unsigned long flags; - dev_data = get_dev_data(dev); + dev_data = dev_iommu_priv_get(dev); domain = dev_data->domain; spin_lock_irqsave(&domain->lock, flags); @@ -2222,68 +2122,60 @@ out: spin_unlock_irqrestore(&domain->lock, flags); } -static int amd_iommu_add_device(struct device *dev) +static struct iommu_device *amd_iommu_probe_device(struct device *dev) { - struct iommu_dev_data *dev_data; - struct iommu_domain *domain; + struct iommu_device *iommu_dev; struct amd_iommu *iommu; int ret, devid; - if (!check_device(dev) || get_dev_data(dev)) - return 0; + if (!check_device(dev)) + return ERR_PTR(-ENODEV); devid = get_device_id(dev); if (devid < 0) - return devid; + return ERR_PTR(devid); iommu = amd_iommu_rlookup_table[devid]; + if (dev_iommu_priv_get(dev)) + return &iommu->iommu; + ret = iommu_init_device(dev); if (ret) { if (ret != -ENOTSUPP) dev_err(dev, "Failed to initialize - trying to proceed anyway\n"); - + iommu_dev = ERR_PTR(ret); iommu_ignore_device(dev); - dev->dma_ops = NULL; - goto out; + } else { + iommu_dev = &iommu->iommu; } - init_iommu_group(dev); - dev_data = get_dev_data(dev); + iommu_completion_wait(iommu); - BUG_ON(!dev_data); + return iommu_dev; +} - if (dev_data->iommu_v2) - iommu_request_dm_for_dev(dev); +static void amd_iommu_probe_finalize(struct device *dev) +{ + struct iommu_domain *domain; /* Domains are initialized for this device - have a look what we ended up with */ domain = iommu_get_domain_for_dev(dev); - if (domain->type == IOMMU_DOMAIN_IDENTITY) - dev_data->passthrough = true; - else if (domain->type == IOMMU_DOMAIN_DMA) + if (domain->type == IOMMU_DOMAIN_DMA) iommu_setup_dma_ops(dev, IOVA_START_PFN << PAGE_SHIFT, 0); - -out: - iommu_completion_wait(iommu); - - return 0; } -static void amd_iommu_remove_device(struct device *dev) +static void amd_iommu_release_device(struct device *dev) { + int devid = get_device_id(dev); struct amd_iommu *iommu; - int devid; if (!check_device(dev)) return; - devid = get_device_id(dev); - if (devid < 0) - return; - iommu = amd_iommu_rlookup_table[devid]; - iommu_uninit_device(dev); + amd_iommu_uninit_device(dev); iommu_completion_wait(iommu); } @@ -2418,27 +2310,46 @@ static void cleanup_domain(struct protection_domain *domain) static void protection_domain_free(struct protection_domain *domain) { + struct domain_pgtable pgtable; + if (!domain) return; if (domain->id) domain_id_free(domain->id); + amd_iommu_domain_get_pgtable(domain, &pgtable); + atomic64_set(&domain->pt_root, 0); + free_pagetable(&pgtable); + kfree(domain); } -static int protection_domain_init(struct protection_domain *domain) +static int protection_domain_init(struct protection_domain *domain, int mode) { + u64 *pt_root = NULL, root; + + BUG_ON(mode < PAGE_MODE_NONE || mode > PAGE_MODE_6_LEVEL); + spin_lock_init(&domain->lock); domain->id = domain_id_alloc(); if (!domain->id) return -ENOMEM; INIT_LIST_HEAD(&domain->dev_list); + if (mode != PAGE_MODE_NONE) { + pt_root = (void *)get_zeroed_page(GFP_KERNEL); + if (!pt_root) + return -ENOMEM; + } + + root = amd_iommu_domain_encode_pgtable(pt_root, mode); + atomic64_set(&domain->pt_root, root); + return 0; } -static struct protection_domain *protection_domain_alloc(void) +static struct protection_domain *protection_domain_alloc(int mode) { struct protection_domain *domain; @@ -2446,7 +2357,7 @@ static struct protection_domain *protection_domain_alloc(void) if (!domain) return NULL; - if (protection_domain_init(domain)) + if (protection_domain_init(domain, mode)) goto out_err; return domain; @@ -2459,54 +2370,35 @@ out_err: static struct iommu_domain *amd_iommu_domain_alloc(unsigned type) { - struct protection_domain *pdomain; - u64 *pt_root, root; + struct protection_domain *domain; + int mode = DEFAULT_PGTABLE_LEVEL; - switch (type) { - case IOMMU_DOMAIN_UNMANAGED: - pdomain = protection_domain_alloc(); - if (!pdomain) - return NULL; + if (type == IOMMU_DOMAIN_IDENTITY) + mode = PAGE_MODE_NONE; - pt_root = (void *)get_zeroed_page(GFP_KERNEL); - if (!pt_root) { - protection_domain_free(pdomain); - return NULL; - } + domain = protection_domain_alloc(mode); + if (!domain) + return NULL; - root = amd_iommu_domain_encode_pgtable(pt_root, PAGE_MODE_3_LEVEL); - atomic64_set(&pdomain->pt_root, root); + domain->domain.geometry.aperture_start = 0; + domain->domain.geometry.aperture_end = ~0ULL; + domain->domain.geometry.force_aperture = true; - pdomain->domain.geometry.aperture_start = 0; - pdomain->domain.geometry.aperture_end = ~0ULL; - pdomain->domain.geometry.force_aperture = true; + if (type == IOMMU_DOMAIN_DMA && + iommu_get_dma_cookie(&domain->domain) == -ENOMEM) + goto free_domain; - break; - case IOMMU_DOMAIN_DMA: - pdomain = dma_ops_domain_alloc(); - if (!pdomain) { - pr_err("Failed to allocate\n"); - return NULL; - } - break; - case IOMMU_DOMAIN_IDENTITY: - pdomain = protection_domain_alloc(); - if (!pdomain) - return NULL; + return &domain->domain; - atomic64_set(&pdomain->pt_root, PAGE_MODE_NONE); - break; - default: - return NULL; - } +free_domain: + protection_domain_free(domain); - return &pdomain->domain; + return NULL; } static void amd_iommu_domain_free(struct iommu_domain *dom) { struct protection_domain *domain; - struct domain_pgtable pgtable; domain = to_pdomain(dom); @@ -2518,29 +2410,19 @@ static void amd_iommu_domain_free(struct iommu_domain *dom) if (!dom) return; - switch (dom->type) { - case IOMMU_DOMAIN_DMA: - /* Now release the domain */ - dma_ops_domain_free(domain); - break; - default: - amd_iommu_domain_get_pgtable(domain, &pgtable); - - if (pgtable.mode != PAGE_MODE_NONE) - free_pagetable(domain); + if (dom->type == IOMMU_DOMAIN_DMA) + iommu_put_dma_cookie(&domain->domain); - if (domain->flags & PD_IOMMUV2_MASK) - free_gcr3_table(domain); + if (domain->flags & PD_IOMMUV2_MASK) + free_gcr3_table(domain); - protection_domain_free(domain); - break; - } + protection_domain_free(domain); } static void amd_iommu_detach_device(struct iommu_domain *dom, struct device *dev) { - struct iommu_dev_data *dev_data = dev->archdata.iommu; + struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); struct amd_iommu *iommu; int devid; @@ -2578,7 +2460,7 @@ static int amd_iommu_attach_device(struct iommu_domain *dom, if (!check_device(dev)) return -EINVAL; - dev_data = dev->archdata.iommu; + dev_data = dev_iommu_priv_get(dev); dev_data->defer_attach = false; iommu = amd_iommu_rlookup_table[dev_data->devid]; @@ -2734,12 +2616,14 @@ static void amd_iommu_get_resv_regions(struct device *dev, list_add_tail(®ion->list, head); } -static bool amd_iommu_is_attach_deferred(struct iommu_domain *domain, - struct device *dev) +bool amd_iommu_is_attach_deferred(struct iommu_domain *domain, + struct device *dev) { - struct iommu_dev_data *dev_data = dev->archdata.iommu; + struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); + return dev_data->defer_attach; } +EXPORT_SYMBOL_GPL(amd_iommu_is_attach_deferred); static void amd_iommu_flush_iotlb_all(struct iommu_domain *domain) { @@ -2758,6 +2642,20 @@ static void amd_iommu_iotlb_sync(struct iommu_domain *domain, amd_iommu_flush_iotlb_all(domain); } +static int amd_iommu_def_domain_type(struct device *dev) +{ + struct iommu_dev_data *dev_data; + + dev_data = dev_iommu_priv_get(dev); + if (!dev_data) + return 0; + + if (dev_data->iommu_v2) + return IOMMU_DOMAIN_IDENTITY; + + return 0; +} + const struct iommu_ops amd_iommu_ops = { .capable = amd_iommu_capable, .domain_alloc = amd_iommu_domain_allo |