diff options
Diffstat (limited to 'drivers/iommu')
26 files changed, 4680 insertions, 1401 deletions
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index baa0d9786f50..40f37a2b4a8a 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -23,6 +23,7 @@ config IOMMU_IO_PGTABLE config IOMMU_IO_PGTABLE_LPAE bool "ARMv7/v8 Long Descriptor Format" select IOMMU_IO_PGTABLE + depends on ARM || ARM64 || COMPILE_TEST help Enable support for the ARM long descriptor pagetable format. This allocator supports 4K/2M/1G, 16K/32M and 64K/512M page @@ -63,6 +64,7 @@ config MSM_IOMMU bool "MSM IOMMU Support" depends on ARM depends on ARCH_MSM8X60 || ARCH_MSM8960 || COMPILE_TEST + depends on BROKEN select IOMMU_API help Support for the IOMMUs found on certain Qualcomm SOCs. @@ -337,6 +339,7 @@ config SPAPR_TCE_IOMMU Enables bits of IOMMU API required by VFIO. The iommu_ops is not implemented as it is not necessary for VFIO. +# ARM IOMMU support config ARM_SMMU bool "ARM Ltd. System MMU (SMMU) Support" depends on (ARM64 || ARM) && MMU @@ -350,4 +353,16 @@ config ARM_SMMU Say Y here if your SoC includes an IOMMU device implementing the ARM SMMU architecture. +config ARM_SMMU_V3 + bool "ARM Ltd. System MMU Version 3 (SMMUv3) Support" + depends on ARM64 && PCI + select IOMMU_API + select IOMMU_IO_PGTABLE_LPAE + help + Support for implementations of the ARM System MMU architecture + version 3 providing translation support to a PCIe root complex. + + Say Y here if your system includes an IOMMU device implementing + the ARM SMMUv3 architecture. + endif # IOMMU_SUPPORT diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 080ffab4ed1c..c6dcc513d711 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -9,6 +9,7 @@ obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o obj-$(CONFIG_ARM_SMMU) += arm-smmu.o +obj-$(CONFIG_ARM_SMMU_V3) += arm-smmu-v3.o obj-$(CONFIG_DMAR_TABLE) += dmar.o obj-$(CONFIG_INTEL_IOMMU) += intel-iommu.o obj-$(CONFIG_IPMMU_VMSA) += ipmmu-vmsa.o diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 31e90c472c76..c5677ed2cd89 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -64,10 +64,6 @@ static DEFINE_RWLOCK(amd_iommu_devtable_lock); -/* A list of preallocated protection domains */ -static LIST_HEAD(iommu_pd_list); -static DEFINE_SPINLOCK(iommu_pd_list_lock); - /* List of all available dev_data structures */ static LIST_HEAD(dev_data_list); static DEFINE_SPINLOCK(dev_data_list_lock); @@ -119,7 +115,7 @@ struct iommu_cmd { struct kmem_cache *amd_iommu_irq_cache; static void update_domain(struct protection_domain *domain); -static int __init alloc_passthrough_domain(void); +static int alloc_passthrough_domain(void); /**************************************************************************** * @@ -127,6 +123,11 @@ static int __init alloc_passthrough_domain(void); * ****************************************************************************/ +static struct protection_domain *to_pdomain(struct iommu_domain *dom) +{ + return container_of(dom, struct protection_domain, domain); +} + static struct iommu_dev_data *alloc_dev_data(u16 devid) { struct iommu_dev_data *dev_data; @@ -229,31 +230,38 @@ static bool pdev_pri_erratum(struct pci_dev *pdev, u32 erratum) } /* - * In this function the list of preallocated protection domains is traversed to - * find the domain for a specific device + * This function actually applies the mapping to the page table of the + * dma_ops domain. */ -static struct dma_ops_domain *find_protection_domain(u16 devid) +static void alloc_unity_mapping(struct dma_ops_domain *dma_dom, + struct unity_map_entry *e) { - struct dma_ops_domain *entry, *ret = NULL; - unsigned long flags; - u16 alias = amd_iommu_alias_table[devid]; - - if (list_empty(&iommu_pd_list)) - return NULL; - - spin_lock_irqsave(&iommu_pd_list_lock, flags); + u64 addr; - list_for_each_entry(entry, &iommu_pd_list, list) { - if (entry->target_dev == devid || - entry->target_dev == alias) { - ret = entry; - break; - } + for (addr = e->address_start; addr < e->address_end; + addr += PAGE_SIZE) { + if (addr < dma_dom->aperture_size) + __set_bit(addr >> PAGE_SHIFT, + dma_dom->aperture[0]->bitmap); } +} - spin_unlock_irqrestore(&iommu_pd_list_lock, flags); +/* + * Inits the unity mappings required for a specific device + */ +static void init_unity_mappings_for_device(struct device *dev, + struct dma_ops_domain *dma_dom) +{ + struct unity_map_entry *e; + u16 devid; - return ret; + devid = get_device_id(dev); + + list_for_each_entry(e, &amd_iommu_unity_map, list) { + if (!(devid >= e->devid_start && devid <= e->devid_end)) + continue; + alloc_unity_mapping(dma_dom, e); + } } /* @@ -285,11 +293,23 @@ static bool check_device(struct device *dev) static void init_iommu_group(struct device *dev) { + struct dma_ops_domain *dma_domain; + struct iommu_domain *domain; struct iommu_group *group; group = iommu_group_get_for_dev(dev); - if (!IS_ERR(group)) - iommu_group_put(group); + if (IS_ERR(group)) + return; + + domain = iommu_group_default_domain(group); + if (!domain) + goto out; + + dma_domain = to_pdomain(domain)->priv; + + init_unity_mappings_for_device(dev, dma_domain); +out: + iommu_group_put(group); } static int __last_alias(struct pci_dev *pdev, u16 alias, void *data) @@ -429,64 +449,15 @@ static void iommu_uninit_device(struct device *dev) /* Unlink from alias, it may change if another device is re-plugged */ dev_data->alias_data = NULL; + /* Remove dma-ops */ + dev->archdata.dma_ops = NULL; + /* * We keep dev_data around for unplugged devices and reuse it when the * device is re-plugged - not doing so would introduce a ton of races. */ } -void __init amd_iommu_uninit_devices(void) -{ - struct iommu_dev_data *dev_data, *n; - struct pci_dev *pdev = NULL; - - for_each_pci_dev(pdev) { - - if (!check_device(&pdev->dev)) - continue; - - iommu_uninit_device(&pdev->dev); - } - - /* Free all of our dev_data structures */ - list_for_each_entry_safe(dev_data, n, &dev_data_list, dev_data_list) - free_dev_data(dev_data); -} - -int __init amd_iommu_init_devices(void) -{ - struct pci_dev *pdev = NULL; - int ret = 0; - - for_each_pci_dev(pdev) { - - if (!check_device(&pdev->dev)) - continue; - - ret = iommu_init_device(&pdev->dev); - if (ret == -ENOTSUPP) - iommu_ignore_device(&pdev->dev); - else if (ret) - goto out_free; - } - - /* - * Initialize IOMMU groups only after iommu_init_device() has - * had a chance to populate any IVRS defined aliases. - */ - for_each_pci_dev(pdev) { - if (check_device(&pdev->dev)) - init_iommu_group(&pdev->dev); - } - - return 0; - -out_free: - - amd_iommu_uninit_devices(); - - return ret; -} #ifdef CONFIG_AMD_IOMMU_STATS /* @@ -1458,94 +1429,6 @@ static unsigned long iommu_unmap_page(struct protection_domain *dom, return unmapped; } -/* - * This function checks if a specific unity mapping entry is needed for - * this specific IOMMU. - */ -static int iommu_for_unity_map(struct amd_iommu *iommu, - struct unity_map_entry *entry) -{ - u16 bdf, i; - - for (i = entry->devid_start; i <= entry->devid_end; ++i) { - bdf = amd_iommu_alias_table[i]; - if (amd_iommu_rlookup_table[bdf] == iommu) - return 1; - } - - return 0; -} - -/* - * This function actually applies the mapping to the page table of the - * dma_ops domain. - */ -static int dma_ops_unity_map(struct dma_ops_domain *dma_dom, - struct unity_map_entry *e) -{ - u64 addr; - int ret; - - for (addr = e->address_start; addr < e->address_end; - addr += PAGE_SIZE) { - ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot, - PAGE_SIZE); - if (ret) - return ret; - /* - * if unity mapping is in aperture range mark the page - * as allocated in the aperture - */ - if (addr < dma_dom->aperture_size) - __set_bit(addr >> PAGE_SHIFT, - dma_dom->aperture[0]->bitmap); - } - - return 0; -} - -/* - * Init the unity mappings for a specific IOMMU in the system - * - * Basically iterates over all unity mapping entries and applies them to - * the default domain DMA of that IOMMU if necessary. - */ -static int iommu_init_unity_mappings(struct amd_iommu *iommu) -{ - struct unity_map_entry *entry; - int ret; - - list_for_each_entry(entry, &amd_iommu_unity_map, list) { - if (!iommu_for_unity_map(iommu, entry)) - continue; - ret = dma_ops_unity_map(iommu->default_dom, entry); - if (ret) - return ret; - } - - return 0; -} - -/* - * Inits the unity mappings required for a specific device - */ -static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom, - u16 devid) -{ - struct unity_map_entry *e; - int ret; - - list_for_each_entry(e, &amd_iommu_unity_map, list) { - if (!(devid >= e->devid_start && devid <= e->devid_end)) - continue; - ret = dma_ops_unity_map(dma_dom, e); - if (ret) - return ret; - } - - return 0; -} - /**************************************************************************** * * The next functions belong to the address allocator for the dma_ops @@ -2011,7 +1894,6 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void) goto free_dma_dom; dma_dom->need_flush = false; - dma_dom->target_dev = 0xffff; add_domain_to_list(&dma_dom->domain); @@ -2376,110 +2258,67 @@ static void detach_device(struct device *dev) dev_data->ats.enabled = false; } -/* - * Find out the protection domain structure for a given PCI device. This - * will give us the pointer to the page table root for example. - */ -static struct protection_domain *domain_for_device(struct device *dev) +static int amd_iommu_add_device(struct device *dev) { struct iommu_dev_data *dev_data; - struct protection_domain *dom = NULL; - unsigned long flags; - - dev_data = get_dev_data(dev); - - if (dev_data->domain) - return dev_data->domain; - - if (dev_data->alias_data != NULL) { - struct iommu_dev_data *alias_data = dev_data->alias_data; - - read_lock_irqsave(&amd_iommu_devtable_lock, flags); - if (alias_data->domain != NULL) { - __attach_device(dev_data, alias_data->domain); - dom = alias_data->domain; - } - read_unlock_irqrestore(&amd_iommu_devtable_lock, flags); - } - - return dom; -} - -static int device_change_notifier(struct notifier_block *nb, - unsigned long action, void *data) -{ - struct dma_ops_domain *dma_domain; - struct protection_domain *domain; - struct iommu_dev_data *dev_data; - struct device *dev = data; + struct iommu_domain *domain; struct amd_iommu *iommu; - unsigned long flags; u16 devid; + int ret; - if (!check_device(dev)) + if (!check_device(dev) || get_dev_data(dev)) return 0; - devid = get_device_id(dev); - iommu = amd_iommu_rlookup_table[devid]; - dev_data = get_dev_data(dev); - - switch (action) { - case BUS_NOTIFY_ADD_DEVICE: + devid = get_device_id(dev); + iommu = amd_iommu_rlookup_table[devid]; - iommu_init_device(dev); - init_iommu_group(dev); + ret = iommu_init_device(dev); + if (ret) { + if (ret != -ENOTSUPP) + pr_err("Failed to initialize device %s - trying to proceed anyway\n", + dev_name(dev)); - /* - * dev_data is still NULL and - * got initialized in iommu_init_device - */ - dev_data = get_dev_data(dev); + iommu_ignore_device(dev); + dev->archdata.dma_ops = &nommu_dma_ops; + goto out; + } + init_iommu_group(dev); - if (iommu_pass_through || dev_data->iommu_v2) { - dev_data->passthrough = true; - attach_device(dev, pt_domain); - break; - } + dev_data = get_dev_data(dev); - domain = domain_for_device(dev); + BUG_ON(!dev_data); - /* allocate a protection domain if a device is added */ - dma_domain = find_protection_domain(devid); - if (!dma_domain) { - dma_domain = dma_ops_domain_alloc(); - if (!dma_domain) - goto out; - dma_domain->target_dev = devid; - - spin_lock_irqsave(&iommu_pd_list_lock, flags); - list_add_tail(&dma_domain->list, &iommu_pd_list); - spin_unlock_irqrestore(&iommu_pd_list_lock, flags); - } + if (dev_data->iommu_v2) + iommu_request_dm_for_dev(dev); + /* Domains are initialized for this device - have a look what we ended up with */ + domain = iommu_get_domain_for_dev(dev); + if (domain->type == IOMMU_DOMAIN_IDENTITY) { + dev_data->passthrough = true; + dev->archdata.dma_ops = &nommu_dma_ops; + } else { dev->archdata.dma_ops = &amd_iommu_dma_ops; - - break; - case BUS_NOTIFY_REMOVED_DEVICE: - - iommu_uninit_device(dev); - - default: - goto out; } +out: iommu_completion_wait(iommu); -out: return 0; } -static struct notifier_block device_nb = { - .notifier_call = device_change_notifier, -}; - -void amd_iommu_init_notifier(void) +static void amd_iommu_remove_device(struct device *dev) { - bus_register_notifier(&pci_bus_type, &device_nb); + struct amd_iommu *iommu; + u16 devid; + + if (!check_device(dev)) + return; + + devid = get_device_id(dev); + iommu = amd_iommu_rlookup_table[devid]; + + iommu_uninit_device(dev); + iommu_completion_wait(iommu); } /***************************************************************************** @@ -2498,28 +2337,20 @@ void amd_iommu_init_notifier(void) static struct protection_domain *get_domain(struct device *dev) { struct protection_domain *domain; - struct dma_ops_domain *dma_dom; - u16 devid = get_device_id(dev); + struct iommu_domain *io_domain; if (!check_device(dev)) return ERR_PTR(-EINVAL); - domain = domain_for_device(dev); - if (domain != NULL && !dma_ops_domain(domain)) - return ERR_PTR(-EBUSY); - - if (domain != NULL) - return domain; + io_domain = iommu_get_domain_for_dev(dev); + if (!io_domain) + return NULL; - /* Device not bound yet - bind it */ - dma_dom = find_protection_domain(devid); - if (!dma_dom) - dma_dom = amd_iommu_rlookup_table[devid]->default_dom; - attach_device(dev, &dma_dom->domain); - DUMP_printk("Using protection domain %d for device %s\n", - dma_dom->domain.id, dev_name(dev)); + domain = to_pdomain(io_domain); + if (!dma_ops_domain(domain)) + return ERR_PTR(-EBUSY); - return &dma_dom->domain; + return domain; } static void update_device_table(struct protection_domain *domain) @@ -2933,6 +2764,7 @@ static void *alloc_coherent(struct device *dev, size_t size, size = PAGE_ALIGN(size); dma_mask = dev->coherent_dma_mask; flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32); + flag |= __GFP_ZERO; page = alloc_pages(flag | __GFP_NOWARN, get_order(size)); if (!page) { @@ -3014,54 +2846,6 @@ static int amd_iommu_dma_supported(struct device *dev, u64 mask) return check_device(dev); } -/* - * The function for pre-allocating protection domains. - * - * If the driver core informs the DMA layer if a driver grabs a device - * we don't need to preallocate the protection domains anymore. - * For now we have to. - */ -static void __init prealloc_protection_domains(void) -{ - struct iommu_dev_data *dev_data; - struct dma_ops_domain *dma_dom; - struct pci_dev *dev = NULL; - u16 devid; - - for_each_pci_dev(dev) { - - /* Do we handle this device? */ - if (!check_device(&dev->dev)) - continue; - - dev_data = get_dev_data(&dev->dev); - if (!amd_iommu_force_isolation && dev_data->iommu_v2) { - /* Make sure passthrough domain is allocated */ - alloc_passthrough_domain(); - dev_data->passthrough = true; - attach_device(&dev->dev, pt_domain); - pr_info("AMD-Vi: Using passthrough domain for device %s\n", - dev_name(&dev->dev)); - } - - /* Is there already any domain for it? */ - if (domain_for_device(&dev->dev)) - continue; - - devid = get_device_id(&dev->dev); - - dma_dom = dma_ops_domain_alloc(); - if (!dma_dom) - continue; - init_unity_mappings_for_device(dma_dom, devid); - dma_dom->target_dev = devid; - - attach_device(&dev->dev, &dma_dom->domain); - - list_add_tail(&dma_dom->list, &iommu_pd_list); - } -} - static struct dma_map_ops amd_iommu_dma_ops = { .alloc = alloc_coherent, .free = free_coherent, @@ -3072,76 +2856,16 @@ static struct dma_map_ops amd_iommu_dma_ops = { .dma_supported = amd_iommu_dma_supported, }; -static unsigned device_dma_ops_init(void) +int __init amd_iommu_init_api(void) { - struct iommu_dev_data *dev_data; - struct pci_dev *pdev = NULL; - unsigned unhandled = 0; - - for_each_pci_dev(pdev) { - if (!check_device(&pdev->dev)) { - - iommu_ignore_device(&pdev->dev); - - unhandled += 1; - continue; - } - - dev_data = get_dev_data(&pdev->dev); - - if (!dev_data->passthrough) - pdev->dev.archdata.dma_ops = &amd_iommu_dma_ops; - else - pdev->dev.archdata.dma_ops = &nommu_dma_ops; - } - - return unhandled; -} - -/* - * The function which clues the AMD IOMMU driver into dma_ops. - */ - -void __init amd_iommu_init_api(void) -{ - bus_set_iommu(&pci_bus_type, &amd_iommu_ops); + return bus_set_iommu(&pci_bus_type, &amd_iommu_ops); } int __init amd_iommu_init_dma_ops(void) { - struct amd_iommu *iommu; - int ret, unhandled; - - /* - * first allocate a default protection domain for every IOMMU we - * found in the system. Devices not assigned to any other - * protection domain will be assigned to the default one. - */ - for_each_iommu(iommu) { - iommu->default_dom = dma_ops_domain_alloc(); - if (iommu->default_dom == NULL) - return -ENOMEM; - iommu->default_dom->domain.flags |= PD_DEFAULT_MASK; - ret = iommu_init_unity_mappings(iommu); - if (ret) - goto free_domains; - } - - /* - * Pre-allocate the protection domains for each device. - */ - prealloc_protection_domains(); - iommu_detected = 1; swiotlb = 0; - /* Make the driver finally visible to the drivers */ - unhandled = device_dma_ops_init(); - if (unhandled && max_pfn > MAX_DMA32_PFN) { - /* There are unhandled devices - initialize swiotlb for them */ - swiotlb = 1; - } - amd_iommu_stats_init(); if (amd_iommu_unmap_flush) @@ -3150,14 +2874,6 @@ int __init amd_iommu_init_dma_ops(void) pr_info("AMD-Vi: Lazy IO/TLB flushing enabled\n"); return 0; - -free_domains: - - for_each_iommu(iommu) { - dma_ops_domain_free(iommu->default_dom); - } - - return ret; } /***************************************************************************** @@ -3224,7 +2940,7 @@ out_err: return NULL; } -static int __init alloc_passthrough_domain(void) +static int alloc_passthrough_domain(void) { if (pt_domain != NULL) return 0; @@ -3238,42 +2954,61 @@ static int __init alloc_passthrough_domain(void) return 0; } -static int amd_iommu_domain_init(struct iommu_domain *dom) -{ - struct protection_domain *domain; - - domain = protection_domain_alloc(); - if (!domain) - goto out_free; - domain->mode = PAGE_MODE_3_LEVEL; - domain->pt_root = (void *)get_zeroed_page(GFP_KERNEL); - if (!domain->pt_root) - goto out_free; +static struct iommu_domain *amd_iommu_domain_alloc(unsigned type) +{ + struct protection_domain *pdomain; + struct dma_ops_domain *dma_domain; - domain->iommu_domain = dom; + switch (type) { + case IOMMU_DOMAIN_UNMANAGED: + pdomain = protection_domain_alloc(); + if (!pdomain) + return NULL; - dom->priv = domain; + pdomain->mode = PAGE_MODE_3_LEVEL; + pdomain->pt_root = (void *)get_zeroed_page(GFP_KERNEL); + if (!pdomain->pt_root) { + protection_domain_free(pdomain); + return NULL; + } - dom->geometry.aperture_start = 0; - dom->geometry.aperture_end = ~0ULL; - dom->geometry.force_aperture = true; + pdomain->domain.geometry.aperture_start = 0; + pdomain->domain.geometry.aperture_end = ~0ULL; + pdomain->domain.geometry.force_aperture = true; - return 0; + break; + case IOMMU_DOMAIN_DMA: + dma_domain = dma_ops_domain_alloc(); + if (!dma_domain) { + pr_err("AMD-Vi: Failed to allocate\n"); + return NULL; + } + pdomain = &dma_domain->domain; + break; + case IOMMU_DOMAIN_IDENTITY: + pdomain = protection_domain_alloc(); + if (!pdomain) + return NULL; -out_free: - protection_domain_free(domain); + pdomain->mode = PAGE_MODE_NONE; + break; + default: + return NULL; + } - return -ENOMEM; + return &pdomain->domain; } -static void amd_iommu_domain_destroy(struct iommu_domain *dom) +static void amd_iommu_domain_free(struct iommu_domain *dom) { - struct protection_domain *domain = dom->priv; + struct protection_domain *domain; - if (!domain) + if (!dom) return; + domain = to_pdomain(dom); + if (domain->dev_cnt > 0) cleanup_domain(domain); @@ -3286,8 +3021,6 @@ static void amd_iommu_domain_destroy(struct iommu_domain *dom) free_gcr3_table(domain); protection_domain_free(domain); - - dom->priv = NULL; } static void amd_iommu_detach_device(struct iommu_domain *dom, @@ -3315,7 +3048,7 @@ static void amd_iommu_detach_device(struct iommu_domain *dom, static int amd_iommu_attach_device(struct iommu_domain *dom, struct device *dev) { - struct protection_domain *domain = dom->priv; + struct protection_domain *domain = to_pdomain(dom); struct iommu_dev_data *dev_data; struct amd_iommu *iommu; int ret; @@ -3342,7 +3075,7 @@ static int amd_iommu_attach_device(struct iommu_domain *dom, static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova, phys_addr_t paddr, size_t page_size, int iommu_prot) { - struct protection_domain *domain = dom->priv; + struct protection_domain *domain = to_pdomain(dom); int prot = 0; int ret; @@ -3364,7 +3097,7 @@ static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova, static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova, size_t page_size) { - struct protection_domain *domain = dom->priv; + struct protection_domain *domain = to_pdomain(dom); size_t unmap_size; if (domain->mode == PAGE_MODE_NONE) @@ -3382,7 +3115,7 @@ static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova, static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom, dma_addr_t iova) { - struct protection_domain *domain = dom->priv; + struct protection_domain *domain = to_pdomain(dom); unsigned long offset_mask, pte_pgsize; u64 *pte, __pte; @@ -3414,16 +3147,61 @@ static bool amd_iommu_capable(enum iommu_cap cap) return false; } +static void amd_iommu_get_dm_regions(struct device *dev, + struct list_head *head) +{ + struct unity_map_entry *entry; + u16 devid; + + devid = get_device_id(dev); + + list_for_each_entry(entry, &amd_iommu_unity_map, list) { + struct iommu_dm_region *region; + + if (devid < entry->devid_start || devid > entry->devid_end) + continue; + + region = kzalloc(sizeof(*region), GFP_KERNEL); + if (!region) { + pr_err("Out of memory allocating dm-regions for %s\n", + dev_name(dev)); + return; + } + + region->start = entry->address_start; + region->length = entry->address_end - entry->address_start; + if (entry->prot & IOMMU_PROT_IR) + region->prot |= IOMMU_READ; + if (entry->prot & IOMMU_PROT_IW) + region->prot |= IOMMU_WRITE; + + list_add_tail(®ion->list, head); + } +} + +static void amd_iommu_put_dm_regions(struct device *dev, + struct list_head *head) +{ + struct iommu_dm_region *entry, *next; + + list_for_each_entry_safe(entry, next, head, list) + kfree(entry); +} + static const struct iommu_ops amd_iommu_ops = { .capable = amd_iommu_capable, - .domain_init = amd_iommu_domain_init, - .domain_destroy = amd_iommu_domain_destroy, + .domain_alloc = amd_iommu_domain_alloc, + .domain_free = amd_iommu_domain_free, .attach_dev = amd_iommu_attach_device, .detach_dev = amd_iommu_detach_device, .map = amd_iommu_map, .unmap = amd_iommu_unmap, .map_sg = default_iommu_map_sg, .iova_to_phys = amd_iommu_iova_to_phys, + .add_device = amd_iommu_add_device, + .remove_device = amd_iommu_remove_device, + .get_dm_regions = amd_iommu_get_dm_regions, + .put_dm_regions = amd_iommu_put_dm_regions, .pgsize_bitmap = AMD_IOMMU_PGSIZES, }; @@ -3479,7 +3257,7 @@ EXPORT_SYMBOL(amd_iommu_unregister_ppr_notifier); void amd_iommu_domain_direct_map(struct iommu_domain *dom) { - struct protection_domain *domain = dom->priv; + struct protection_domain *domain = to_pdomain(dom); unsigned long flags; spin_lock_irqsave(&domain->lock, flags); @@ -3500,7 +3278,7 @@ EXPORT_SYMBOL(amd_iommu_domain_direct_map); int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids) { - struct protection_domain *domain = dom->priv; + struct protection_domain *domain = to_pdomain(dom); unsigned long flags; int levels, ret; @@ -3612,7 +3390,7 @@ static int __amd_iommu_flush_page(struct protection_domain *domain, int pasid, int amd_iommu_flush_page(struct iommu_domain *dom, int pasid, u64 address) { - struct protection_domain *domain = dom->priv; + struct protection_domain *domain = to_pdomain(dom); unsigned long flags; int ret; @@ -3634,7 +3412,7 @@ static int __amd_iommu_flush_tlb(struct protection_domain *domain, int pasid) int amd_iommu_flush_tlb(struct iommu_domain *dom, int pasid) { - struct protection_domain *domain = dom->priv; + struct protection_domain *domain = to_pdomain(dom); unsigned long flags; int ret; @@ -3714,7 +3492,7 @@ static int __clear_gcr3(struct protection_domain *domain, int pasid) int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, int pasid, unsigned long cr3) { - struct protection_domain *domain = dom->priv; + struct protection_domain *domain = to_pdomain(dom); unsigned long flags; int ret; @@ -3728,7 +3506,7 @@ EXPORT_SYMBOL(amd_iommu_domain_set_gcr3); int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, int pasid) { - struct protection_domain *domain = dom->priv; + struct protection_domain *domain = to_pdomain(dom); unsigned long flags; int ret; @@ -3761,17 +3539,17 @@ EXPORT_SYMBOL(amd_iommu_complete_ppr); struct iommu_domain *amd_iommu_get_v2_domain(struct pci_dev *pdev) { - struct protection_domain *domain; + struct protection_domain *pdomain; - domain = get_domain(&pdev->dev); - if (IS_ERR(domain)) + pdomain = get_domain(&pdev->dev); + if (IS_ERR( |