Add a flush_iotlb_range to allow flushing of an iova range instead of a
full flush in the dma-iommu path.
Allow the iommu_unmap_fast to return newly freed page table pages and
pass the freelist to queue_iova in the dma-iommu ops path.
This patch is useful for iommu drivers (in this case the intel iommu
driver) which need to wait for the ioTLB to be flushed before newly
free/unmapped page table pages can be freed. This way we can still batch
ioTLB free operations and handle the freelists.
Change-log:
V2:
-fix missing parameter in mtk_iommu_v1.c
Signed-off-by: Tom Murphy <[email protected]>
---
drivers/iommu/amd/iommu.c | 14 ++++++++-
drivers/iommu/arm-smmu-v3.c | 3 +-
drivers/iommu/arm-smmu.c | 3 +-
drivers/iommu/dma-iommu.c | 45 ++++++++++++++++++++-------
drivers/iommu/exynos-iommu.c | 3 +-
drivers/iommu/intel/iommu.c | 54 +++++++++++++++++++++------------
drivers/iommu/iommu.c | 25 +++++++++++----
drivers/iommu/ipmmu-vmsa.c | 3 +-
drivers/iommu/msm_iommu.c | 3 +-
drivers/iommu/mtk_iommu.c | 3 +-
drivers/iommu/mtk_iommu_v1.c | 3 +-
drivers/iommu/omap-iommu.c | 3 +-
drivers/iommu/qcom_iommu.c | 3 +-
drivers/iommu/rockchip-iommu.c | 3 +-
drivers/iommu/s390-iommu.c | 3 +-
drivers/iommu/sun50i-iommu.c | 3 +-
drivers/iommu/tegra-gart.c | 3 +-
drivers/iommu/tegra-smmu.c | 3 +-
drivers/iommu/virtio-iommu.c | 3 +-
drivers/vfio/vfio_iommu_type1.c | 2 +-
include/linux/iommu.h | 21 +++++++++++--
21 files changed, 150 insertions(+), 56 deletions(-)
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index 2f22326ee4df..25fbacab23c3 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -2513,7 +2513,8 @@ static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
size_t page_size,
- struct iommu_iotlb_gather *gather)
+ struct iommu_iotlb_gather *gather,
+ struct page **freelist)
{
struct protection_domain *domain = to_pdomain(dom);
struct domain_pgtable pgtable;
@@ -2636,6 +2637,16 @@ static void amd_iommu_flush_iotlb_all(struct iommu_domain *domain)
spin_unlock_irqrestore(&dom->lock, flags);
}
+static void amd_iommu_flush_iotlb_range(struct iommu_domain *domain,
+ unsigned long iova, size_t size,
+ struct page *freelist)
+{
+ struct protection_domain *dom = to_pdomain(domain);
+
+ domain_flush_pages(dom, iova, size);
+ domain_flush_complete(dom);
+}
+
static void amd_iommu_iotlb_sync(struct iommu_domain *domain,
struct iommu_iotlb_gather *gather)
{
@@ -2675,6 +2686,7 @@ const struct iommu_ops amd_iommu_ops = {
.is_attach_deferred = amd_iommu_is_attach_deferred,
.pgsize_bitmap = AMD_IOMMU_PGSIZES,
.flush_iotlb_all = amd_iommu_flush_iotlb_all,
+ .flush_iotlb_range = amd_iommu_flush_iotlb_range,
.iotlb_sync = amd_iommu_iotlb_sync,
.def_domain_type = amd_iommu_def_domain_type,
};
diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
index f578677a5c41..8d328dc25326 100644
--- a/drivers/iommu/arm-smmu-v3.c
+++ b/drivers/iommu/arm-smmu-v3.c
@@ -2854,7 +2854,8 @@ static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
}
static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
- size_t size, struct iommu_iotlb_gather *gather)
+ size_t size, struct iommu_iotlb_gather *gather,
+ struct page **freelist)
{
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index 243bc4cb2705..0cd0dfc89875 100644
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -1234,7 +1234,8 @@ static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
}
static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
- size_t size, struct iommu_iotlb_gather *gather)
+ size_t size, struct iommu_iotlb_gather *gather,
+ struct page **freelist)
{
struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 4959f5df21bd..7433f74d921a 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -50,6 +50,19 @@ struct iommu_dma_cookie {
struct iommu_domain *fq_domain;
};
+
+static void iommu_dma_entry_dtor(unsigned long data)
+{
+ struct page *freelist = (struct page *)data;
+
+ while (freelist != NULL) {
+ unsigned long p = (unsigned long)page_address(freelist);
+
+ freelist = freelist->freelist;
+ free_page(p);
+ }
+}
+
static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie)
{
if (cookie->type == IOMMU_DMA_IOVA_COOKIE)
@@ -344,7 +357,8 @@ static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base,
if (!cookie->fq_domain && !iommu_domain_get_attr(domain,
DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE, &attr) && attr) {
cookie->fq_domain = domain;
- init_iova_flush_queue(iovad, iommu_dma_flush_iotlb_all, NULL);
+ init_iova_flush_queue(iovad, iommu_dma_flush_iotlb_all,
+ iommu_dma_entry_dtor);
}
if (!dev)
@@ -438,7 +452,7 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
}
static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
- dma_addr_t iova, size_t size)
+ dma_addr_t iova, size_t size, struct page *freelist)
{
struct iova_domain *iovad = &cookie->iovad;
@@ -447,7 +461,8 @@ static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
cookie->msi_iova -= size;
else if (cookie->fq_domain) /* non-strict mode */
queue_iova(iovad, iova_pfn(iovad, iova),
- size >> iova_shift(iovad), 0);
+ size >> iova_shift(iovad),
+ (unsigned long) freelist);
else
free_iova_fast(iovad, iova_pfn(iovad, iova),
size >> iova_shift(iovad));
@@ -461,18 +476,26 @@ static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr,
struct iova_domain *iovad = &cookie->iovad;
size_t iova_off = iova_offset(iovad, dma_addr);
struct iommu_iotlb_gather iotlb_gather;
+ struct page *freelist = NULL;
size_t unmapped;
dma_addr -= iova_off;
size = iova_align(iovad, size + iova_off);
iommu_iotlb_gather_init(&iotlb_gather);
- unmapped = iommu_unmap_fast(domain, dma_addr, size, &iotlb_gather);
+ unmapped = iommu_unmap_fast(domain, dma_addr, size, &iotlb_gather,
+ &freelist);
WARN_ON(unmapped != size);
- if (!cookie->fq_domain)
- iommu_tlb_sync(domain, &iotlb_gather);
- iommu_dma_free_iova(cookie, dma_addr, size);
+ if (!cookie->fq_domain) {
+ if (domain->ops->flush_iotlb_range)
+ domain->ops->flush_iotlb_range(domain, dma_addr, size,
+ freelist);
+ else
+ iommu_tlb_sync(domain, &iotlb_gather);
+ }
+
+ iommu_dma_free_iova(cookie, dma_addr, size, freelist);
}
static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
@@ -494,7 +517,7 @@ static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
return DMA_MAPPING_ERROR;
if (iommu_map_atomic(domain, iova, phys - iova_off, size, prot)) {
- iommu_dma_free_iova(cookie, iova, size);
+ iommu_dma_free_iova(cookie, iova, size, NULL);
return DMA_MAPPING_ERROR;
}
return iova + iova_off;
@@ -649,7 +672,7 @@ static void *iommu_dma_alloc_remap(struct device *dev, size_t size,
out_free_sg:
sg_free_table(&sgt);
out_free_iova:
- iommu_dma_free_iova(cookie, iova, size);
+ iommu_dma_free_iova(cookie, iova, size, NULL);
out_free_pages:
__iommu_dma_free_pages(pages, count);
return NULL;
@@ -900,7 +923,7 @@ static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
return __finalise_sg(dev, sg, nents, iova);
out_free_iova:
- iommu_dma_free_iova(cookie, iova, iova_len);
+ iommu_dma_free_iova(cookie, iova, iova_len, NULL);
out_restore_sg:
__invalidate_sg(sg, nents);
return 0;
@@ -1194,7 +1217,7 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,
return msi_page;
out_free_iova:
- iommu_dma_free_iova(cookie, iova, size);
+ iommu_dma_free_iova(cookie, iova, size, NULL);
out_free_page:
kfree(msi_page);
return NULL;
diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c
index 60c8a56e4a3f..5d98985e60a2 100644
--- a/drivers/iommu/exynos-iommu.c
+++ b/drivers/iommu/exynos-iommu.c
@@ -1129,7 +1129,8 @@ static void exynos_iommu_tlb_invalidate_entry(struct exynos_iommu_domain *domain
static size_t exynos_iommu_unmap(struct iommu_domain *iommu_domain,
unsigned long l_iova, size_t size,
- struct iommu_iotlb_gather *gather)
+ struct iommu_iotlb_gather *gather,
+ struct page **freelist)
{
struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
sysmmu_iova_t iova = (sysmmu_iova_t)l_iova;
diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
index 237a470e1e9c..878178fe48f8 100644
--- a/drivers/iommu/intel/iommu.c
+++ b/drivers/iommu/intel/iommu.c
@@ -1160,17 +1160,17 @@ static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
pages can only be freed after the IOTLB flush has been done. */
static struct page *domain_unmap(struct dmar_domain *domain,
unsigned long start_pfn,
- unsigned long last_pfn)
+ unsigned long last_pfn,
+ struct page *freelist)
{
- struct page *freelist;
-
BUG_ON(!domain_pfn_supported(domain, start_pfn));
BUG_ON(!domain_pfn_supported(domain, last_pfn));
BUG_ON(start_pfn > last_pfn);
/* we don't need lock here; nobody else touches the iova range */
freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
- domain->pgd, 0, start_pfn, last_pfn, NULL);
+ domain->pgd, 0, start_pfn, last_pfn,
+ freelist);
/* free pgd */
if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
@@ -1924,7 +1924,8 @@ static void domain_exit(struct dmar_domain *domain)
if (domain->pgd) {
struct page *freelist;
- freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
+ freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw),
+ NULL);
dma_free_pagelist(freelist);
}
@@ -3480,7 +3481,7 @@ static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
if (dev_is_pci(dev))
pdev = to_pci_dev(dev);
- freelist = domain_unmap(domain, start_pfn, last_pfn);
+ freelist = domain_unmap(domain, start_pfn, last_pfn, NULL);
if (intel_iommu_strict || (pdev && pdev->untrusted) ||
!has_iova_flush_queue(&domain->iovad)) {
iommu_flush_iotlb_psi(iommu, domain, start_pfn,
@@ -4575,7 +4576,8 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb,
struct page *freelist;
freelist = domain_unmap(si_domain,
- start_vpfn, last_vpfn);
+ start_vpfn, last_vpfn,
+ NULL);
rcu_read_lock();
for_each_active_iommu(iommu, drhd)
@@ -5540,13 +5542,12 @@ static int intel_iommu_map(struct iommu_domain *domain,
static size_t intel_iommu_unmap(struct iommu_domain *domain,
unsigned long iova, size_t size,
- struct iommu_iotlb_gather *gather)
+ struct iommu_iotlb_gather *gather,
+ struct page **freelist)
{
struct dmar_domain *dmar_domain = to_dmar_domain(domain);
- struct page *freelist = NULL;
unsigned long start_pfn, last_pfn;
- unsigned int npages;
- int iommu_id, level = 0;
+ int level = 0;
/* Cope with horrid API which requires us to unmap more than the
size argument if it happens to be a large-page mapping. */
@@ -5558,22 +5559,36 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain,
start_pfn = iova >> VTD_PAGE_SHIFT;
last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
- freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
+ *freelist = domain_unmap(dmar_domain, start_pfn, last_pfn, *freelist);
+
+ if (dmar_domain->max_addr == iova + size)
+ dmar_domain->max_addr = iova;
- npages = last_pfn - start_pfn + 1;
+ return size;
+}
+
+static void intel_iommu_flush_iotlb_range(struct iommu_domain *domain, unsigned
+ long iova, size_t size,
+ struct page *freelist)
+{
+ struct dmar_domain *dmar_domain = to_dmar_domain(domain);
+ unsigned long start_pfn, last_pfn;
+ unsigned long iova_pfn = IOVA_PFN(iova);
+ unsigned long nrpages;
+ int iommu_id;
+
+ nrpages = aligned_nrpages(iova, size);
+ start_pfn = mm_to_dma_pfn(iova_pfn);
+ last_pfn = start_pfn + nrpages - 1;
for_each_domain_iommu(iommu_id, dmar_domain)
iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
- start_pfn, npages, !freelist, 0);
+ start_pfn, nrpages, !freelist, 0);
dma_free_pagelist(freelist);
-
- if (dmar_domain->max_addr == iova + size)
- dmar_domain->max_addr = iova;
-
- return size;
}
+
static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
dma_addr_t iova)
{
@@ -6058,6 +6073,7 @@ const struct iommu_ops intel_iommu_ops = {
.aux_get_pasid = intel_iommu_aux_get_pasid,
.map = intel_iommu_map,
.unmap = intel_iommu_unmap,
+ .flush_iotlb_range = intel_iommu_flush_iotlb_range,
.iova_to_phys = intel_iommu_iova_to_phys,
.probe_device = intel_iommu_probe_device,
.probe_finalize = intel_iommu_probe_finalize,
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index b6858adc4f17..9065127d7e9c 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -2245,11 +2245,13 @@ EXPORT_SYMBOL_GPL(iommu_map_atomic);
static size_t __iommu_unmap(struct iommu_domain *domain,
unsigned long iova, size_t size,
- struct iommu_iotlb_gather *iotlb_gather)
+ struct iommu_iotlb_gather *iotlb_gather,
+ struct page **freelist)
{
const struct iommu_ops *ops = domain->ops;
size_t unmapped_page, unmapped = 0;
unsigned long orig_iova = iova;
+ struct page *freelist_head = NULL;
unsigned int min_pagesz;
if (unlikely(ops->unmap == NULL ||
@@ -2282,7 +2284,8 @@ static size_t __iommu_unmap(struct iommu_domain *domain,
while (unmapped < size) {
size_t pgsize = iommu_pgsize(domain, iova, size - unmapped);
- unmapped_page = ops->unmap(domain, iova, pgsize, iotlb_gather);
+ unmapped_page = ops->unmap(domain, iova, pgsize, iotlb_gather,
+ &freelist_head);
if (!unmapped_page)
break;
@@ -2293,6 +2296,9 @@ static size_t __iommu_unmap(struct iommu_domain *domain,
unmapped += unmapped_page;
}
+ if (freelist)
+ *freelist = freelist_head;
+
trace_unmap(orig_iova, size, unmapped);
return unmapped;
}
@@ -2300,12 +2306,18 @@ static size_t __iommu_unmap(struct iommu_domain *domain,
size_t iommu_unmap(struct iommu_domain *domain,
unsigned long iova, size_t size)
{
+ const struct iommu_ops *ops = domain->ops;
struct iommu_iotlb_gather iotlb_gather;
+ struct page *freelist;
size_t ret;
iommu_iotlb_gather_init(&iotlb_gather);
- ret = __iommu_unmap(domain, iova, size, &iotlb_gather);
- iommu_tlb_sync(domain, &iotlb_gather);
+ ret = __iommu_unmap(domain, iova, size, &iotlb_gather, &freelist);
+
+ if (ops->flush_iotlb_range)
+ ops->flush_iotlb_range(domain, iova, ret, freelist);
+ else
+ iommu_tlb_sync(domain, &iotlb_gather);
return ret;
}
@@ -2313,9 +2325,10 @@ EXPORT_SYMBOL_GPL(iommu_unmap);
size_t iommu_unmap_fast(struct iommu_domain *domain,
unsigned long iova, size_t size,
- struct iommu_iotlb_gather *iotlb_gather)
+ struct iommu_iotlb_gather *iotlb_gather,
+ struct page **freelist)
{
- return __iommu_unmap(domain, iova, size, iotlb_gather);
+ return __iommu_unmap(domain, iova, size, iotlb_gather, freelist);
}
EXPORT_SYMBOL_GPL(iommu_unmap_fast);
diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c
index 6de86e73dfc3..08c39e95e387 100644
--- a/drivers/iommu/ipmmu-vmsa.c
+++ b/drivers/iommu/ipmmu-vmsa.c
@@ -690,7 +690,8 @@ static int ipmmu_map(struct iommu_domain *io_domain, unsigned long iova,
}
static size_t ipmmu_unmap(struct iommu_domain *io_domain, unsigned long iova,
- size_t size, struct iommu_iotlb_gather *gather)
+ size_t size, struct iommu_iotlb_gather *gather,
+ struct page **freelist)
{
struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c
index 3d8a63555c25..8a987067771c 100644
--- a/drivers/iommu/msm_iommu.c
+++ b/drivers/iommu/msm_iommu.c
@@ -498,7 +498,8 @@ static int msm_iommu_map(struct iommu_domain *domain, unsigned long iova,
}
static size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
- size_t len, struct iommu_iotlb_gather *gather)
+ size_t len, struct iommu_iotlb_gather *gather,
+ struct page **freelist)
{
struct msm_priv *priv = to_msm_priv(domain);
unsigned long flags;
diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c
index 2be96f1cdbd2..b97d35a7d817 100644
--- a/drivers/iommu/mtk_iommu.c
+++ b/drivers/iommu/mtk_iommu.c
@@ -402,7 +402,8 @@ static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova,
static size_t mtk_iommu_unmap(struct iommu_domain *domain,
unsigned long iova, size_t size,
- struct iommu_iotlb_gather *gather)
+ struct iommu_iotlb_gather *gather,
+ struct page **freelist)
{
struct mtk_iommu_domain *dom = to_mtk_domain(domain);
diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c
index c9d79cff4d17..ac44498183b7 100644
--- a/drivers/iommu/mtk_iommu_v1.c
+++ b/drivers/iommu/mtk_iommu_v1.c
@@ -325,7 +325,8 @@ static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova,
static size_t mtk_iommu_unmap(struct iommu_domain *domain,
unsigned long iova, size_t size,
- struct iommu_iotlb_gather *gather)
+ struct iommu_iotlb_gather *gather,
+ struct page **freelist)
{
struct mtk_iommu_domain *dom = to_mtk_domain(domain);
unsigned long flags;
diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c
index c8282cc212cb..17d3cfcb8cd3 100644
--- a/drivers/iommu/omap-iommu.c
+++ b/drivers/iommu/omap-iommu.c
@@ -1367,7 +1367,8 @@ static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,
}
static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da,
- size_t size, struct iommu_iotlb_gather *gather)
+ size_t size, struct iommu_iotlb_gather *gather,
+ struct page **freelist)
{
struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
struct device *dev = omap_domain->dev;
diff --git a/drivers/iommu/qcom_iommu.c b/drivers/iommu/qcom_iommu.c
index d176df569af8..9c66261ce59e 100644
--- a/drivers/iommu/qcom_iommu.c
+++ b/drivers/iommu/qcom_iommu.c
@@ -444,7 +444,8 @@ static int qcom_iommu_map(struct iommu_domain *domain, unsigned long iova,
}
static size_t qcom_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
- size_t size, struct iommu_iotlb_gather *gather)
+ size_t size, struct iommu_iotlb_gather *gather,
+ struct page **freelist)
{
size_t ret;
unsigned long flags;
diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c
index d25c2486ca07..d508c037081a 100644
--- a/drivers/iommu/rockchip-iommu.c
+++ b/drivers/iommu/rockchip-iommu.c
@@ -795,7 +795,8 @@ static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova,
}
static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova,
- size_t size, struct iommu_iotlb_gather *gather)
+ size_t size, struct iommu_iotlb_gather *gather,
+ struct page **freelist)
{
struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
unsigned long flags;
diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c
index 8895dbb705eb..8c96659acbe2 100644
--- a/drivers/iommu/s390-iommu.c
+++ b/drivers/iommu/s390-iommu.c
@@ -305,7 +305,8 @@ static phys_addr_t s390_iommu_iova_to_phys(struct iommu_domain *domain,
static size_t s390_iommu_unmap(struct iommu_domain *domain,
unsigned long iova, size_t size,
- struct iommu_iotlb_gather *gather)
+ struct iommu_iotlb_gather *gather,
+ struct page **freelist)
{
struct s390_domain *s390_domain = to_s390_domain(domain);
int flags = ZPCI_PTE_INVALID;
diff --git a/drivers/iommu/sun50i-iommu.c b/drivers/iommu/sun50i-iommu.c
index 3b1bf2fb94f5..fb0558b82dfa 100644
--- a/drivers/iommu/sun50i-iommu.c
+++ b/drivers/iommu/sun50i-iommu.c
@@ -552,7 +552,8 @@ static int sun50i_iommu_map(struct iommu_domain *domain, unsigned long iova,
}
static size_t sun50i_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
- size_t size, struct iommu_iotlb_gather *gather)
+ size_t size, struct iommu_iotlb_gather *gather,
+ struct page **freelist)
{
struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
phys_addr_t pt_phys;
diff --git a/drivers/iommu/tegra-gart.c b/drivers/iommu/tegra-gart.c
index 5fbdff6ff41a..22168376c429 100644
--- a/drivers/iommu/tegra-gart.c
+++ b/drivers/iommu/tegra-gart.c
@@ -207,7 +207,8 @@ static inline int __gart_iommu_unmap(struct gart_device *gart,
}
static size_t gart_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
- size_t bytes, struct iommu_iotlb_gather *gather)
+ size_t bytes, struct iommu_iotlb_gather *gather,
+ struct page **freelist)
{
struct gart_device *gart = gart_handle;
int err;
diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c
index 7426b7666e2b..1ac2ac851cfe 100644
--- a/drivers/iommu/tegra-smmu.c
+++ b/drivers/iommu/tegra-smmu.c
@@ -686,7 +686,8 @@ static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,
}
static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
- size_t size, struct iommu_iotlb_gather *gather)
+ size_t size, struct iommu_iotlb_gather *gather,
+ struct page **freelist)
{
struct tegra_smmu_as *as = to_smmu_as(domain);
dma_addr_t pte_dma;
diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c
index f6f07489a9aa..ea4d527b5e64 100644
--- a/drivers/iommu/virtio-iommu.c
+++ b/drivers/iommu/virtio-iommu.c
@@ -762,7 +762,8 @@ static int viommu_map(struct iommu_domain *domain, unsigned long iova,
}
static size_t viommu_unmap(struct iommu_domain *domain, unsigned long iova,
- size_t size, struct iommu_iotlb_gather *gather)
+ size_t size, struct iommu_iotlb_gather *gather,
+ struct page **freelist)
{
int ret = 0;
size_t unmapped;
diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index 5e556ac9102a..570ebf878fea 100644
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -812,7 +812,7 @@ static size_t unmap_unpin_fast(struct vfio_domain *domain,
if (entry) {
unmapped = iommu_unmap_fast(domain->domain, *iova, len,
- iotlb_gather);
+ iotlb_gather, NULL);
if (!unmapped) {
kfree(entry);
diff --git a/include/linux/iommu.h b/include/linux/iommu.h
index 5f0b7859d2eb..77e773d03f22 100644
--- a/include/linux/iommu.h
+++ b/include/linux/iommu.h
@@ -198,6 +198,7 @@ struct iommu_iotlb_gather {
* @map: map a physically contiguous memory region to an iommu domain
* @unmap: unmap a physically contiguous memory region from an iommu domain
* @flush_iotlb_all: Synchronously flush all hardware TLBs for this domain
+ * @flush_iotlb_range: Flush given iova range of hardware TLBs for this domain
* @iotlb_sync_map: Sync mappings created recently using @map to the hardware
* @iotlb_sync: Flush all queued ranges from the hardware TLBs and empty flush
* queue
@@ -248,8 +249,12 @@ struct iommu_ops {
int (*map)(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp);
size_t (*unmap)(struct iommu_domain *domain, unsigned long iova,
- size_t size, struct iommu_iotlb_gather *iotlb_gather);
+ size_t size, struct iommu_iotlb_gather *iotlb_gather,
+ struct page **freelist);
void (*flush_iotlb_all)(struct iommu_domain *domain);
+ void (*flush_iotlb_range)(struct iommu_domain *domain,
+ unsigned long iova, size_t size,
+ struct page *freelist);
void (*iotlb_sync_map)(struct iommu_domain *domain);
void (*iotlb_sync)(struct iommu_domain *domain,
struct iommu_iotlb_gather *iotlb_gather);
@@ -447,7 +452,8 @@ extern size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova,
size_t size);
extern size_t iommu_unmap_fast(struct iommu_domain *domain,
unsigned long iova, size_t size,
- struct iommu_iotlb_gather *iotlb_gather);
+ struct iommu_iotlb_gather *iotlb_gather,
+ struct page **freelist);
extern size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
struct scatterlist *sg,unsigned int nents, int prot);
extern size_t iommu_map_sg_atomic(struct iommu_domain *domain,
@@ -542,6 +548,14 @@ static inline void iommu_flush_tlb_all(struct iommu_domain *domain)
domain->ops->flush_iotlb_all(domain);
}
+static inline void flush_iotlb_range(struct iommu_domain *domain,
+ unsigned long iova, size_t size,
+ struct page *freelist)
+{
+ if (domain->ops->flush_iotlb_range)
+ domain->ops->flush_iotlb_range(domain, iova, size, freelist);
+}
+
static inline void iommu_tlb_sync(struct iommu_domain *domain,
struct iommu_iotlb_gather *iotlb_gather)
{
@@ -728,7 +742,8 @@ static inline size_t iommu_unmap(struct iommu_domain *domain,
static inline size_t iommu_unmap_fast(struct iommu_domain *domain,
unsigned long iova, int gfp_order,
- struct iommu_iotlb_gather *iotlb_gather)
+ struct iommu_iotlb_gather *iotlb_gather,
+ struct page **freelist)
{
return 0;
}
--
2.20.1
init_iova_flush_queue can fail if we run out of memory. Fall back to no
flush queue if it fails.
Signed-off-by: Tom Murphy <[email protected]>
---
drivers/iommu/dma-iommu.c | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 7433f74d921a..5445e2be08b5 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -356,9 +356,11 @@ static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base,
if (!cookie->fq_domain && !iommu_domain_get_attr(domain,
DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE, &attr) && attr) {
- cookie->fq_domain = domain;
- init_iova_flush_queue(iovad, iommu_dma_flush_iotlb_all,
- iommu_dma_entry_dtor);
+ if (init_iova_flush_queue(iovad, iommu_dma_flush_iotlb_all,
+ iommu_dma_entry_dtor))
+ pr_warn("iova flush queue initialization failed\n");
+ else
+ cookie->fq_domain = domain;
}
if (!dev)
--
2.20.1
On 2020-08-18 07:04, Tom Murphy wrote:
> Add a flush_iotlb_range to allow flushing of an iova range instead of a
> full flush in the dma-iommu path.
>
> Allow the iommu_unmap_fast to return newly freed page table pages and
> pass the freelist to queue_iova in the dma-iommu ops path.
>
> This patch is useful for iommu drivers (in this case the intel iommu
> driver) which need to wait for the ioTLB to be flushed before newly
> free/unmapped page table pages can be freed. This way we can still batch
> ioTLB free operations and handle the freelists.
It sounds like the freelist is something that logically belongs in the
iommu_iotlb_gather structure. And even if it's not a perfect fit I'd be
inclined to jam it in there anyway just to avoid this giant argument
explosion ;)
Why exactly do we need to introduce a new flush_iotlb_range() op? Can't
the AMD driver simply use the gather mechanism like everyone else?
Robin.
> Change-log:
> V2:
> -fix missing parameter in mtk_iommu_v1.c
>
> Signed-off-by: Tom Murphy <[email protected]>
> ---
> drivers/iommu/amd/iommu.c | 14 ++++++++-
> drivers/iommu/arm-smmu-v3.c | 3 +-
> drivers/iommu/arm-smmu.c | 3 +-
> drivers/iommu/dma-iommu.c | 45 ++++++++++++++++++++-------
> drivers/iommu/exynos-iommu.c | 3 +-
> drivers/iommu/intel/iommu.c | 54 +++++++++++++++++++++------------
> drivers/iommu/iommu.c | 25 +++++++++++----
> drivers/iommu/ipmmu-vmsa.c | 3 +-
> drivers/iommu/msm_iommu.c | 3 +-
> drivers/iommu/mtk_iommu.c | 3 +-
> drivers/iommu/mtk_iommu_v1.c | 3 +-
> drivers/iommu/omap-iommu.c | 3 +-
> drivers/iommu/qcom_iommu.c | 3 +-
> drivers/iommu/rockchip-iommu.c | 3 +-
> drivers/iommu/s390-iommu.c | 3 +-
> drivers/iommu/sun50i-iommu.c | 3 +-
> drivers/iommu/tegra-gart.c | 3 +-
> drivers/iommu/tegra-smmu.c | 3 +-
> drivers/iommu/virtio-iommu.c | 3 +-
> drivers/vfio/vfio_iommu_type1.c | 2 +-
> include/linux/iommu.h | 21 +++++++++++--
> 21 files changed, 150 insertions(+), 56 deletions(-)
>
> diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
> index 2f22326ee4df..25fbacab23c3 100644
> --- a/drivers/iommu/amd/iommu.c
> +++ b/drivers/iommu/amd/iommu.c
> @@ -2513,7 +2513,8 @@ static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
>
> static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
> size_t page_size,
> - struct iommu_iotlb_gather *gather)
> + struct iommu_iotlb_gather *gather,
> + struct page **freelist)
> {
> struct protection_domain *domain = to_pdomain(dom);
> struct domain_pgtable pgtable;
> @@ -2636,6 +2637,16 @@ static void amd_iommu_flush_iotlb_all(struct iommu_domain *domain)
> spin_unlock_irqrestore(&dom->lock, flags);
> }
>
> +static void amd_iommu_flush_iotlb_range(struct iommu_domain *domain,
> + unsigned long iova, size_t size,
> + struct page *freelist)
> +{
> + struct protection_domain *dom = to_pdomain(domain);
> +
> + domain_flush_pages(dom, iova, size);
> + domain_flush_complete(dom);
> +}
> +
> static void amd_iommu_iotlb_sync(struct iommu_domain *domain,
> struct iommu_iotlb_gather *gather)
> {
> @@ -2675,6 +2686,7 @@ const struct iommu_ops amd_iommu_ops = {
> .is_attach_deferred = amd_iommu_is_attach_deferred,
> .pgsize_bitmap = AMD_IOMMU_PGSIZES,
> .flush_iotlb_all = amd_iommu_flush_iotlb_all,
> + .flush_iotlb_range = amd_iommu_flush_iotlb_range,
> .iotlb_sync = amd_iommu_iotlb_sync,
> .def_domain_type = amd_iommu_def_domain_type,
> };
> diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
> index f578677a5c41..8d328dc25326 100644
> --- a/drivers/iommu/arm-smmu-v3.c
> +++ b/drivers/iommu/arm-smmu-v3.c
> @@ -2854,7 +2854,8 @@ static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
> }
>
> static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
> - size_t size, struct iommu_iotlb_gather *gather)
> + size_t size, struct iommu_iotlb_gather *gather,
> + struct page **freelist)
> {
> struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
> struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
> diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
> index 243bc4cb2705..0cd0dfc89875 100644
> --- a/drivers/iommu/arm-smmu.c
> +++ b/drivers/iommu/arm-smmu.c
> @@ -1234,7 +1234,8 @@ static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
> }
>
> static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
> - size_t size, struct iommu_iotlb_gather *gather)
> + size_t size, struct iommu_iotlb_gather *gather,
> + struct page **freelist)
> {
> struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
> struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
> diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
> index 4959f5df21bd..7433f74d921a 100644
> --- a/drivers/iommu/dma-iommu.c
> +++ b/drivers/iommu/dma-iommu.c
> @@ -50,6 +50,19 @@ struct iommu_dma_cookie {
> struct iommu_domain *fq_domain;
> };
>
> +
> +static void iommu_dma_entry_dtor(unsigned long data)
> +{
> + struct page *freelist = (struct page *)data;
> +
> + while (freelist != NULL) {
> + unsigned long p = (unsigned long)page_address(freelist);
> +
> + freelist = freelist->freelist;
> + free_page(p);
> + }
> +}
> +
> static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie)
> {
> if (cookie->type == IOMMU_DMA_IOVA_COOKIE)
> @@ -344,7 +357,8 @@ static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base,
> if (!cookie->fq_domain && !iommu_domain_get_attr(domain,
> DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE, &attr) && attr) {
> cookie->fq_domain = domain;
> - init_iova_flush_queue(iovad, iommu_dma_flush_iotlb_all, NULL);
> + init_iova_flush_queue(iovad, iommu_dma_flush_iotlb_all,
> + iommu_dma_entry_dtor);
> }
>
> if (!dev)
> @@ -438,7 +452,7 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
> }
>
> static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
> - dma_addr_t iova, size_t size)
> + dma_addr_t iova, size_t size, struct page *freelist)
> {
> struct iova_domain *iovad = &cookie->iovad;
>
> @@ -447,7 +461,8 @@ static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
> cookie->msi_iova -= size;
> else if (cookie->fq_domain) /* non-strict mode */
> queue_iova(iovad, iova_pfn(iovad, iova),
> - size >> iova_shift(iovad), 0);
> + size >> iova_shift(iovad),
> + (unsigned long) freelist);
> else
> free_iova_fast(iovad, iova_pfn(iovad, iova),
> size >> iova_shift(iovad));
> @@ -461,18 +476,26 @@ static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr,
> struct iova_domain *iovad = &cookie->iovad;
> size_t iova_off = iova_offset(iovad, dma_addr);
> struct iommu_iotlb_gather iotlb_gather;
> + struct page *freelist = NULL;
> size_t unmapped;
>
> dma_addr -= iova_off;
> size = iova_align(iovad, size + iova_off);
> iommu_iotlb_gather_init(&iotlb_gather);
>
> - unmapped = iommu_unmap_fast(domain, dma_addr, size, &iotlb_gather);
> + unmapped = iommu_unmap_fast(domain, dma_addr, size, &iotlb_gather,
> + &freelist);
> WARN_ON(unmapped != size);
>
> - if (!cookie->fq_domain)
> - iommu_tlb_sync(domain, &iotlb_gather);
> - iommu_dma_free_iova(cookie, dma_addr, size);
> + if (!cookie->fq_domain) {
> + if (domain->ops->flush_iotlb_range)
> + domain->ops->flush_iotlb_range(domain, dma_addr, size,
> + freelist);
> + else
> + iommu_tlb_sync(domain, &iotlb_gather);
> + }
> +
> + iommu_dma_free_iova(cookie, dma_addr, size, freelist);
> }
>
> static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
> @@ -494,7 +517,7 @@ static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
> return DMA_MAPPING_ERROR;
>
> if (iommu_map_atomic(domain, iova, phys - iova_off, size, prot)) {
> - iommu_dma_free_iova(cookie, iova, size);
> + iommu_dma_free_iova(cookie, iova, size, NULL);
> return DMA_MAPPING_ERROR;
> }
> return iova + iova_off;
> @@ -649,7 +672,7 @@ static void *iommu_dma_alloc_remap(struct device *dev, size_t size,
> out_free_sg:
> sg_free_table(&sgt);
> out_free_iova:
> - iommu_dma_free_iova(cookie, iova, size);
> + iommu_dma_free_iova(cookie, iova, size, NULL);
> out_free_pages:
> __iommu_dma_free_pages(pages, count);
> return NULL;
> @@ -900,7 +923,7 @@ static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
> return __finalise_sg(dev, sg, nents, iova);
>
> out_free_iova:
> - iommu_dma_free_iova(cookie, iova, iova_len);
> + iommu_dma_free_iova(cookie, iova, iova_len, NULL);
> out_restore_sg:
> __invalidate_sg(sg, nents);
> return 0;
> @@ -1194,7 +1217,7 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,
> return msi_page;
>
> out_free_iova:
> - iommu_dma_free_iova(cookie, iova, size);
> + iommu_dma_free_iova(cookie, iova, size, NULL);
> out_free_page:
> kfree(msi_page);
> return NULL;
> diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c
> index 60c8a56e4a3f..5d98985e60a2 100644
> --- a/drivers/iommu/exynos-iommu.c
> +++ b/drivers/iommu/exynos-iommu.c
> @@ -1129,7 +1129,8 @@ static void exynos_iommu_tlb_invalidate_entry(struct exynos_iommu_domain *domain
>
> static size_t exynos_iommu_unmap(struct iommu_domain *iommu_domain,
> unsigned long l_iova, size_t size,
> - struct iommu_iotlb_gather *gather)
> + struct iommu_iotlb_gather *gather,
> + struct page **freelist)
> {
> struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
> sysmmu_iova_t iova = (sysmmu_iova_t)l_iova;
> diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
> index 237a470e1e9c..878178fe48f8 100644
> --- a/drivers/iommu/intel/iommu.c
> +++ b/drivers/iommu/intel/iommu.c
> @@ -1160,17 +1160,17 @@ static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
> pages can only be freed after the IOTLB flush has been done. */
> static struct page *domain_unmap(struct dmar_domain *domain,
> unsigned long start_pfn,
> - unsigned long last_pfn)
> + unsigned long last_pfn,
> + struct page *freelist)
> {
> - struct page *freelist;
> -
> BUG_ON(!domain_pfn_supported(domain, start_pfn));
> BUG_ON(!domain_pfn_supported(domain, last_pfn));
> BUG_ON(start_pfn > last_pfn);
>
> /* we don't need lock here; nobody else touches the iova range */
> freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
> - domain->pgd, 0, start_pfn, last_pfn, NULL);
> + domain->pgd, 0, start_pfn, last_pfn,
> + freelist);
>
> /* free pgd */
> if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
> @@ -1924,7 +1924,8 @@ static void domain_exit(struct dmar_domain *domain)
> if (domain->pgd) {
> struct page *freelist;
>
> - freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
> + freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw),
> + NULL);
> dma_free_pagelist(freelist);
> }
>
> @@ -3480,7 +3481,7 @@ static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
> if (dev_is_pci(dev))
> pdev = to_pci_dev(dev);
>
> - freelist = domain_unmap(domain, start_pfn, last_pfn);
> + freelist = domain_unmap(domain, start_pfn, last_pfn, NULL);
> if (intel_iommu_strict || (pdev && pdev->untrusted) ||
> !has_iova_flush_queue(&domain->iovad)) {
> iommu_flush_iotlb_psi(iommu, domain, start_pfn,
> @@ -4575,7 +4576,8 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb,
> struct page *freelist;
>
> freelist = domain_unmap(si_domain,
> - start_vpfn, last_vpfn);
> + start_vpfn, last_vpfn,
> + NULL);
>
> rcu_read_lock();
> for_each_active_iommu(iommu, drhd)
> @@ -5540,13 +5542,12 @@ static int intel_iommu_map(struct iommu_domain *domain,
>
> static size_t intel_iommu_unmap(struct iommu_domain *domain,
> unsigned long iova, size_t size,
> - struct iommu_iotlb_gather *gather)
> + struct iommu_iotlb_gather *gather,
> + struct page **freelist)
> {
> struct dmar_domain *dmar_domain = to_dmar_domain(domain);
> - struct page *freelist = NULL;
> unsigned long start_pfn, last_pfn;
> - unsigned int npages;
> - int iommu_id, level = 0;
> + int level = 0;
>
> /* Cope with horrid API which requires us to unmap more than the
> size argument if it happens to be a large-page mapping. */
> @@ -5558,22 +5559,36 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain,
> start_pfn = iova >> VTD_PAGE_SHIFT;
> last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
>
> - freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
> + *freelist = domain_unmap(dmar_domain, start_pfn, last_pfn, *freelist);
> +
> + if (dmar_domain->max_addr == iova + size)
> + dmar_domain->max_addr = iova;
>
> - npages = last_pfn - start_pfn + 1;
> + return size;
> +}
> +
> +static void intel_iommu_flush_iotlb_range(struct iommu_domain *domain, unsigned
> + long iova, size_t size,
> + struct page *freelist)
> +{
> + struct dmar_domain *dmar_domain = to_dmar_domain(domain);
> + unsigned long start_pfn, last_pfn;
> + unsigned long iova_pfn = IOVA_PFN(iova);
> + unsigned long nrpages;
> + int iommu_id;
> +
> + nrpages = aligned_nrpages(iova, size);
> + start_pfn = mm_to_dma_pfn(iova_pfn);
> + last_pfn = start_pfn + nrpages - 1;
>
> for_each_domain_iommu(iommu_id, dmar_domain)
> iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
> - start_pfn, npages, !freelist, 0);
> + start_pfn, nrpages, !freelist, 0);
>
> dma_free_pagelist(freelist);
> -
> - if (dmar_domain->max_addr == iova + size)
> - dmar_domain->max_addr = iova;
> -
> - return size;
> }
>
> +
> static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
> dma_addr_t iova)
> {
> @@ -6058,6 +6073,7 @@ const struct iommu_ops intel_iommu_ops = {
> .aux_get_pasid = intel_iommu_aux_get_pasid,
> .map = intel_iommu_map,
> .unmap = intel_iommu_unmap,
> + .flush_iotlb_range = intel_iommu_flush_iotlb_range,
> .iova_to_phys = intel_iommu_iova_to_phys,
> .probe_device = intel_iommu_probe_device,
> .probe_finalize = intel_iommu_probe_finalize,
> diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
> index b6858adc4f17..9065127d7e9c 100644
> --- a/drivers/iommu/iommu.c
> +++ b/drivers/iommu/iommu.c
> @@ -2245,11 +2245,13 @@ EXPORT_SYMBOL_GPL(iommu_map_atomic);
>
> static size_t __iommu_unmap(struct iommu_domain *domain,
> unsigned long iova, size_t size,
> - struct iommu_iotlb_gather *iotlb_gather)
> + struct iommu_iotlb_gather *iotlb_gather,
> + struct page **freelist)
> {
> const struct iommu_ops *ops = domain->ops;
> size_t unmapped_page, unmapped = 0;
> unsigned long orig_iova = iova;
> + struct page *freelist_head = NULL;
> unsigned int min_pagesz;
>
> if (unlikely(ops->unmap == NULL ||
> @@ -2282,7 +2284,8 @@ static size_t __iommu_unmap(struct iommu_domain *domain,
> while (unmapped < size) {
> size_t pgsize = iommu_pgsize(domain, iova, size - unmapped);
>
> - unmapped_page = ops->unmap(domain, iova, pgsize, iotlb_gather);
> + unmapped_page = ops->unmap(domain, iova, pgsize, iotlb_gather,
> + &freelist_head);
> if (!unmapped_page)
> break;
>
> @@ -2293,6 +2296,9 @@ static size_t __iommu_unmap(struct iommu_domain *domain,
> unmapped += unmapped_page;
> }
>
> + if (freelist)
> + *freelist = freelist_head;
> +
> trace_unmap(orig_iova, size, unmapped);
> return unmapped;
> }
> @@ -2300,12 +2306,18 @@ static size_t __iommu_unmap(struct iommu_domain *domain,
> size_t iommu_unmap(struct iommu_domain *domain,
> unsigned long iova, size_t size)
> {
> + const struct iommu_ops *ops = domain->ops;
> struct iommu_iotlb_gather iotlb_gather;
> + struct page *freelist;
> size_t ret;
>
> iommu_iotlb_gather_init(&iotlb_gather);
> - ret = __iommu_unmap(domain, iova, size, &iotlb_gather);
> - iommu_tlb_sync(domain, &iotlb_gather);
> + ret = __iommu_unmap(domain, iova, size, &iotlb_gather, &freelist);
> +
> + if (ops->flush_iotlb_range)
> + ops->flush_iotlb_range(domain, iova, ret, freelist);
> + else
> + iommu_tlb_sync(domain, &iotlb_gather);
>
> return ret;
> }
> @@ -2313,9 +2325,10 @@ EXPORT_SYMBOL_GPL(iommu_unmap);
>
> size_t iommu_unmap_fast(struct iommu_domain *domain,
> unsigned long iova, size_t size,
> - struct iommu_iotlb_gather *iotlb_gather)
> + struct iommu_iotlb_gather *iotlb_gather,
> + struct page **freelist)
> {
> - return __iommu_unmap(domain, iova, size, iotlb_gather);
> + return __iommu_unmap(domain, iova, size, iotlb_gather, freelist);
> }
> EXPORT_SYMBOL_GPL(iommu_unmap_fast);
>
> diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c
> index 6de86e73dfc3..08c39e95e387 100644
> --- a/drivers/iommu/ipmmu-vmsa.c
> +++ b/drivers/iommu/ipmmu-vmsa.c
> @@ -690,7 +690,8 @@ static int ipmmu_map(struct iommu_domain *io_domain, unsigned long iova,
> }
>
> static size_t ipmmu_unmap(struct iommu_domain *io_domain, unsigned long iova,
> - size_t size, struct iommu_iotlb_gather *gather)
> + size_t size, struct iommu_iotlb_gather *gather,
> + struct page **freelist)
> {
> struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
>
> diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c
> index 3d8a63555c25..8a987067771c 100644
> --- a/drivers/iommu/msm_iommu.c
> +++ b/drivers/iommu/msm_iommu.c
> @@ -498,7 +498,8 @@ static int msm_iommu_map(struct iommu_domain *domain, unsigned long iova,
> }
>
> static size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
> - size_t len, struct iommu_iotlb_gather *gather)
> + size_t len, struct iommu_iotlb_gather *gather,
> + struct page **freelist)
> {
> struct msm_priv *priv = to_msm_priv(domain);
> unsigned long flags;
> diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c
> index 2be96f1cdbd2..b97d35a7d817 100644
> --- a/drivers/iommu/mtk_iommu.c
> +++ b/drivers/iommu/mtk_iommu.c
> @@ -402,7 +402,8 @@ static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova,
>
> static size_t mtk_iommu_unmap(struct iommu_domain *domain,
> unsigned long iova, size_t size,
> - struct iommu_iotlb_gather *gather)
> + struct iommu_iotlb_gather *gather,
> + struct page **freelist)
> {
> struct mtk_iommu_domain *dom = to_mtk_domain(domain);
>
> diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c
> index c9d79cff4d17..ac44498183b7 100644
> --- a/drivers/iommu/mtk_iommu_v1.c
> +++ b/drivers/iommu/mtk_iommu_v1.c
> @@ -325,7 +325,8 @@ static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova,
>
> static size_t mtk_iommu_unmap(struct iommu_domain *domain,
> unsigned long iova, size_t size,
> - struct iommu_iotlb_gather *gather)
> + struct iommu_iotlb_gather *gather,
> + struct page **freelist)
> {
> struct mtk_iommu_domain *dom = to_mtk_domain(domain);
> unsigned long flags;
> diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c
> index c8282cc212cb..17d3cfcb8cd3 100644
> --- a/drivers/iommu/omap-iommu.c
> +++ b/drivers/iommu/omap-iommu.c
> @@ -1367,7 +1367,8 @@ static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,
> }
>
> static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da,
> - size_t size, struct iommu_iotlb_gather *gather)
> + size_t size, struct iommu_iotlb_gather *gather,
> + struct page **freelist)
> {
> struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
> struct device *dev = omap_domain->dev;
> diff --git a/drivers/iommu/qcom_iommu.c b/drivers/iommu/qcom_iommu.c
> index d176df569af8..9c66261ce59e 100644
> --- a/drivers/iommu/qcom_iommu.c
> +++ b/drivers/iommu/qcom_iommu.c
> @@ -444,7 +444,8 @@ static int qcom_iommu_map(struct iommu_domain *domain, unsigned long iova,
> }
>
> static size_t qcom_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
> - size_t size, struct iommu_iotlb_gather *gather)
> + size_t size, struct iommu_iotlb_gather *gather,
> + struct page **freelist)
> {
> size_t ret;
> unsigned long flags;
> diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c
> index d25c2486ca07..d508c037081a 100644
> --- a/drivers/iommu/rockchip-iommu.c
> +++ b/drivers/iommu/rockchip-iommu.c
> @@ -795,7 +795,8 @@ static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova,
> }
>
> static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova,
> - size_t size, struct iommu_iotlb_gather *gather)
> + size_t size, struct iommu_iotlb_gather *gather,
> + struct page **freelist)
> {
> struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
> unsigned long flags;
> diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c
> index 8895dbb705eb..8c96659acbe2 100644
> --- a/drivers/iommu/s390-iommu.c
> +++ b/drivers/iommu/s390-iommu.c
> @@ -305,7 +305,8 @@ static phys_addr_t s390_iommu_iova_to_phys(struct iommu_domain *domain,
>
> static size_t s390_iommu_unmap(struct iommu_domain *domain,
> unsigned long iova, size_t size,
> - struct iommu_iotlb_gather *gather)
> + struct iommu_iotlb_gather *gather,
> + struct page **freelist)
> {
> struct s390_domain *s390_domain = to_s390_domain(domain);
> int flags = ZPCI_PTE_INVALID;
> diff --git a/drivers/iommu/sun50i-iommu.c b/drivers/iommu/sun50i-iommu.c
> index 3b1bf2fb94f5..fb0558b82dfa 100644
> --- a/drivers/iommu/sun50i-iommu.c
> +++ b/drivers/iommu/sun50i-iommu.c
> @@ -552,7 +552,8 @@ static int sun50i_iommu_map(struct iommu_domain *domain, unsigned long iova,
> }
>
> static size_t sun50i_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
> - size_t size, struct iommu_iotlb_gather *gather)
> + size_t size, struct iommu_iotlb_gather *gather,
> + struct page **freelist)
> {
> struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
> phys_addr_t pt_phys;
> diff --git a/drivers/iommu/tegra-gart.c b/drivers/iommu/tegra-gart.c
> index 5fbdff6ff41a..22168376c429 100644
> --- a/drivers/iommu/tegra-gart.c
> +++ b/drivers/iommu/tegra-gart.c
> @@ -207,7 +207,8 @@ static inline int __gart_iommu_unmap(struct gart_device *gart,
> }
>
> static size_t gart_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
> - size_t bytes, struct iommu_iotlb_gather *gather)
> + size_t bytes, struct iommu_iotlb_gather *gather,
> + struct page **freelist)
> {
> struct gart_device *gart = gart_handle;
> int err;
> diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c
> index 7426b7666e2b..1ac2ac851cfe 100644
> --- a/drivers/iommu/tegra-smmu.c
> +++ b/drivers/iommu/tegra-smmu.c
> @@ -686,7 +686,8 @@ static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,
> }
>
> static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
> - size_t size, struct iommu_iotlb_gather *gather)
> + size_t size, struct iommu_iotlb_gather *gather,
> + struct page **freelist)
> {
> struct tegra_smmu_as *as = to_smmu_as(domain);
> dma_addr_t pte_dma;
> diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c
> index f6f07489a9aa..ea4d527b5e64 100644
> --- a/drivers/iommu/virtio-iommu.c
> +++ b/drivers/iommu/virtio-iommu.c
> @@ -762,7 +762,8 @@ static int viommu_map(struct iommu_domain *domain, unsigned long iova,
> }
>
> static size_t viommu_unmap(struct iommu_domain *domain, unsigned long iova,
> - size_t size, struct iommu_iotlb_gather *gather)
> + size_t size, struct iommu_iotlb_gather *gather,
> + struct page **freelist)
> {
> int ret = 0;
> size_t unmapped;
> diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
> index 5e556ac9102a..570ebf878fea 100644
> --- a/drivers/vfio/vfio_iommu_type1.c
> +++ b/drivers/vfio/vfio_iommu_type1.c
> @@ -812,7 +812,7 @@ static size_t unmap_unpin_fast(struct vfio_domain *domain,
>
> if (entry) {
> unmapped = iommu_unmap_fast(domain->domain, *iova, len,
> - iotlb_gather);
> + iotlb_gather, NULL);
>
> if (!unmapped) {
> kfree(entry);
> diff --git a/include/linux/iommu.h b/include/linux/iommu.h
> index 5f0b7859d2eb..77e773d03f22 100644
> --- a/include/linux/iommu.h
> +++ b/include/linux/iommu.h
> @@ -198,6 +198,7 @@ struct iommu_iotlb_gather {
> * @map: map a physically contiguous memory region to an iommu domain
> * @unmap: unmap a physically contiguous memory region from an iommu domain
> * @flush_iotlb_all: Synchronously flush all hardware TLBs for this domain
> + * @flush_iotlb_range: Flush given iova range of hardware TLBs for this domain
> * @iotlb_sync_map: Sync mappings created recently using @map to the hardware
> * @iotlb_sync: Flush all queued ranges from the hardware TLBs and empty flush
> * queue
> @@ -248,8 +249,12 @@ struct iommu_ops {
> int (*map)(struct iommu_domain *domain, unsigned long iova,
> phys_addr_t paddr, size_t size, int prot, gfp_t gfp);
> size_t (*unmap)(struct iommu_domain *domain, unsigned long iova,
> - size_t size, struct iommu_iotlb_gather *iotlb_gather);
> + size_t size, struct iommu_iotlb_gather *iotlb_gather,
> + struct page **freelist);
> void (*flush_iotlb_all)(struct iommu_domain *domain);
> + void (*flush_iotlb_range)(struct iommu_domain *domain,
> + unsigned long iova, size_t size,
> + struct page *freelist);
> void (*iotlb_sync_map)(struct iommu_domain *domain);
> void (*iotlb_sync)(struct iommu_domain *domain,
> struct iommu_iotlb_gather *iotlb_gather);
> @@ -447,7 +452,8 @@ extern size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova,
> size_t size);
> extern size_t iommu_unmap_fast(struct iommu_domain *domain,
> unsigned long iova, size_t size,
> - struct iommu_iotlb_gather *iotlb_gather);
> + struct iommu_iotlb_gather *iotlb_gather,
> + struct page **freelist);
> extern size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
> struct scatterlist *sg,unsigned int nents, int prot);
> extern size_t iommu_map_sg_atomic(struct iommu_domain *domain,
> @@ -542,6 +548,14 @@ static inline void iommu_flush_tlb_all(struct iommu_domain *domain)
> domain->ops->flush_iotlb_all(domain);
> }
>
> +static inline void flush_iotlb_range(struct iommu_domain *domain,
> + unsigned long iova, size_t size,
> + struct page *freelist)
> +{
> + if (domain->ops->flush_iotlb_range)
> + domain->ops->flush_iotlb_range(domain, iova, size, freelist);
> +}
> +
> static inline void iommu_tlb_sync(struct iommu_domain *domain,
> struct iommu_iotlb_gather *iotlb_gather)
> {
> @@ -728,7 +742,8 @@ static inline size_t iommu_unmap(struct iommu_domain *domain,
>
> static inline size_t iommu_unmap_fast(struct iommu_domain *domain,
> unsigned long iova, int gfp_order,
> - struct iommu_iotlb_gather *iotlb_gather)
> + struct iommu_iotlb_gather *iotlb_gather,
> + struct page **freelist)
> {
> return 0;
> }
>
On Tue, 18 Aug 2020 at 16:17, Robin Murphy <[email protected]> wrote:
>
> On 2020-08-18 07:04, Tom Murphy wrote:
> > Add a flush_iotlb_range to allow flushing of an iova range instead of a
> > full flush in the dma-iommu path.
> >
> > Allow the iommu_unmap_fast to return newly freed page table pages and
> > pass the freelist to queue_iova in the dma-iommu ops path.
> >
> > This patch is useful for iommu drivers (in this case the intel iommu
> > driver) which need to wait for the ioTLB to be flushed before newly
> > free/unmapped page table pages can be freed. This way we can still batch
> > ioTLB free operations and handle the freelists.
>
> It sounds like the freelist is something that logically belongs in the
> iommu_iotlb_gather structure. And even if it's not a perfect fit I'd be
> inclined to jam it in there anyway just to avoid this giant argument
> explosion ;)
Good point, I'll do that.
>
> Why exactly do we need to introduce a new flush_iotlb_range() op? Can't
> the AMD driver simply use the gather mechanism like everyone else?
No, there's no reason it can't simply use the gather mechanism. I will
use the gather mechanism.
I think I wrote this patch way back before the gather mechanism was
introduced and I've been rebasing/slightly updating it since then
without paying proper attention to the code.
>
> Robin.
>
> > Change-log:
> > V2:
> > -fix missing parameter in mtk_iommu_v1.c
> >
> > Signed-off-by: Tom Murphy <[email protected]>
> > ---
> > drivers/iommu/amd/iommu.c | 14 ++++++++-
> > drivers/iommu/arm-smmu-v3.c | 3 +-
> > drivers/iommu/arm-smmu.c | 3 +-
> > drivers/iommu/dma-iommu.c | 45 ++++++++++++++++++++-------
> > drivers/iommu/exynos-iommu.c | 3 +-
> > drivers/iommu/intel/iommu.c | 54 +++++++++++++++++++++------------
> > drivers/iommu/iommu.c | 25 +++++++++++----
> > drivers/iommu/ipmmu-vmsa.c | 3 +-
> > drivers/iommu/msm_iommu.c | 3 +-
> > drivers/iommu/mtk_iommu.c | 3 +-
> > drivers/iommu/mtk_iommu_v1.c | 3 +-
> > drivers/iommu/omap-iommu.c | 3 +-
> > drivers/iommu/qcom_iommu.c | 3 +-
> > drivers/iommu/rockchip-iommu.c | 3 +-
> > drivers/iommu/s390-iommu.c | 3 +-
> > drivers/iommu/sun50i-iommu.c | 3 +-
> > drivers/iommu/tegra-gart.c | 3 +-
> > drivers/iommu/tegra-smmu.c | 3 +-
> > drivers/iommu/virtio-iommu.c | 3 +-
> > drivers/vfio/vfio_iommu_type1.c | 2 +-
> > include/linux/iommu.h | 21 +++++++++++--
> > 21 files changed, 150 insertions(+), 56 deletions(-)
> >
> > diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
> > index 2f22326ee4df..25fbacab23c3 100644
> > --- a/drivers/iommu/amd/iommu.c
> > +++ b/drivers/iommu/amd/iommu.c
> > @@ -2513,7 +2513,8 @@ static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
> >
> > static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
> > size_t page_size,
> > - struct iommu_iotlb_gather *gather)
> > + struct iommu_iotlb_gather *gather,
> > + struct page **freelist)
> > {
> > struct protection_domain *domain = to_pdomain(dom);
> > struct domain_pgtable pgtable;
> > @@ -2636,6 +2637,16 @@ static void amd_iommu_flush_iotlb_all(struct iommu_domain *domain)
> > spin_unlock_irqrestore(&dom->lock, flags);
> > }
> >
> > +static void amd_iommu_flush_iotlb_range(struct iommu_domain *domain,
> > + unsigned long iova, size_t size,
> > + struct page *freelist)
> > +{
> > + struct protection_domain *dom = to_pdomain(domain);
> > +
> > + domain_flush_pages(dom, iova, size);
> > + domain_flush_complete(dom);
> > +}
> > +
> > static void amd_iommu_iotlb_sync(struct iommu_domain *domain,
> > struct iommu_iotlb_gather *gather)
> > {
> > @@ -2675,6 +2686,7 @@ const struct iommu_ops amd_iommu_ops = {
> > .is_attach_deferred = amd_iommu_is_attach_deferred,
> > .pgsize_bitmap = AMD_IOMMU_PGSIZES,
> > .flush_iotlb_all = amd_iommu_flush_iotlb_all,
> > + .flush_iotlb_range = amd_iommu_flush_iotlb_range,
> > .iotlb_sync = amd_iommu_iotlb_sync,
> > .def_domain_type = amd_iommu_def_domain_type,
> > };
> > diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
> > index f578677a5c41..8d328dc25326 100644
> > --- a/drivers/iommu/arm-smmu-v3.c
> > +++ b/drivers/iommu/arm-smmu-v3.c
> > @@ -2854,7 +2854,8 @@ static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
> > }
> >
> > static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
> > - size_t size, struct iommu_iotlb_gather *gather)
> > + size_t size, struct iommu_iotlb_gather *gather,
> > + struct page **freelist)
> > {
> > struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
> > struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
> > diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
> > index 243bc4cb2705..0cd0dfc89875 100644
> > --- a/drivers/iommu/arm-smmu.c
> > +++ b/drivers/iommu/arm-smmu.c
> > @@ -1234,7 +1234,8 @@ static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
> > }
> >
> > static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
> > - size_t size, struct iommu_iotlb_gather *gather)
> > + size_t size, struct iommu_iotlb_gather *gather,
> > + struct page **freelist)
> > {
> > struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
> > struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
> > diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
> > index 4959f5df21bd..7433f74d921a 100644
> > --- a/drivers/iommu/dma-iommu.c
> > +++ b/drivers/iommu/dma-iommu.c
> > @@ -50,6 +50,19 @@ struct iommu_dma_cookie {
> > struct iommu_domain *fq_domain;
> > };
> >
> > +
> > +static void iommu_dma_entry_dtor(unsigned long data)
> > +{
> > + struct page *freelist = (struct page *)data;
> > +
> > + while (freelist != NULL) {
> > + unsigned long p = (unsigned long)page_address(freelist);
> > +
> > + freelist = freelist->freelist;
> > + free_page(p);
> > + }
> > +}
> > +
> > static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie)
> > {
> > if (cookie->type == IOMMU_DMA_IOVA_COOKIE)
> > @@ -344,7 +357,8 @@ static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base,
> > if (!cookie->fq_domain && !iommu_domain_get_attr(domain,
> > DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE, &attr) && attr) {
> > cookie->fq_domain = domain;
> > - init_iova_flush_queue(iovad, iommu_dma_flush_iotlb_all, NULL);
> > + init_iova_flush_queue(iovad, iommu_dma_flush_iotlb_all,
> > + iommu_dma_entry_dtor);
> > }
> >
> > if (!dev)
> > @@ -438,7 +452,7 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
> > }
> >
> > static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
> > - dma_addr_t iova, size_t size)
> > + dma_addr_t iova, size_t size, struct page *freelist)
> > {
> > struct iova_domain *iovad = &cookie->iovad;
> >
> > @@ -447,7 +461,8 @@ static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
> > cookie->msi_iova -= size;
> > else if (cookie->fq_domain) /* non-strict mode */
> > queue_iova(iovad, iova_pfn(iovad, iova),
> > - size >> iova_shift(iovad), 0);
> > + size >> iova_shift(iovad),
> > + (unsigned long) freelist);
> > else
> > free_iova_fast(iovad, iova_pfn(iovad, iova),
> > size >> iova_shift(iovad));
> > @@ -461,18 +476,26 @@ static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr,
> > struct iova_domain *iovad = &cookie->iovad;
> > size_t iova_off = iova_offset(iovad, dma_addr);
> > struct iommu_iotlb_gather iotlb_gather;
> > + struct page *freelist = NULL;
> > size_t unmapped;
> >
> > dma_addr -= iova_off;
> > size = iova_align(iovad, size + iova_off);
> > iommu_iotlb_gather_init(&iotlb_gather);
> >
> > - unmapped = iommu_unmap_fast(domain, dma_addr, size, &iotlb_gather);
> > + unmapped = iommu_unmap_fast(domain, dma_addr, size, &iotlb_gather,
> > + &freelist);
> > WARN_ON(unmapped != size);
> >
> > - if (!cookie->fq_domain)
> > - iommu_tlb_sync(domain, &iotlb_gather);
> > - iommu_dma_free_iova(cookie, dma_addr, size);
> > + if (!cookie->fq_domain) {
> > + if (domain->ops->flush_iotlb_range)
> > + domain->ops->flush_iotlb_range(domain, dma_addr, size,
> > + freelist);
> > + else
> > + iommu_tlb_sync(domain, &iotlb_gather);
> > + }
> > +
> > + iommu_dma_free_iova(cookie, dma_addr, size, freelist);
> > }
> >
> > static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
> > @@ -494,7 +517,7 @@ static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
> > return DMA_MAPPING_ERROR;
> >
> > if (iommu_map_atomic(domain, iova, phys - iova_off, size, prot)) {
> > - iommu_dma_free_iova(cookie, iova, size);
> > + iommu_dma_free_iova(cookie, iova, size, NULL);
> > return DMA_MAPPING_ERROR;
> > }
> > return iova + iova_off;
> > @@ -649,7 +672,7 @@ static void *iommu_dma_alloc_remap(struct device *dev, size_t size,
> > out_free_sg:
> > sg_free_table(&sgt);
> > out_free_iova:
> > - iommu_dma_free_iova(cookie, iova, size);
> > + iommu_dma_free_iova(cookie, iova, size, NULL);
> > out_free_pages:
> > __iommu_dma_free_pages(pages, count);
> > return NULL;
> > @@ -900,7 +923,7 @@ static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
> > return __finalise_sg(dev, sg, nents, iova);
> >
> > out_free_iova:
> > - iommu_dma_free_iova(cookie, iova, iova_len);
> > + iommu_dma_free_iova(cookie, iova, iova_len, NULL);
> > out_restore_sg:
> > __invalidate_sg(sg, nents);
> > return 0;
> > @@ -1194,7 +1217,7 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,
> > return msi_page;
> >
> > out_free_iova:
> > - iommu_dma_free_iova(cookie, iova, size);
> > + iommu_dma_free_iova(cookie, iova, size, NULL);
> > out_free_page:
> > kfree(msi_page);
> > return NULL;
> > diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c
> > index 60c8a56e4a3f..5d98985e60a2 100644
> > --- a/drivers/iommu/exynos-iommu.c
> > +++ b/drivers/iommu/exynos-iommu.c
> > @@ -1129,7 +1129,8 @@ static void exynos_iommu_tlb_invalidate_entry(struct exynos_iommu_domain *domain
> >
> > static size_t exynos_iommu_unmap(struct iommu_domain *iommu_domain,
> > unsigned long l_iova, size_t size,
> > - struct iommu_iotlb_gather *gather)
> > + struct iommu_iotlb_gather *gather,
> > + struct page **freelist)
> > {
> > struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
> > sysmmu_iova_t iova = (sysmmu_iova_t)l_iova;
> > diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
> > index 237a470e1e9c..878178fe48f8 100644
> > --- a/drivers/iommu/intel/iommu.c
> > +++ b/drivers/iommu/intel/iommu.c
> > @@ -1160,17 +1160,17 @@ static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
> > pages can only be freed after the IOTLB flush has been done. */
> > static struct page *domain_unmap(struct dmar_domain *domain,
> > unsigned long start_pfn,
> > - unsigned long last_pfn)
> > + unsigned long last_pfn,
> > + struct page *freelist)
> > {
> > - struct page *freelist;
> > -
> > BUG_ON(!domain_pfn_supported(domain, start_pfn));
> > BUG_ON(!domain_pfn_supported(domain, last_pfn));
> > BUG_ON(start_pfn > last_pfn);
> >
> > /* we don't need lock here; nobody else touches the iova range */
> > freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
> > - domain->pgd, 0, start_pfn, last_pfn, NULL);
> > + domain->pgd, 0, start_pfn, last_pfn,
> > + freelist);
> >
> > /* free pgd */
> > if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
> > @@ -1924,7 +1924,8 @@ static void domain_exit(struct dmar_domain *domain)
> > if (domain->pgd) {
> > struct page *freelist;
> >
> > - freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
> > + freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw),
> > + NULL);
> > dma_free_pagelist(freelist);
> > }
> >
> > @@ -3480,7 +3481,7 @@ static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
> > if (dev_is_pci(dev))
> > pdev = to_pci_dev(dev);
> >
> > - freelist = domain_unmap(domain, start_pfn, last_pfn);
> > + freelist = domain_unmap(domain, start_pfn, last_pfn, NULL);
> > if (intel_iommu_strict || (pdev && pdev->untrusted) ||
> > !has_iova_flush_queue(&domain->iovad)) {
> > iommu_flush_iotlb_psi(iommu, domain, start_pfn,
> > @@ -4575,7 +4576,8 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb,
> > struct page *freelist;
> >
> > freelist = domain_unmap(si_domain,
> > - start_vpfn, last_vpfn);
> > + start_vpfn, last_vpfn,
> > + NULL);
> >
> > rcu_read_lock();
> > for_each_active_iommu(iommu, drhd)
> > @@ -5540,13 +5542,12 @@ static int intel_iommu_map(struct iommu_domain *domain,
> >
> > static size_t intel_iommu_unmap(struct iommu_domain *domain,
> > unsigned long iova, size_t size,
> > - struct iommu_iotlb_gather *gather)
> > + struct iommu_iotlb_gather *gather,
> > + struct page **freelist)
> > {
> > struct dmar_domain *dmar_domain = to_dmar_domain(domain);
> > - struct page *freelist = NULL;
> > unsigned long start_pfn, last_pfn;
> > - unsigned int npages;
> > - int iommu_id, level = 0;
> > + int level = 0;
> >
> > /* Cope with horrid API which requires us to unmap more than the
> > size argument if it happens to be a large-page mapping. */
> > @@ -5558,22 +5559,36 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain,
> > start_pfn = iova >> VTD_PAGE_SHIFT;
> > last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
> >
> > - freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
> > + *freelist = domain_unmap(dmar_domain, start_pfn, last_pfn, *freelist);
> > +
> > + if (dmar_domain->max_addr == iova + size)
> > + dmar_domain->max_addr = iova;
> >
> > - npages = last_pfn - start_pfn + 1;
> > + return size;
> > +}
> > +
> > +static void intel_iommu_flush_iotlb_range(struct iommu_domain *domain, unsigned
> > + long iova, size_t size,
> > + struct page *freelist)
> > +{
> > + struct dmar_domain *dmar_domain = to_dmar_domain(domain);
> > + unsigned long start_pfn, last_pfn;
> > + unsigned long iova_pfn = IOVA_PFN(iova);
> > + unsigned long nrpages;
> > + int iommu_id;
> > +
> > + nrpages = aligned_nrpages(iova, size);
> > + start_pfn = mm_to_dma_pfn(iova_pfn);
> > + last_pfn = start_pfn + nrpages - 1;
> >
> > for_each_domain_iommu(iommu_id, dmar_domain)
> > iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
> > - start_pfn, npages, !freelist, 0);
> > + start_pfn, nrpages, !freelist, 0);
> >
> > dma_free_pagelist(freelist);
> > -
> > - if (dmar_domain->max_addr == iova + size)
> > - dmar_domain->max_addr = iova;
> > -
> > - return size;
> > }
> >
> > +
> > static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
> > dma_addr_t iova)
> > {
> > @@ -6058,6 +6073,7 @@ const struct iommu_ops intel_iommu_ops = {
> > .aux_get_pasid = intel_iommu_aux_get_pasid,
> > .map = intel_iommu_map,
> > .unmap = intel_iommu_unmap,
> > + .flush_iotlb_range = intel_iommu_flush_iotlb_range,
> > .iova_to_phys = intel_iommu_iova_to_phys,
> > .probe_device = intel_iommu_probe_device,
> > .probe_finalize = intel_iommu_probe_finalize,
> > diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
> > index b6858adc4f17..9065127d7e9c 100644
> > --- a/drivers/iommu/iommu.c
> > +++ b/drivers/iommu/iommu.c
> > @@ -2245,11 +2245,13 @@ EXPORT_SYMBOL_GPL(iommu_map_atomic);
> >
> > static size_t __iommu_unmap(struct iommu_domain *domain,
> > unsigned long iova, size_t size,
> > - struct iommu_iotlb_gather *iotlb_gather)
> > + struct iommu_iotlb_gather *iotlb_gather,
> > + struct page **freelist)
> > {
> > const struct iommu_ops *ops = domain->ops;
> > size_t unmapped_page, unmapped = 0;
> > unsigned long orig_iova = iova;
> > + struct page *freelist_head = NULL;
> > unsigned int min_pagesz;
> >
> > if (unlikely(ops->unmap == NULL ||
> > @@ -2282,7 +2284,8 @@ static size_t __iommu_unmap(struct iommu_domain *domain,
> > while (unmapped < size) {
> > size_t pgsize = iommu_pgsize(domain, iova, size - unmapped);
> >
> > - unmapped_page = ops->unmap(domain, iova, pgsize, iotlb_gather);
> > + unmapped_page = ops->unmap(domain, iova, pgsize, iotlb_gather,
> > + &freelist_head);
> > if (!unmapped_page)
> > break;
> >
> > @@ -2293,6 +2296,9 @@ static size_t __iommu_unmap(struct iommu_domain *domain,
> > unmapped += unmapped_page;
> > }
> >
> > + if (freelist)
> > + *freelist = freelist_head;
> > +
> > trace_unmap(orig_iova, size, unmapped);
> > return unmapped;
> > }
> > @@ -2300,12 +2306,18 @@ static size_t __iommu_unmap(struct iommu_domain *domain,
> > size_t iommu_unmap(struct iommu_domain *domain,
> > unsigned long iova, size_t size)
> > {
> > + const struct iommu_ops *ops = domain->ops;
> > struct iommu_iotlb_gather iotlb_gather;
> > + struct page *freelist;
> > size_t ret;
> >
> > iommu_iotlb_gather_init(&iotlb_gather);
> > - ret = __iommu_unmap(domain, iova, size, &iotlb_gather);
> > - iommu_tlb_sync(domain, &iotlb_gather);
> > + ret = __iommu_unmap(domain, iova, size, &iotlb_gather, &freelist);
> > +
> > + if (ops->flush_iotlb_range)
> > + ops->flush_iotlb_range(domain, iova, ret, freelist);
> > + else
> > + iommu_tlb_sync(domain, &iotlb_gather);
> >
> > return ret;
> > }
> > @@ -2313,9 +2325,10 @@ EXPORT_SYMBOL_GPL(iommu_unmap);
> >
> > size_t iommu_unmap_fast(struct iommu_domain *domain,
> > unsigned long iova, size_t size,
> > - struct iommu_iotlb_gather *iotlb_gather)
> > + struct iommu_iotlb_gather *iotlb_gather,
> > + struct page **freelist)
> > {
> > - return __iommu_unmap(domain, iova, size, iotlb_gather);
> > + return __iommu_unmap(domain, iova, size, iotlb_gather, freelist);
> > }
> > EXPORT_SYMBOL_GPL(iommu_unmap_fast);
> >
> > diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c
> > index 6de86e73dfc3..08c39e95e387 100644
> > --- a/drivers/iommu/ipmmu-vmsa.c
> > +++ b/drivers/iommu/ipmmu-vmsa.c
> > @@ -690,7 +690,8 @@ static int ipmmu_map(struct iommu_domain *io_domain, unsigned long iova,
> > }
> >
> > static size_t ipmmu_unmap(struct iommu_domain *io_domain, unsigned long iova,
> > - size_t size, struct iommu_iotlb_gather *gather)
> > + size_t size, struct iommu_iotlb_gather *gather,
> > + struct page **freelist)
> > {
> > struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
> >
> > diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c
> > index 3d8a63555c25..8a987067771c 100644
> > --- a/drivers/iommu/msm_iommu.c
> > +++ b/drivers/iommu/msm_iommu.c
> > @@ -498,7 +498,8 @@ static int msm_iommu_map(struct iommu_domain *domain, unsigned long iova,
> > }
> >
> > static size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
> > - size_t len, struct iommu_iotlb_gather *gather)
> > + size_t len, struct iommu_iotlb_gather *gather,
> > + struct page **freelist)
> > {
> > struct msm_priv *priv = to_msm_priv(domain);
> > unsigned long flags;
> > diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c
> > index 2be96f1cdbd2..b97d35a7d817 100644
> > --- a/drivers/iommu/mtk_iommu.c
> > +++ b/drivers/iommu/mtk_iommu.c
> > @@ -402,7 +402,8 @@ static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova,
> >
> > static size_t mtk_iommu_unmap(struct iommu_domain *domain,
> > unsigned long iova, size_t size,
> > - struct iommu_iotlb_gather *gather)
> > + struct iommu_iotlb_gather *gather,
> > + struct page **freelist)
> > {
> > struct mtk_iommu_domain *dom = to_mtk_domain(domain);
> >
> > diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c
> > index c9d79cff4d17..ac44498183b7 100644
> > --- a/drivers/iommu/mtk_iommu_v1.c
> > +++ b/drivers/iommu/mtk_iommu_v1.c
> > @@ -325,7 +325,8 @@ static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova,
> >
> > static size_t mtk_iommu_unmap(struct iommu_domain *domain,
> > unsigned long iova, size_t size,
> > - struct iommu_iotlb_gather *gather)
> > + struct iommu_iotlb_gather *gather,
> > + struct page **freelist)
> > {
> > struct mtk_iommu_domain *dom = to_mtk_domain(domain);
> > unsigned long flags;
> > diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c
> > index c8282cc212cb..17d3cfcb8cd3 100644
> > --- a/drivers/iommu/omap-iommu.c
> > +++ b/drivers/iommu/omap-iommu.c
> > @@ -1367,7 +1367,8 @@ static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,
> > }
> >
> > static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da,
> > - size_t size, struct iommu_iotlb_gather *gather)
> > + size_t size, struct iommu_iotlb_gather *gather,
> > + struct page **freelist)
> > {
> > struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
> > struct device *dev = omap_domain->dev;
> > diff --git a/drivers/iommu/qcom_iommu.c b/drivers/iommu/qcom_iommu.c
> > index d176df569af8..9c66261ce59e 100644
> > --- a/drivers/iommu/qcom_iommu.c
> > +++ b/drivers/iommu/qcom_iommu.c
> > @@ -444,7 +444,8 @@ static int qcom_iommu_map(struct iommu_domain *domain, unsigned long iova,
> > }
> >
> > static size_t qcom_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
> > - size_t size, struct iommu_iotlb_gather *gather)
> > + size_t size, struct iommu_iotlb_gather *gather,
> > + struct page **freelist)
> > {
> > size_t ret;
> > unsigned long flags;
> > diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c
> > index d25c2486ca07..d508c037081a 100644
> > --- a/drivers/iommu/rockchip-iommu.c
> > +++ b/drivers/iommu/rockchip-iommu.c
> > @@ -795,7 +795,8 @@ static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova,
> > }
> >
> > static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova,
> > - size_t size, struct iommu_iotlb_gather *gather)
> > + size_t size, struct iommu_iotlb_gather *gather,
> > + struct page **freelist)
> > {
> > struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
> > unsigned long flags;
> > diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c
> > index 8895dbb705eb..8c96659acbe2 100644
> > --- a/drivers/iommu/s390-iommu.c
> > +++ b/drivers/iommu/s390-iommu.c
> > @@ -305,7 +305,8 @@ static phys_addr_t s390_iommu_iova_to_phys(struct iommu_domain *domain,
> >
> > static size_t s390_iommu_unmap(struct iommu_domain *domain,
> > unsigned long iova, size_t size,
> > - struct iommu_iotlb_gather *gather)
> > + struct iommu_iotlb_gather *gather,
> > + struct page **freelist)
> > {
> > struct s390_domain *s390_domain = to_s390_domain(domain);
> > int flags = ZPCI_PTE_INVALID;
> > diff --git a/drivers/iommu/sun50i-iommu.c b/drivers/iommu/sun50i-iommu.c
> > index 3b1bf2fb94f5..fb0558b82dfa 100644
> > --- a/drivers/iommu/sun50i-iommu.c
> > +++ b/drivers/iommu/sun50i-iommu.c
> > @@ -552,7 +552,8 @@ static int sun50i_iommu_map(struct iommu_domain *domain, unsigned long iova,
> > }
> >
> > static size_t sun50i_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
> > - size_t size, struct iommu_iotlb_gather *gather)
> > + size_t size, struct iommu_iotlb_gather *gather,
> > + struct page **freelist)
> > {
> > struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
> > phys_addr_t pt_phys;
> > diff --git a/drivers/iommu/tegra-gart.c b/drivers/iommu/tegra-gart.c
> > index 5fbdff6ff41a..22168376c429 100644
> > --- a/drivers/iommu/tegra-gart.c
> > +++ b/drivers/iommu/tegra-gart.c
> > @@ -207,7 +207,8 @@ static inline int __gart_iommu_unmap(struct gart_device *gart,
> > }
> >
> > static size_t gart_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
> > - size_t bytes, struct iommu_iotlb_gather *gather)
> > + size_t bytes, struct iommu_iotlb_gather *gather,
> > + struct page **freelist)
> > {
> > struct gart_device *gart = gart_handle;
> > int err;
> > diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c
> > index 7426b7666e2b..1ac2ac851cfe 100644
> > --- a/drivers/iommu/tegra-smmu.c
> > +++ b/drivers/iommu/tegra-smmu.c
> > @@ -686,7 +686,8 @@ static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,
> > }
> >
> > static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
> > - size_t size, struct iommu_iotlb_gather *gather)
> > + size_t size, struct iommu_iotlb_gather *gather,
> > + struct page **freelist)
> > {
> > struct tegra_smmu_as *as = to_smmu_as(domain);
> > dma_addr_t pte_dma;
> > diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c
> > index f6f07489a9aa..ea4d527b5e64 100644
> > --- a/drivers/iommu/virtio-iommu.c
> > +++ b/drivers/iommu/virtio-iommu.c
> > @@ -762,7 +762,8 @@ static int viommu_map(struct iommu_domain *domain, unsigned long iova,
> > }
> >
> > static size_t viommu_unmap(struct iommu_domain *domain, unsigned long iova,
> > - size_t size, struct iommu_iotlb_gather *gather)
> > + size_t size, struct iommu_iotlb_gather *gather,
> > + struct page **freelist)
> > {
> > int ret = 0;
> > size_t unmapped;
> > diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
> > index 5e556ac9102a..570ebf878fea 100644
> > --- a/drivers/vfio/vfio_iommu_type1.c
> > +++ b/drivers/vfio/vfio_iommu_type1.c
> > @@ -812,7 +812,7 @@ static size_t unmap_unpin_fast(struct vfio_domain *domain,
> >
> > if (entry) {
> > unmapped = iommu_unmap_fast(domain->domain, *iova, len,
> > - iotlb_gather);
> > + iotlb_gather, NULL);
> >
> > if (!unmapped) {
> > kfree(entry);
> > diff --git a/include/linux/iommu.h b/include/linux/iommu.h
> > index 5f0b7859d2eb..77e773d03f22 100644
> > --- a/include/linux/iommu.h
> > +++ b/include/linux/iommu.h
> > @@ -198,6 +198,7 @@ struct iommu_iotlb_gather {
> > * @map: map a physically contiguous memory region to an iommu domain
> > * @unmap: unmap a physically contiguous memory region from an iommu domain
> > * @flush_iotlb_all: Synchronously flush all hardware TLBs for this domain
> > + * @flush_iotlb_range: Flush given iova range of hardware TLBs for this domain
> > * @iotlb_sync_map: Sync mappings created recently using @map to the hardware
> > * @iotlb_sync: Flush all queued ranges from the hardware TLBs and empty flush
> > * queue
> > @@ -248,8 +249,12 @@ struct iommu_ops {
> > int (*map)(struct iommu_domain *domain, unsigned long iova,
> > phys_addr_t paddr, size_t size, int prot, gfp_t gfp);
> > size_t (*unmap)(struct iommu_domain *domain, unsigned long iova,
> > - size_t size, struct iommu_iotlb_gather *iotlb_gather);
> > + size_t size, struct iommu_iotlb_gather *iotlb_gather,
> > + struct page **freelist);
> > void (*flush_iotlb_all)(struct iommu_domain *domain);
> > + void (*flush_iotlb_range)(struct iommu_domain *domain,
> > + unsigned long iova, size_t size,
> > + struct page *freelist);
> > void (*iotlb_sync_map)(struct iommu_domain *domain);
> > void (*iotlb_sync)(struct iommu_domain *domain,
> > struct iommu_iotlb_gather *iotlb_gather);
> > @@ -447,7 +452,8 @@ extern size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova,
> > size_t size);
> > extern size_t iommu_unmap_fast(struct iommu_domain *domain,
> > unsigned long iova, size_t size,
> > - struct iommu_iotlb_gather *iotlb_gather);
> > + struct iommu_iotlb_gather *iotlb_gather,
> > + struct page **freelist);
> > extern size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
> > struct scatterlist *sg,unsigned int nents, int prot);
> > extern size_t iommu_map_sg_atomic(struct iommu_domain *domain,
> > @@ -542,6 +548,14 @@ static inline void iommu_flush_tlb_all(struct iommu_domain *domain)
> > domain->ops->flush_iotlb_all(domain);
> > }
> >
> > +static inline void flush_iotlb_range(struct iommu_domain *domain,
> > + unsigned long iova, size_t size,
> > + struct page *freelist)
> > +{
> > + if (domain->ops->flush_iotlb_range)
> > + domain->ops->flush_iotlb_range(domain, iova, size, freelist);
> > +}
> > +
> > static inline void iommu_tlb_sync(struct iommu_domain *domain,
> > struct iommu_iotlb_gather *iotlb_gather)
> > {
> > @@ -728,7 +742,8 @@ static inline size_t iommu_unmap(struct iommu_domain *domain,
> >
> > static inline size_t iommu_unmap_fast(struct iommu_domain *domain,
> > unsigned long iova, int gfp_order,
> > - struct iommu_iotlb_gather *iotlb_gather)
> > + struct iommu_iotlb_gather *iotlb_gather,
> > + struct page **freelist)
> > {
> > return 0;
> > }
> >