The framework allows callable implementation of IO page table.
This allows AMD IOMMU driver to switch between different types
of AMD IOMMU page tables (e.g. v1 vs. v2).
This series refactors the current implementation of AMD IOMMU v1 page table
to adopt the framework. There should be no functional change.
Subsequent series will introduce support for the AMD IOMMU v2 page table.
Thanks,
Suravee
Change from V1 (https://lkml.org/lkml/2020/9/23/251)
- Do not specify struct io_pgtable_cfg.coherent_walk, since it is
not currently used. (per Robin)
- Remove unused struct iommu_flush_ops. (patch 2/13)
- Move amd_iommu_setup_io_pgtable_ops to iommu.c instead of io_pgtable.c
patch 13/13)
Suravee Suthikulpanit (13):
iommu/amd: Re-define amd_iommu_domain_encode_pgtable as inline
iommu/amd: Prepare for generic IO page table framework
iommu/amd: Move pt_root to to struct amd_io_pgtable
iommu/amd: Convert to using amd_io_pgtable
iommu/amd: Declare functions as extern
iommu/amd: Move IO page table related functions
iommu/amd: Restructure code for freeing page table
iommu/amd: Remove amd_iommu_domain_get_pgtable
iommu/amd: Rename variables to be consistent with struct
io_pgtable_ops
iommu/amd: Refactor fetch_pte to use struct amd_io_pgtable
iommu/amd: Introduce iommu_v1_iova_to_phys
iommu/amd: Introduce iommu_v1_map_page and iommu_v1_unmap_page
iommu/amd: Adopt IO page table framework
drivers/iommu/amd/Kconfig | 1 +
drivers/iommu/amd/Makefile | 2 +-
drivers/iommu/amd/amd_iommu.h | 22 +
drivers/iommu/amd/amd_iommu_types.h | 40 +-
drivers/iommu/amd/io_pgtable.c | 534 +++++++++++++++++++++++
drivers/iommu/amd/iommu.c | 644 +++-------------------------
drivers/iommu/io-pgtable.c | 3 +
include/linux/io-pgtable.h | 2 +
8 files changed, 656 insertions(+), 592 deletions(-)
create mode 100644 drivers/iommu/amd/io_pgtable.c
--
2.17.1
Move the function to header file to allow inclusion in other files.
Signed-off-by: Suravee Suthikulpanit <[email protected]>
---
drivers/iommu/amd/amd_iommu.h | 13 +++++++++++++
drivers/iommu/amd/iommu.c | 10 ----------
2 files changed, 13 insertions(+), 10 deletions(-)
diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h
index 57309716fd18..97cdb235ce69 100644
--- a/drivers/iommu/amd/amd_iommu.h
+++ b/drivers/iommu/amd/amd_iommu.h
@@ -93,6 +93,19 @@ static inline void *iommu_phys_to_virt(unsigned long paddr)
return phys_to_virt(__sme_clr(paddr));
}
+static inline
+void amd_iommu_domain_set_pt_root(struct protection_domain *domain, u64 root)
+{
+ atomic64_set(&domain->pt_root, root);
+}
+
+static inline
+void amd_iommu_domain_clr_pt_root(struct protection_domain *domain)
+{
+ amd_iommu_domain_set_pt_root(domain, 0);
+}
+
+
extern bool translation_pre_enabled(struct amd_iommu *iommu);
extern bool amd_iommu_is_attach_deferred(struct iommu_domain *domain,
struct device *dev);
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index db4fb840c59c..e92b3f744292 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -162,16 +162,6 @@ static void amd_iommu_domain_get_pgtable(struct protection_domain *domain,
pgtable->mode = pt_root & 7; /* lowest 3 bits encode pgtable mode */
}
-static void amd_iommu_domain_set_pt_root(struct protection_domain *domain, u64 root)
-{
- atomic64_set(&domain->pt_root, root);
-}
-
-static void amd_iommu_domain_clr_pt_root(struct protection_domain *domain)
-{
- amd_iommu_domain_set_pt_root(domain, 0);
-}
-
static void amd_iommu_domain_set_pgtable(struct protection_domain *domain,
u64 *root, int mode)
{
--
2.17.1
And move declaration to header file so that they can be included across
multiple files. There is no functional change.
Signed-off-by: Suravee Suthikulpanit <[email protected]>
---
drivers/iommu/amd/amd_iommu.h | 3 +++
drivers/iommu/amd/iommu.c | 39 +++++++++++++++++------------------
2 files changed, 22 insertions(+), 20 deletions(-)
diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h
index 22ecacb71675..8b7be9171030 100644
--- a/drivers/iommu/amd/amd_iommu.h
+++ b/drivers/iommu/amd/amd_iommu.h
@@ -48,6 +48,9 @@ extern int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids);
extern int amd_iommu_flush_page(struct iommu_domain *dom, int pasid,
u64 address);
extern void amd_iommu_update_and_flush_device_table(struct protection_domain *domain);
+extern void amd_iommu_domain_update(struct protection_domain *domain);
+extern void amd_iommu_domain_flush_complete(struct protection_domain *domain);
+extern void amd_iommu_domain_flush_tlb_pde(struct protection_domain *domain);
extern int amd_iommu_flush_tlb(struct iommu_domain *dom, int pasid);
extern int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, int pasid,
unsigned long cr3);
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index 09da37c4c9c4..f91f35edb7ba 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -88,7 +88,6 @@ struct iommu_cmd {
struct kmem_cache *amd_iommu_irq_cache;
-static void update_domain(struct protection_domain *domain);
static void detach_device(struct device *dev);
/****************************************************************************
@@ -1294,12 +1293,12 @@ static void domain_flush_pages(struct protection_domain *domain,
}
/* Flush the whole IO/TLB for a given protection domain - including PDE */
-static void domain_flush_tlb_pde(struct protection_domain *domain)
+void amd_iommu_domain_flush_tlb_pde(struct protection_domain *domain)
{
__domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1);
}
-static void domain_flush_complete(struct protection_domain *domain)
+void amd_iommu_domain_flush_complete(struct protection_domain *domain)
{
int i;
@@ -1324,7 +1323,7 @@ static void domain_flush_np_cache(struct protection_domain *domain,
spin_lock_irqsave(&domain->lock, flags);
domain_flush_pages(domain, iova, size);
- domain_flush_complete(domain);
+ amd_iommu_domain_flush_complete(domain);
spin_unlock_irqrestore(&domain->lock, flags);
}
}
@@ -1481,7 +1480,7 @@ static bool increase_address_space(struct protection_domain *domain,
pgtable.root = pte;
pgtable.mode += 1;
amd_iommu_update_and_flush_device_table(domain);
- domain_flush_complete(domain);
+ amd_iommu_domain_flush_complete(domain);
/*
* Device Table needs to be updated and flushed before the new root can
@@ -1734,8 +1733,8 @@ static int iommu_map_page(struct protection_domain *dom,
* Updates and flushing already happened in
* increase_address_space().
*/
- domain_flush_tlb_pde(dom);
- domain_flush_complete(dom);
+ amd_iommu_domain_flush_tlb_pde(dom);
+ amd_iommu_domain_flush_complete(dom);
spin_unlock_irqrestore(&dom->lock, flags);
}
@@ -1978,10 +1977,10 @@ static void do_detach(struct iommu_dev_data *dev_data)
device_flush_dte(dev_data);
/* Flush IOTLB */
- domain_flush_tlb_pde(domain);
+ amd_iommu_domain_flush_tlb_pde(domain);
/* Wait for the flushes to finish */
- domain_flush_complete(domain);
+ amd_iommu_domain_flush_complete(domain);
/* decrease reference counters - needs to happen after the flushes */
domain->dev_iommu[iommu->index] -= 1;
@@ -2114,9 +2113,9 @@ static int attach_device(struct device *dev,
* left the caches in the IOMMU dirty. So we have to flush
* here to evict all dirty stuff.
*/
- domain_flush_tlb_pde(domain);
+ amd_iommu_domain_flush_tlb_pde(domain);
- domain_flush_complete(domain);
+ amd_iommu_domain_flush_complete(domain);
out:
spin_unlock(&dev_data->lock);
@@ -2277,7 +2276,7 @@ void amd_iommu_update_and_flush_device_table(struct protection_domain *domain)
domain_flush_devices(domain);
}
-static void update_domain(struct protection_domain *domain)
+void amd_iommu_domain_update(struct protection_domain *domain)
{
struct domain_pgtable pgtable;
@@ -2286,8 +2285,8 @@ static void update_domain(struct protection_domain *domain)
amd_iommu_update_and_flush_device_table(domain);
/* Flush domain TLB(s) and wait for completion */
- domain_flush_tlb_pde(domain);
- domain_flush_complete(domain);
+ amd_iommu_domain_flush_tlb_pde(domain);
+ amd_iommu_domain_flush_complete(domain);
}
int __init amd_iommu_init_api(void)
@@ -2675,8 +2674,8 @@ static void amd_iommu_flush_iotlb_all(struct iommu_domain *domain)
unsigned long flags;
spin_lock_irqsave(&dom->lock, flags);
- domain_flush_tlb_pde(dom);
- domain_flush_complete(dom);
+ amd_iommu_domain_flush_tlb_pde(dom);
+ amd_iommu_domain_flush_complete(dom);
spin_unlock_irqrestore(&dom->lock, flags);
}
@@ -2766,7 +2765,7 @@ void amd_iommu_domain_direct_map(struct iommu_domain *dom)
amd_iommu_domain_clr_pt_root(domain);
/* Make changes visible to IOMMUs */
- update_domain(domain);
+ amd_iommu_domain_update(domain);
/* Page-table is not visible to IOMMU anymore, so free it */
free_pagetable(&pgtable);
@@ -2810,7 +2809,7 @@ int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids)
domain->glx = levels;
domain->flags |= PD_IOMMUV2_MASK;
- update_domain(domain);
+ amd_iommu_domain_update(domain);
ret = 0;
@@ -2847,7 +2846,7 @@ static int __flush_pasid(struct protection_domain *domain, int pasid,
}
/* Wait until IOMMU TLB flushes are complete */
- domain_flush_complete(domain);
+ amd_iommu_domain_flush_complete(domain);
/* Now flush device TLBs */
list_for_each_entry(dev_data, &domain->dev_list, list) {
@@ -2873,7 +2872,7 @@ static int __flush_pasid(struct protection_domain *domain, int pasid,
}
/* Wait until all device TLBs are flushed */
- domain_flush_complete(domain);
+ amd_iommu_domain_flush_complete(domain);
ret = 0;
--
2.17.1
This implements iova_to_phys for AMD IOMMU v1 pagetable,
which will be used by the IO page table framework.
Signed-off-by: Suravee Suthikulpanit <[email protected]>
---
drivers/iommu/amd/io_pgtable.c | 21 +++++++++++++++++++++
drivers/iommu/amd/iommu.c | 16 +---------------
2 files changed, 22 insertions(+), 15 deletions(-)
diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c
index 1729c303bae5..bbbf18d2514a 100644
--- a/drivers/iommu/amd/io_pgtable.c
+++ b/drivers/iommu/amd/io_pgtable.c
@@ -494,6 +494,26 @@ unsigned long iommu_unmap_page(struct protection_domain *dom,
return unmapped;
}
+static phys_addr_t iommu_v1_iova_to_phys(struct io_pgtable_ops *ops, unsigned long iova)
+{
+ struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops);
+ unsigned long offset_mask, pte_pgsize;
+ u64 *pte, __pte;
+
+ if (pgtable->mode == PAGE_MODE_NONE)
+ return iova;
+
+ pte = fetch_pte(pgtable, iova, &pte_pgsize);
+
+ if (!pte || !IOMMU_PTE_PRESENT(*pte))
+ return 0;
+
+ offset_mask = pte_pgsize - 1;
+ __pte = __sme_clr(*pte & PM_ADDR_MASK);
+
+ return (__pte & ~offset_mask) | (iova & offset_mask);
+}
+
/*
* ----------------------------------------------------
*/
@@ -505,6 +525,7 @@ static struct io_pgtable *v1_alloc_pgtable(struct io_pgtable_cfg *cfg, void *coo
{
struct protection_domain *pdom = (struct protection_domain *)cookie;
+ pdom->iop.iop.ops.iova_to_phys = iommu_v1_iova_to_phys;
return &pdom->iop.iop;
}
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index 87cea1cde414..9a1a16031e00 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -2079,22 +2079,8 @@ static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
{
struct protection_domain *domain = to_pdomain(dom);
struct io_pgtable_ops *ops = &domain->iop.iop.ops;
- struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops);
- unsigned long offset_mask, pte_pgsize;
- u64 *pte, __pte;
- if (domain->iop.mode == PAGE_MODE_NONE)
- return iova;
-
- pte = fetch_pte(pgtable, iova, &pte_pgsize);
-
- if (!pte || !IOMMU_PTE_PRESENT(*pte))
- return 0;
-
- offset_mask = pte_pgsize - 1;
- __pte = __sme_clr(*pte & PM_ADDR_MASK);
-
- return (__pte & ~offset_mask) | (iova & offset_mask);
+ return ops->iova_to_phys(ops, iova);
}
static bool amd_iommu_capable(enum iommu_cap cap)
--
2.17.1
These implement map and unmap for AMD IOMMU v1 pagetable, which
will be used by the IO pagetable framework.
Also clean up unused extern function declarations.
Signed-off-by: Suravee Suthikulpanit <[email protected]>
---
drivers/iommu/amd/amd_iommu.h | 13 -------------
drivers/iommu/amd/io_pgtable.c | 25 ++++++++++++-------------
drivers/iommu/amd/iommu.c | 7 ++++---
3 files changed, 16 insertions(+), 29 deletions(-)
diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h
index 69996e57fae2..2e8dc2a1ec0f 100644
--- a/drivers/iommu/amd/amd_iommu.h
+++ b/drivers/iommu/amd/amd_iommu.h
@@ -124,19 +124,6 @@ void amd_iommu_apply_ivrs_quirks(void);
static inline void amd_iommu_apply_ivrs_quirks(void) { }
#endif
-/* TODO: These are temporary and will be removed once fully transition */
-extern int iommu_map_page(struct protection_domain *dom,
- unsigned long bus_addr,
- unsigned long phys_addr,
- unsigned long page_size,
- int prot,
- gfp_t gfp);
-extern unsigned long iommu_unmap_page(struct protection_domain *dom,
- unsigned long bus_addr,
- unsigned long page_size);
-extern u64 *fetch_pte(struct amd_io_pgtable *pgtable,
- unsigned long address,
- unsigned long *page_size);
extern void amd_iommu_domain_set_pgtable(struct protection_domain *domain,
u64 *root, int mode);
extern void amd_iommu_free_pgtable(struct amd_io_pgtable *pgtable);
diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c
index bbbf18d2514a..a5f8d80a9d35 100644
--- a/drivers/iommu/amd/io_pgtable.c
+++ b/drivers/iommu/amd/io_pgtable.c
@@ -317,9 +317,9 @@ static u64 *alloc_pte(struct protection_domain *domain,
* This function checks if there is a PTE for a given dma address. If
* there is one, it returns the pointer to it.
*/
-u64 *fetch_pte(struct amd_io_pgtable *pgtable,
- unsigned long address,
- unsigned long *page_size)
+static u64 *fetch_pte(struct amd_io_pgtable *pgtable,
+ unsigned long address,
+ unsigned long *page_size)
{
int level;
u64 *pte;
@@ -392,13 +392,10 @@ static struct page *free_clear_pte(u64 *pte, u64 pteval, struct page *freelist)
* supporting all features of AMD IOMMU page tables like level skipping
* and full 64 bit address spaces.
*/
-int iommu_map_page(struct protection_domain *dom,
- unsigned long iova,
- unsigned long paddr,
- unsigned long size,
- int prot,
- gfp_t gfp)
+static int iommu_v1_map_page(struct io_pgtable_ops *ops, unsigned long iova,
+ phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
{
+ struct protection_domain *dom = io_pgtable_ops_to_domain(ops);
struct page *freelist = NULL;
bool updated = false;
u64 __pte, *pte;
@@ -461,11 +458,11 @@ int iommu_map_page(struct protection_domain *dom,
return ret;
}
-unsigned long iommu_unmap_page(struct protection_domain *dom,
- unsigned long iova,
- unsigned long size)
+static unsigned long iommu_v1_unmap_page(struct io_pgtable_ops *ops,
+ unsigned long iova,
+ size_t size,
+ struct iommu_iotlb_gather *gather)
{
- struct io_pgtable_ops *ops = &dom->iop.iop.ops;
struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops);
unsigned long long unmapped;
unsigned long unmap_size;
@@ -525,6 +522,8 @@ static struct io_pgtable *v1_alloc_pgtable(struct io_pgtable_cfg *cfg, void *coo
{
struct protection_domain *pdom = (struct protection_domain *)cookie;
+ pdom->iop.iop.ops.map = iommu_v1_map_page;
+ pdom->iop.iop.ops.unmap = iommu_v1_unmap_page;
pdom->iop.iop.ops.iova_to_phys = iommu_v1_iova_to_phys;
return &pdom->iop.iop;
}
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index 9a1a16031e00..77f44b927ae7 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -2044,6 +2044,7 @@ static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
gfp_t gfp)
{
struct protection_domain *domain = to_pdomain(dom);
+ struct io_pgtable_ops *ops = &domain->iop.iop.ops;
int prot = 0;
int ret;
@@ -2055,8 +2056,7 @@ static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
if (iommu_prot & IOMMU_WRITE)
prot |= IOMMU_PROT_IW;
- ret = iommu_map_page(domain, iova, paddr, page_size, prot, gfp);
-
+ ret = ops->map(ops, iova, paddr, page_size, prot, gfp);
domain_flush_np_cache(domain, iova, page_size);
return ret;
@@ -2067,11 +2067,12 @@ static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
struct iommu_iotlb_gather *gather)
{
struct protection_domain *domain = to_pdomain(dom);
+ struct io_pgtable_ops *ops = &domain->iop.iop.ops;
if (domain->iop.mode == PAGE_MODE_NONE)
return 0;
- return iommu_unmap_page(domain, iova, page_size);
+ return ops->unmap(ops, iova, page_size, gather);
}
static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
--
2.17.1
I found an issue w/ this series. Please ignore. I'll send out V3.
Regards,
Suravee
On 10/2/20 7:28 PM, Suravee Suthikulpanit wrote:
> The framework allows callable implementation of IO page table.
> This allows AMD IOMMU driver to switch between different types
> of AMD IOMMU page tables (e.g. v1 vs. v2).
>
> This series refactors the current implementation of AMD IOMMU v1 page table
> to adopt the framework. There should be no functional change.
> Subsequent series will introduce support for the AMD IOMMU v2 page table.
>
> Thanks,
> Suravee
>
> Change from V1 (https://lkml.org/lkml/2020/9/23/251)
> - Do not specify struct io_pgtable_cfg.coherent_walk, since it is
> not currently used. (per Robin)
> - Remove unused struct iommu_flush_ops. (patch 2/13)
> - Move amd_iommu_setup_io_pgtable_ops to iommu.c instead of io_pgtable.c
> patch 13/13)
>
> Suravee Suthikulpanit (13):
> iommu/amd: Re-define amd_iommu_domain_encode_pgtable as inline
> iommu/amd: Prepare for generic IO page table framework
> iommu/amd: Move pt_root to to struct amd_io_pgtable
> iommu/amd: Convert to using amd_io_pgtable
> iommu/amd: Declare functions as extern
> iommu/amd: Move IO page table related functions
> iommu/amd: Restructure code for freeing page table
> iommu/amd: Remove amd_iommu_domain_get_pgtable
> iommu/amd: Rename variables to be consistent with struct
> io_pgtable_ops
> iommu/amd: Refactor fetch_pte to use struct amd_io_pgtable
> iommu/amd: Introduce iommu_v1_iova_to_phys
> iommu/amd: Introduce iommu_v1_map_page and iommu_v1_unmap_page
> iommu/amd: Adopt IO page table framework
>
> drivers/iommu/amd/Kconfig | 1 +
> drivers/iommu/amd/Makefile | 2 +-
> drivers/iommu/amd/amd_iommu.h | 22 +
> drivers/iommu/amd/amd_iommu_types.h | 40 +-
> drivers/iommu/amd/io_pgtable.c | 534 +++++++++++++++++++++++
> drivers/iommu/amd/iommu.c | 644 +++-------------------------
> drivers/iommu/io-pgtable.c | 3 +
> include/linux/io-pgtable.h | 2 +
> 8 files changed, 656 insertions(+), 592 deletions(-)
> create mode 100644 drivers/iommu/amd/io_pgtable.c
>