Hi all,
this series adds support for using the global coherent (aka uncached)
pool to the generic dma-direct code and then switches arm-nommu and
hexagon over to it, together with a bunch of cleanups.
Diffstat:
arch/arm/Kconfig | 5 -
arch/arm/mm/dma-mapping-nommu.c | 173 +---------------------------------------
arch/hexagon/Kconfig | 1
arch/hexagon/kernel/dma.c | 57 ++-----------
include/linux/dma-map-ops.h | 18 ++--
kernel/dma/Kconfig | 4
kernel/dma/coherent.c | 159 +++++++++++++++++-------------------
kernel/dma/direct.c | 15 +++
8 files changed, 124 insertions(+), 308 deletions(-)
Return the allocated dma_coherent_mem structure, set the
use_dma_pfn_offset and print the failure warning inside of
dma_init_coherent_memory instead of leaving that to the callers.
Signed-off-by: Christoph Hellwig <[email protected]>
Tested-by: Dillon Min <[email protected]>
---
kernel/dma/coherent.c | 78 ++++++++++++++++++-------------------------
1 file changed, 33 insertions(+), 45 deletions(-)
diff --git a/kernel/dma/coherent.c b/kernel/dma/coherent.c
index 67b126afac5a..ab397ebfd5ad 100644
--- a/kernel/dma/coherent.c
+++ b/kernel/dma/coherent.c
@@ -37,51 +37,44 @@ static inline dma_addr_t dma_get_device_base(struct device *dev,
return mem->device_base;
}
-static int dma_init_coherent_memory(phys_addr_t phys_addr,
- dma_addr_t device_addr, size_t size,
- struct dma_coherent_mem **mem)
+static struct dma_coherent_mem *dma_init_coherent_memory(phys_addr_t phys_addr,
+ dma_addr_t device_addr, size_t size, bool use_dma_pfn_offset)
{
- struct dma_coherent_mem *dma_mem = NULL;
- void *mem_base = NULL;
+ struct dma_coherent_mem *dma_mem;
int pages = size >> PAGE_SHIFT;
int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long);
- int ret;
+ void *mem_base;
- if (!size) {
- ret = -EINVAL;
- goto out;
- }
+ if (!size)
+ return ERR_PTR(-EINVAL);
mem_base = memremap(phys_addr, size, MEMREMAP_WC);
- if (!mem_base) {
- ret = -EINVAL;
- goto out;
- }
+ if (!mem_base)
+ return ERR_PTR(-EINVAL);
+
dma_mem = kzalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL);
- if (!dma_mem) {
- ret = -ENOMEM;
- goto out;
- }
+ if (!dma_mem)
+ goto out_unmap_membase;
dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
- if (!dma_mem->bitmap) {
- ret = -ENOMEM;
- goto out;
- }
+ if (!dma_mem->bitmap)
+ goto out_free_dma_mem;
dma_mem->virt_base = mem_base;
dma_mem->device_base = device_addr;
dma_mem->pfn_base = PFN_DOWN(phys_addr);
dma_mem->size = pages;
+ dma_mem->use_dev_dma_pfn_offset = use_dma_pfn_offset;
spin_lock_init(&dma_mem->spinlock);
- *mem = dma_mem;
- return 0;
+ return dma_mem;
-out:
+out_free_dma_mem:
kfree(dma_mem);
- if (mem_base)
- memunmap(mem_base);
- return ret;
+out_unmap_membase:
+ memunmap(mem_base);
+ pr_err("Reserved memory: failed to init DMA memory pool at %pa, size %zd MiB\n",
+ &phys_addr, size / SZ_1M);
+ return ERR_PTR(-ENOMEM);
}
static void dma_release_coherent_memory(struct dma_coherent_mem *mem)
@@ -130,9 +123,9 @@ int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
struct dma_coherent_mem *mem;
int ret;
- ret = dma_init_coherent_memory(phys_addr, device_addr, size, &mem);
- if (ret)
- return ret;
+ mem = dma_init_coherent_memory(phys_addr, device_addr, size, false);
+ if (IS_ERR(mem))
+ return PTR_ERR(mem);
ret = dma_assign_coherent_memory(dev, mem);
if (ret)
@@ -319,21 +312,16 @@ static struct reserved_mem *dma_reserved_default_memory __initdata;
static int rmem_dma_device_init(struct reserved_mem *rmem, struct device *dev)
{
- struct dma_coherent_mem *mem = rmem->priv;
- int ret;
-
- if (!mem) {
- ret = dma_init_coherent_memory(rmem->base, rmem->base,
- rmem->size, &mem);
- if (ret) {
- pr_err("Reserved memory: failed to init DMA memory pool at %pa, size %ld MiB\n",
- &rmem->base, (unsigned long)rmem->size / SZ_1M);
- return ret;
- }
+ if (!rmem->priv) {
+ struct dma_coherent_mem *mem;
+
+ mem = dma_init_coherent_memory(rmem->base, rmem->base,
+ rmem->size, true);
+ if (IS_ERR(mem))
+ return PTR_ERR(mem);
+ rmem->priv = mem;
}
- mem->use_dev_dma_pfn_offset = true;
- rmem->priv = mem;
- dma_assign_coherent_memory(dev, mem);
+ dma_assign_coherent_memory(dev, rmem->priv);
return 0;
}
--
2.30.2
Switch an ifdef so that the global coherent pool is initialized for
any architecture that selects the DMA_GLOBAL_POOL symbol insted of
hardcoding ARM.
Signed-off-by: Christoph Hellwig <[email protected]>
Tested-by: Dillon Min <[email protected]>
---
kernel/dma/coherent.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/kernel/dma/coherent.c b/kernel/dma/coherent.c
index 794e76b03b34..67b126afac5a 100644
--- a/kernel/dma/coherent.c
+++ b/kernel/dma/coherent.c
@@ -361,7 +361,9 @@ static int __init rmem_dma_setup(struct reserved_mem *rmem)
pr_err("Reserved memory: regions without no-map are not yet supported\n");
return -EINVAL;
}
+#endif
+#ifdef CONFIG_DMA_GLOBAL_POOL
if (of_get_flat_dt_prop(node, "linux,dma-default", NULL)) {
WARN(dma_reserved_default_memory,
"Reserved memory: region for default DMA coherent area is redefined\n");
--
2.30.2
Add a new helper to initialize the global coherent pool. This both
cleans up the existing initialization which indirects through the
reserved_mem_ops that are normally only used for struct device, and
also allows using the global pool for non-devicetree architectures.
Signed-off-by: Christoph Hellwig <[email protected]>
Tested-by: Dillon Min <[email protected]>
---
include/linux/dma-map-ops.h | 2 +-
kernel/dma/coherent.c | 32 ++++++++++++++------------------
2 files changed, 15 insertions(+), 19 deletions(-)
diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h
index 0d53a96a3d64..7e49bb86a0c0 100644
--- a/include/linux/dma-map-ops.h
+++ b/include/linux/dma-map-ops.h
@@ -176,7 +176,7 @@ void *dma_alloc_from_global_coherent(struct device *dev, ssize_t size,
int dma_release_from_global_coherent(int order, void *vaddr);
int dma_mmap_from_global_coherent(struct vm_area_struct *vma, void *cpu_addr,
size_t size, int *ret);
-
+int dma_init_global_coherent(phys_addr_t phys_addr, size_t size);
#else
static inline int dma_declare_coherent_memory(struct device *dev,
phys_addr_t phys_addr, dma_addr_t device_addr, size_t size)
diff --git a/kernel/dma/coherent.c b/kernel/dma/coherent.c
index ab397ebfd5ad..160d4e246ecb 100644
--- a/kernel/dma/coherent.c
+++ b/kernel/dma/coherent.c
@@ -300,6 +300,18 @@ int dma_mmap_from_global_coherent(struct vm_area_struct *vma, void *vaddr,
vaddr, size, ret);
}
+int dma_init_global_coherent(phys_addr_t phys_addr, size_t size)
+{
+ struct dma_coherent_mem *mem;
+
+ mem = dma_init_coherent_memory(phys_addr, phys_addr, size, true);
+ if (IS_ERR(mem))
+ return PTR_ERR(mem);
+ dma_coherent_default_memory = mem;
+ pr_info("DMA: default coherent area is set\n");
+ return 0;
+}
+
/*
* Support for reserved memory regions defined in device tree
*/
@@ -367,26 +379,10 @@ static int __init rmem_dma_setup(struct reserved_mem *rmem)
static int __init dma_init_reserved_memory(void)
{
- const struct reserved_mem_ops *ops;
- int ret;
-
if (!dma_reserved_default_memory)
return -ENOMEM;
-
- ops = dma_reserved_default_memory->ops;
-
- /*
- * We rely on rmem_dma_device_init() does not propagate error of
- * dma_assign_coherent_memory() for "NULL" device.
- */
- ret = ops->device_init(dma_reserved_default_memory, NULL);
-
- if (!ret) {
- dma_coherent_default_memory = dma_reserved_default_memory->priv;
- pr_info("DMA: default coherent area is set\n");
- }
-
- return ret;
+ return dma_init_global_coherent(dma_reserved_default_memory->base,
+ dma_reserved_default_memory->size);
}
core_initcall(dma_init_reserved_memory);
--
2.30.2
Switch hexagon to use the generic code for dma_alloc_coherent from
a global pre-filled pool.
Signed-off-by: Christoph Hellwig <[email protected]>
---
arch/hexagon/Kconfig | 1 +
arch/hexagon/kernel/dma.c | 57 ++++++++-------------------------------
2 files changed, 12 insertions(+), 46 deletions(-)
diff --git a/arch/hexagon/Kconfig b/arch/hexagon/Kconfig
index e5a852080730..aab1a40eb653 100644
--- a/arch/hexagon/Kconfig
+++ b/arch/hexagon/Kconfig
@@ -7,6 +7,7 @@ config HEXAGON
select ARCH_32BIT_OFF_T
select ARCH_HAS_SYNC_DMA_FOR_DEVICE
select ARCH_NO_PREEMPT
+ select DMA_GLOBAL_POOL
# Other pending projects/to-do items.
# select HAVE_REGS_AND_STACK_ACCESS_API
# select HAVE_HW_BREAKPOINT if PERF_EVENTS
diff --git a/arch/hexagon/kernel/dma.c b/arch/hexagon/kernel/dma.c
index 00b9a81075dd..882680e81a30 100644
--- a/arch/hexagon/kernel/dma.c
+++ b/arch/hexagon/kernel/dma.c
@@ -7,54 +7,8 @@
#include <linux/dma-map-ops.h>
#include <linux/memblock.h>
-#include <linux/genalloc.h>
-#include <linux/module.h>
#include <asm/page.h>
-static struct gen_pool *coherent_pool;
-
-
-/* Allocates from a pool of uncached memory that was reserved at boot time */
-
-void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_addr,
- gfp_t flag, unsigned long attrs)
-{
- void *ret;
-
- /*
- * Our max_low_pfn should have been backed off by 16MB in
- * mm/init.c to create DMA coherent space. Use that as the VA
- * for the pool.
- */
-
- if (coherent_pool == NULL) {
- coherent_pool = gen_pool_create(PAGE_SHIFT, -1);
-
- if (coherent_pool == NULL)
- panic("Can't create %s() memory pool!", __func__);
- else
- gen_pool_add(coherent_pool,
- (unsigned long)pfn_to_virt(max_low_pfn),
- hexagon_coherent_pool_size, -1);
- }
-
- ret = (void *) gen_pool_alloc(coherent_pool, size);
-
- if (ret) {
- memset(ret, 0, size);
- *dma_addr = (dma_addr_t) virt_to_phys(ret);
- } else
- *dma_addr = ~0;
-
- return ret;
-}
-
-void arch_dma_free(struct device *dev, size_t size, void *vaddr,
- dma_addr_t dma_addr, unsigned long attrs)
-{
- gen_pool_free(coherent_pool, (unsigned long) vaddr, size);
-}
-
void arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
enum dma_data_direction dir)
{
@@ -77,3 +31,14 @@ void arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
BUG();
}
}
+
+/*
+ * Our max_low_pfn should have been backed off by 16MB in mm/init.c to create
+ * DMA coherent space. Use that for the pool.
+ */
+static int __init hexagon_dma_init(void)
+{
+ return dma_init_global_coherent(PFN_PHYS(max_low_pfn),
+ hexagon_coherent_pool_size);
+}
+core_initcall(hexagon_dma_init);
--
2.30.2
Add an option to allocate uncached memory for dma_alloc_coherent from
the global dma_coherent_default_memory. This will allow to move
arm-nommu (and eventually other platforms) to use generic code for
allocating uncached memory from a pre-populated pool.
Note that this is a different pool from the one that platforms that
can remap at runtime use for GFP_ATOMIC allocations for now, although
there might be opportunities to eventually end up with a common codebase
for the two use cases.
Signed-off-by: Christoph Hellwig <[email protected]>
Tested-by: Dillon Min <[email protected]>
---
kernel/dma/Kconfig | 4 ++++
kernel/dma/direct.c | 15 +++++++++++++++
2 files changed, 19 insertions(+)
diff --git a/kernel/dma/Kconfig b/kernel/dma/Kconfig
index 77b405508743..725cfd51762b 100644
--- a/kernel/dma/Kconfig
+++ b/kernel/dma/Kconfig
@@ -93,6 +93,10 @@ config DMA_COHERENT_POOL
select GENERIC_ALLOCATOR
bool
+config DMA_GLOBAL_POOL
+ select DMA_DECLARE_COHERENT
+ bool
+
config DMA_REMAP
bool
depends on MMU
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index f737e3347059..d1d0258ed6d0 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -156,9 +156,14 @@ void *dma_direct_alloc(struct device *dev, size_t size,
if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_SET_UNCACHED) &&
!IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
+ !IS_ENABLED(CONFIG_DMA_GLOBAL_POOL) &&
!dev_is_dma_coherent(dev))
return arch_dma_alloc(dev, size, dma_handle, gfp, attrs);
+ if (IS_ENABLED(CONFIG_DMA_GLOBAL_POOL) &&
+ !dev_is_dma_coherent(dev))
+ return dma_alloc_from_global_coherent(dev, size, dma_handle);
+
/*
* Remapping or decrypting memory may block. If either is required and
* we can't block, allocate the memory from the atomic pools.
@@ -255,11 +260,19 @@ void dma_direct_free(struct device *dev, size_t size,
if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_SET_UNCACHED) &&
!IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
+ !IS_ENABLED(CONFIG_DMA_GLOBAL_POOL) &&
!dev_is_dma_coherent(dev)) {
arch_dma_free(dev, size, cpu_addr, dma_addr, attrs);
return;
}
+ if (IS_ENABLED(CONFIG_DMA_GLOBAL_POOL) &&
+ !dev_is_dma_coherent(dev)) {
+ if (!dma_release_from_global_coherent(page_order, cpu_addr))
+ WARN_ON_ONCE(1);
+ return;
+ }
+
/* If cpu_addr is not from an atomic pool, dma_free_from_pool() fails */
if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
dma_free_from_pool(dev, cpu_addr, PAGE_ALIGN(size)))
@@ -462,6 +475,8 @@ int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma,
if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
return ret;
+ if (dma_mmap_from_global_coherent(vma, cpu_addr, size, &ret))
+ return ret;
if (vma->vm_pgoff >= count || user_count > count - vma->vm_pgoff)
return -ENXIO;
--
2.30.2
Only build the code to support the global coherent pool if support for
it is enabled.
Signed-off-by: Christoph Hellwig <[email protected]>
Tested-by: Dillon Min <[email protected]>
---
include/linux/dma-map-ops.h | 18 +++++++-------
kernel/dma/coherent.c | 47 ++++++++++++++++++++-----------------
2 files changed, 35 insertions(+), 30 deletions(-)
diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h
index 7e49bb86a0c0..9b79aa4dd300 100644
--- a/include/linux/dma-map-ops.h
+++ b/include/linux/dma-map-ops.h
@@ -170,13 +170,6 @@ int dma_alloc_from_dev_coherent(struct device *dev, ssize_t size,
int dma_release_from_dev_coherent(struct device *dev, int order, void *vaddr);
int dma_mmap_from_dev_coherent(struct device *dev, struct vm_area_struct *vma,
void *cpu_addr, size_t size, int *ret);
-
-void *dma_alloc_from_global_coherent(struct device *dev, ssize_t size,
- dma_addr_t *dma_handle);
-int dma_release_from_global_coherent(int order, void *vaddr);
-int dma_mmap_from_global_coherent(struct vm_area_struct *vma, void *cpu_addr,
- size_t size, int *ret);
-int dma_init_global_coherent(phys_addr_t phys_addr, size_t size);
#else
static inline int dma_declare_coherent_memory(struct device *dev,
phys_addr_t phys_addr, dma_addr_t device_addr, size_t size)
@@ -186,7 +179,16 @@ static inline int dma_declare_coherent_memory(struct device *dev,
#define dma_alloc_from_dev_coherent(dev, size, handle, ret) (0)
#define dma_release_from_dev_coherent(dev, order, vaddr) (0)
#define dma_mmap_from_dev_coherent(dev, vma, vaddr, order, ret) (0)
+#endif /* CONFIG_DMA_DECLARE_COHERENT */
+#ifdef CONFIG_DMA_GLOBAL_POOL
+void *dma_alloc_from_global_coherent(struct device *dev, ssize_t size,
+ dma_addr_t *dma_handle);
+int dma_release_from_global_coherent(int order, void *vaddr);
+int dma_mmap_from_global_coherent(struct vm_area_struct *vma, void *cpu_addr,
+ size_t size, int *ret);
+int dma_init_global_coherent(phys_addr_t phys_addr, size_t size);
+#else
static inline void *dma_alloc_from_global_coherent(struct device *dev,
ssize_t size, dma_addr_t *dma_handle)
{
@@ -201,7 +203,7 @@ static inline int dma_mmap_from_global_coherent(struct vm_area_struct *vma,
{
return 0;
}
-#endif /* CONFIG_DMA_DECLARE_COHERENT */
+#endif /* CONFIG_DMA_GLOBAL_POOL */
/*
* This is the actual return value from the ->alloc_noncontiguous method.
diff --git a/kernel/dma/coherent.c b/kernel/dma/coherent.c
index 160d4e246ecb..c05408902a68 100644
--- a/kernel/dma/coherent.c
+++ b/kernel/dma/coherent.c
@@ -20,8 +20,6 @@ struct dma_coherent_mem {
bool use_dev_dma_pfn_offset;
};
-static struct dma_coherent_mem *dma_coherent_default_memory __ro_after_init;
-
static inline struct dma_coherent_mem *dev_get_coherent_memory(struct device *dev)
{
if (dev && dev->dma_mem)
@@ -191,16 +189,6 @@ int dma_alloc_from_dev_coherent(struct device *dev, ssize_t size,
return 1;
}
-void *dma_alloc_from_global_coherent(struct device *dev, ssize_t size,
- dma_addr_t *dma_handle)
-{
- if (!dma_coherent_default_memory)
- return NULL;
-
- return __dma_alloc_from_coherent(dev, dma_coherent_default_memory, size,
- dma_handle);
-}
-
static int __dma_release_from_coherent(struct dma_coherent_mem *mem,
int order, void *vaddr)
{
@@ -236,15 +224,6 @@ int dma_release_from_dev_coherent(struct device *dev, int order, void *vaddr)
return __dma_release_from_coherent(mem, order, vaddr);
}
-int dma_release_from_global_coherent(int order, void *vaddr)
-{
- if (!dma_coherent_default_memory)
- return 0;
-
- return __dma_release_from_coherent(dma_coherent_default_memory, order,
- vaddr);
-}
-
static int __dma_mmap_from_coherent(struct dma_coherent_mem *mem,
struct vm_area_struct *vma, void *vaddr, size_t size, int *ret)
{
@@ -290,6 +269,28 @@ int dma_mmap_from_dev_coherent(struct device *dev, struct vm_area_struct *vma,
return __dma_mmap_from_coherent(mem, vma, vaddr, size, ret);
}
+#ifdef CONFIG_DMA_GLOBAL_POOL
+static struct dma_coherent_mem *dma_coherent_default_memory __ro_after_init;
+
+void *dma_alloc_from_global_coherent(struct device *dev, ssize_t size,
+ dma_addr_t *dma_handle)
+{
+ if (!dma_coherent_default_memory)
+ return NULL;
+
+ return __dma_alloc_from_coherent(dev, dma_coherent_default_memory, size,
+ dma_handle);
+}
+
+int dma_release_from_global_coherent(int order, void *vaddr)
+{
+ if (!dma_coherent_default_memory)
+ return 0;
+
+ return __dma_release_from_coherent(dma_coherent_default_memory, order,
+ vaddr);
+}
+
int dma_mmap_from_global_coherent(struct vm_area_struct *vma, void *vaddr,
size_t size, int *ret)
{
@@ -311,6 +312,7 @@ int dma_init_global_coherent(phys_addr_t phys_addr, size_t size)
pr_info("DMA: default coherent area is set\n");
return 0;
}
+#endif /* CONFIG_DMA_GLOBAL_POOL */
/*
* Support for reserved memory regions defined in device tree
@@ -377,6 +379,7 @@ static int __init rmem_dma_setup(struct reserved_mem *rmem)
return 0;
}
+#ifdef CONFIG_DMA_GLOBAL_POOL
static int __init dma_init_reserved_memory(void)
{
if (!dma_reserved_default_memory)
@@ -384,8 +387,8 @@ static int __init dma_init_reserved_memory(void)
return dma_init_global_coherent(dma_reserved_default_memory->base,
dma_reserved_default_memory->size);
}
-
core_initcall(dma_init_reserved_memory);
+#endif /* CONFIG_DMA_GLOBAL_POOL */
RESERVEDMEM_OF_DECLARE(dma, "shared-dma-pool", rmem_dma_setup);
#endif
--
2.30.2
Select the right options to just use the generic dma-direct code
instead of reimplementing it.
Signed-off-by: Christoph Hellwig <[email protected]>
Tested-by: Dillon Min <[email protected]>
---
arch/arm/Kconfig | 5 +-
arch/arm/mm/dma-mapping-nommu.c | 173 ++------------------------------
2 files changed, 9 insertions(+), 169 deletions(-)
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index 3ea1c417339f..fcb192bfd35a 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -18,8 +18,8 @@ config ARM
select ARCH_HAS_SET_MEMORY
select ARCH_HAS_STRICT_KERNEL_RWX if MMU && !XIP_KERNEL
select ARCH_HAS_STRICT_MODULE_RWX if MMU
- select ARCH_HAS_SYNC_DMA_FOR_DEVICE if SWIOTLB
- select ARCH_HAS_SYNC_DMA_FOR_CPU if SWIOTLB
+ select ARCH_HAS_SYNC_DMA_FOR_DEVICE if SWIOTLB || !MMU
+ select ARCH_HAS_SYNC_DMA_FOR_CPU if SWIOTLB || !MMU
select ARCH_HAS_TEARDOWN_DMA_OPS if MMU
select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
select ARCH_HAVE_CUSTOM_GPIO_H
@@ -44,6 +44,7 @@ config ARM
select CPU_PM if SUSPEND || CPU_IDLE
select DCACHE_WORD_ACCESS if HAVE_EFFICIENT_UNALIGNED_ACCESS
select DMA_DECLARE_COHERENT
+ select DMA_GLOBAL_POOL if !MMU
select DMA_OPS
select DMA_REMAP if MMU
select EDAC_SUPPORT
diff --git a/arch/arm/mm/dma-mapping-nommu.c b/arch/arm/mm/dma-mapping-nommu.c
index 6bfd2b884505..cfd9c933d2f0 100644
--- a/arch/arm/mm/dma-mapping-nommu.c
+++ b/arch/arm/mm/dma-mapping-nommu.c
@@ -5,12 +5,7 @@
* Copyright (C) 2000-2004 Russell King
*/
-#include <linux/export.h>
-#include <linux/mm.h>
-#include <linux/dma-direct.h>
#include <linux/dma-map-ops.h>
-#include <linux/scatterlist.h>
-
#include <asm/cachetype.h>
#include <asm/cacheflush.h>
#include <asm/outercache.h>
@@ -18,65 +13,8 @@
#include "dma.h"
-/*
- * The generic direct mapping code is used if
- * - MMU/MPU is off
- * - cpu is v7m w/o cache support
- * - device is coherent
- * otherwise arm_nommu_dma_ops is used.
- *
- * arm_nommu_dma_ops rely on consistent DMA memory (please, refer to
- * [1] on how to declare such memory).
- *
- * [1] Documentation/devicetree/bindings/reserved-memory/reserved-memory.txt
- */
-
-static void *arm_nommu_dma_alloc(struct device *dev, size_t size,
- dma_addr_t *dma_handle, gfp_t gfp,
- unsigned long attrs)
-
-{
- void *ret = dma_alloc_from_global_coherent(dev, size, dma_handle);
-
- /*
- * dma_alloc_from_global_coherent() may fail because:
- *
- * - no consistent DMA region has been defined, so we can't
- * continue.
- * - there is no space left in consistent DMA region, so we
- * only can fallback to generic allocator if we are
- * advertised that consistency is not required.
- */
-
- WARN_ON_ONCE(ret == NULL);
- return ret;
-}
-
-static void arm_nommu_dma_free(struct device *dev, size_t size,
- void *cpu_addr, dma_addr_t dma_addr,
- unsigned long attrs)
-{
- int ret = dma_release_from_global_coherent(get_order(size), cpu_addr);
-
- WARN_ON_ONCE(ret == 0);
-}
-
-static int arm_nommu_dma_mmap(struct device *dev, struct vm_area_struct *vma,
- void *cpu_addr, dma_addr_t dma_addr, size_t size,
- unsigned long attrs)
-{
- int ret;
-
- if (dma_mmap_from_global_coherent(vma, cpu_addr, size, &ret))
- return ret;
- if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
- return ret;
- return -ENXIO;
-}
-
-
-static void __dma_page_cpu_to_dev(phys_addr_t paddr, size_t size,
- enum dma_data_direction dir)
+void arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
+ enum dma_data_direction dir)
{
dmac_map_area(__va(paddr), size, dir);
@@ -86,8 +24,8 @@ static void __dma_page_cpu_to_dev(phys_addr_t paddr, size_t size,
outer_clean_range(paddr, paddr + size);
}
-static void __dma_page_dev_to_cpu(phys_addr_t paddr, size_t size,
- enum dma_data_direction dir)
+void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size,
+ enum dma_data_direction dir)
{
if (dir != DMA_TO_DEVICE) {
outer_inv_range(paddr, paddr + size);
@@ -95,102 +33,6 @@ static void __dma_page_dev_to_cpu(phys_addr_t paddr, size_t size,
}
}
-static dma_addr_t arm_nommu_dma_map_page(struct device *dev, struct page *page,
- unsigned long offset, size_t size,
- enum dma_data_direction dir,
- unsigned long attrs)
-{
- dma_addr_t handle = page_to_phys(page) + offset;
-
- __dma_page_cpu_to_dev(handle, size, dir);
-
- return handle;
-}
-
-static void arm_nommu_dma_unmap_page(struct device *dev, dma_addr_t handle,
- size_t size, enum dma_data_direction dir,
- unsigned long attrs)
-{
- __dma_page_dev_to_cpu(handle, size, dir);
-}
-
-
-static int arm_nommu_dma_map_sg(struct device *dev, struct scatterlist *sgl,
- int nents, enum dma_data_direction dir,
- unsigned long attrs)
-{
- int i;
- struct scatterlist *sg;
-
- for_each_sg(sgl, sg, nents, i) {
- sg_dma_address(sg) = sg_phys(sg);
- sg_dma_len(sg) = sg->length;
- __dma_page_cpu_to_dev(sg_dma_address(sg), sg_dma_len(sg), dir);
- }
-
- return nents;
-}
-
-static void arm_nommu_dma_unmap_sg(struct device *dev, struct scatterlist *sgl,
- int nents, enum dma_data_direction dir,
- unsigned long attrs)
-{
- struct scatterlist *sg;
- int i;
-
- for_each_sg(sgl, sg, nents, i)
- __dma_page_dev_to_cpu(sg_dma_address(sg), sg_dma_len(sg), dir);
-}
-
-static void arm_nommu_dma_sync_single_for_device(struct device *dev,
- dma_addr_t handle, size_t size, enum dma_data_direction dir)
-{
- __dma_page_cpu_to_dev(handle, size, dir);
-}
-
-static void arm_nommu_dma_sync_single_for_cpu(struct device *dev,
- dma_addr_t handle, size_t size, enum dma_data_direction dir)
-{
- __dma_page_cpu_to_dev(handle, size, dir);
-}
-
-static void arm_nommu_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sgl,
- int nents, enum dma_data_direction dir)
-{
- struct scatterlist *sg;
- int i;
-
- for_each_sg(sgl, sg, nents, i)
- __dma_page_cpu_to_dev(sg_dma_address(sg), sg_dma_len(sg), dir);
-}
-
-static void arm_nommu_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sgl,
- int nents, enum dma_data_direction dir)
-{
- struct scatterlist *sg;
- int i;
-
- for_each_sg(sgl, sg, nents, i)
- __dma_page_dev_to_cpu(sg_dma_address(sg), sg_dma_len(sg), dir);
-}
-
-const struct dma_map_ops arm_nommu_dma_ops = {
- .alloc = arm_nommu_dma_alloc,
- .free = arm_nommu_dma_free,
- .alloc_pages = dma_direct_alloc_pages,
- .free_pages = dma_direct_free_pages,
- .mmap = arm_nommu_dma_mmap,
- .map_page = arm_nommu_dma_map_page,
- .unmap_page = arm_nommu_dma_unmap_page,
- .map_sg = arm_nommu_dma_map_sg,
- .unmap_sg = arm_nommu_dma_unmap_sg,
- .sync_single_for_device = arm_nommu_dma_sync_single_for_device,
- .sync_single_for_cpu = arm_nommu_dma_sync_single_for_cpu,
- .sync_sg_for_device = arm_nommu_dma_sync_sg_for_device,
- .sync_sg_for_cpu = arm_nommu_dma_sync_sg_for_cpu,
-};
-EXPORT_SYMBOL(arm_nommu_dma_ops);
-
void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
const struct iommu_ops *iommu, bool coherent)
{
@@ -201,14 +43,11 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
* enough to check if MPU is in use or not since in absense of
* MPU system memory map is used.
*/
- dev->archdata.dma_coherent = (cacheid) ? coherent : true;
+ dev->dma_coherent = cacheid ? coherent : true;
} else {
/*
* Assume coherent DMA in case MMU/MPU has not been set up.
*/
- dev->archdata.dma_coherent = (get_cr() & CR_M) ? coherent : true;
+ dev->dma_coherent = (get_cr() & CR_M) ? coherent : true;
}
-
- if (!dev->archdata.dma_coherent)
- set_dma_ops(dev, &arm_nommu_dma_ops);
}
--
2.30.2
On Sun, Jul 11, 2021 at 11:25 PM Christoph Hellwig <[email protected]> wrote:
>
> Only build the code to support the global coherent pool if support for
> it is enabled.
>
> Signed-off-by: Christoph Hellwig <[email protected]>
> Tested-by: Dillon Min <[email protected]>
> ---
> include/linux/dma-map-ops.h | 18 +++++++-------
> kernel/dma/coherent.c | 47 ++++++++++++++++++++-----------------
> 2 files changed, 35 insertions(+), 30 deletions(-)
>
> diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h
> index 7e49bb86a0c0..9b79aa4dd300 100644
> --- a/include/linux/dma-map-ops.h
> +++ b/include/linux/dma-map-ops.h
> @@ -170,13 +170,6 @@ int dma_alloc_from_dev_coherent(struct device *dev, ssize_t size,
> int dma_release_from_dev_coherent(struct device *dev, int order, void *vaddr);
> int dma_mmap_from_dev_coherent(struct device *dev, struct vm_area_struct *vma,
> void *cpu_addr, size_t size, int *ret);
> -
> -void *dma_alloc_from_global_coherent(struct device *dev, ssize_t size,
> - dma_addr_t *dma_handle);
> -int dma_release_from_global_coherent(int order, void *vaddr);
> -int dma_mmap_from_global_coherent(struct vm_area_struct *vma, void *cpu_addr,
> - size_t size, int *ret);
> -int dma_init_global_coherent(phys_addr_t phys_addr, size_t size);
> #else
> static inline int dma_declare_coherent_memory(struct device *dev,
> phys_addr_t phys_addr, dma_addr_t device_addr, size_t size)
> @@ -186,7 +179,16 @@ static inline int dma_declare_coherent_memory(struct device *dev,
> #define dma_alloc_from_dev_coherent(dev, size, handle, ret) (0)
> #define dma_release_from_dev_coherent(dev, order, vaddr) (0)
> #define dma_mmap_from_dev_coherent(dev, vma, vaddr, order, ret) (0)
> +#endif /* CONFIG_DMA_DECLARE_COHERENT */
>
> +#ifdef CONFIG_DMA_GLOBAL_POOL
> +void *dma_alloc_from_global_coherent(struct device *dev, ssize_t size,
> + dma_addr_t *dma_handle);
> +int dma_release_from_global_coherent(int order, void *vaddr);
> +int dma_mmap_from_global_coherent(struct vm_area_struct *vma, void *cpu_addr,
> + size_t size, int *ret);
> +int dma_init_global_coherent(phys_addr_t phys_addr, size_t size);
> +#else
> static inline void *dma_alloc_from_global_coherent(struct device *dev,
> ssize_t size, dma_addr_t *dma_handle)
> {
> @@ -201,7 +203,7 @@ static inline int dma_mmap_from_global_coherent(struct vm_area_struct *vma,
> {
> return 0;
> }
> -#endif /* CONFIG_DMA_DECLARE_COHERENT */
> +#endif /* CONFIG_DMA_GLOBAL_POOL */
>
> /*
> * This is the actual return value from the ->alloc_noncontiguous method.
> diff --git a/kernel/dma/coherent.c b/kernel/dma/coherent.c
> index 160d4e246ecb..c05408902a68 100644
> --- a/kernel/dma/coherent.c
> +++ b/kernel/dma/coherent.c
> @@ -20,8 +20,6 @@ struct dma_coherent_mem {
> bool use_dev_dma_pfn_offset;
> };
>
> -static struct dma_coherent_mem *dma_coherent_default_memory __ro_after_init;
> -
> static inline struct dma_coherent_mem *dev_get_coherent_memory(struct device *dev)
> {
> if (dev && dev->dma_mem)
> @@ -191,16 +189,6 @@ int dma_alloc_from_dev_coherent(struct device *dev, ssize_t size,
> return 1;
> }
>
> -void *dma_alloc_from_global_coherent(struct device *dev, ssize_t size,
> - dma_addr_t *dma_handle)
> -{
> - if (!dma_coherent_default_memory)
> - return NULL;
> -
> - return __dma_alloc_from_coherent(dev, dma_coherent_default_memory, size,
> - dma_handle);
> -}
> -
> static int __dma_release_from_coherent(struct dma_coherent_mem *mem,
> int order, void *vaddr)
> {
> @@ -236,15 +224,6 @@ int dma_release_from_dev_coherent(struct device *dev, int order, void *vaddr)
> return __dma_release_from_coherent(mem, order, vaddr);
> }
>
> -int dma_release_from_global_coherent(int order, void *vaddr)
> -{
> - if (!dma_coherent_default_memory)
> - return 0;
> -
> - return __dma_release_from_coherent(dma_coherent_default_memory, order,
> - vaddr);
> -}
> -
> static int __dma_mmap_from_coherent(struct dma_coherent_mem *mem,
> struct vm_area_struct *vma, void *vaddr, size_t size, int *ret)
> {
> @@ -290,6 +269,28 @@ int dma_mmap_from_dev_coherent(struct device *dev, struct vm_area_struct *vma,
> return __dma_mmap_from_coherent(mem, vma, vaddr, size, ret);
> }
>
> +#ifdef CONFIG_DMA_GLOBAL_POOL
> +static struct dma_coherent_mem *dma_coherent_default_memory __ro_after_init;
> +
> +void *dma_alloc_from_global_coherent(struct device *dev, ssize_t size,
> + dma_addr_t *dma_handle)
> +{
> + if (!dma_coherent_default_memory)
> + return NULL;
> +
> + return __dma_alloc_from_coherent(dev, dma_coherent_default_memory, size,
> + dma_handle);
> +}
> +
> +int dma_release_from_global_coherent(int order, void *vaddr)
> +{
> + if (!dma_coherent_default_memory)
> + return 0;
> +
> + return __dma_release_from_coherent(dma_coherent_default_memory, order,
> + vaddr);
> +}
> +
> int dma_mmap_from_global_coherent(struct vm_area_struct *vma, void *vaddr,
> size_t size, int *ret)
> {
> @@ -311,6 +312,7 @@ int dma_init_global_coherent(phys_addr_t phys_addr, size_t size)
> pr_info("DMA: default coherent area is set\n");
> return 0;
> }
> +#endif /* CONFIG_DMA_GLOBAL_POOL */
>
> /*
> * Support for reserved memory regions defined in device tree
> @@ -377,6 +379,7 @@ static int __init rmem_dma_setup(struct reserved_mem *rmem)
> return 0;
> }
>
> +#ifdef CONFIG_DMA_GLOBAL_POOL
> static int __init dma_init_reserved_memory(void)
> {
> if (!dma_reserved_default_memory)
> @@ -384,8 +387,8 @@ static int __init dma_init_reserved_memory(void)
> return dma_init_global_coherent(dma_reserved_default_memory->base,
> dma_reserved_default_memory->size);
dma_reserved_default_memory also needs to be moved inside
CONFIG_DMA_GLOBAL_POOL because
all the usages are already part of that config.
> }
> -
> core_initcall(dma_init_reserved_memory);
> +#endif /* CONFIG_DMA_GLOBAL_POOL */
>
> RESERVEDMEM_OF_DECLARE(dma, "shared-dma-pool", rmem_dma_setup);
> #endif
> --
> 2.30.2
>
> _______________________________________________
> iommu mailing list
> [email protected]
> https://lists.linuxfoundation.org/mailman/listinfo/iommu
--
Regards,
Atish
> -----Original Message-----
> From: Christoph Hellwig <[email protected]>
> Sent: Monday, July 12, 2021 1:17 AM
> To: [email protected]; Russell King
<[email protected]>;
> Brian Cain <[email protected]>
> Cc: Dillon Min <[email protected]>; Vladimir Murzin
> <[email protected]>; [email protected]; linux-
> [email protected]; [email protected]
> Subject: add support for the global coherent pool to the dma core
>
> Hi all,
>
> this series adds support for using the global coherent (aka uncached)
> pool to the generic dma-direct code and then switches arm-nommu and
> hexagon over to it, together with a bunch of cleanups.
Christoph,
Thanks for sending this -- I will take a look and give some feedback soon.
> Diffstat:
> arch/arm/Kconfig | 5 -
> arch/arm/mm/dma-mapping-nommu.c | 173
+--------------------------------------
> -
> arch/hexagon/Kconfig | 1
> arch/hexagon/kernel/dma.c | 57 ++-----------
> include/linux/dma-map-ops.h | 18 ++--
> kernel/dma/Kconfig | 4
> kernel/dma/coherent.c | 159
+++++++++++++++++-------------------
> kernel/dma/direct.c | 15 +++
> 8 files changed, 124 insertions(+), 308 deletions(-)
On Wed, Jul 28, 2021 at 10:20:34AM -0500, Brian Cain wrote:
> > this series adds support for using the global coherent (aka uncached)
> > pool to the generic dma-direct code and then switches arm-nommu and
> > hexagon over to it, together with a bunch of cleanups.
>
> Christoph,
>
> Thanks for sending this -- I will take a look and give some feedback soon.
Any chance you'll get to this in the next days? I'd love to merge this
series for the current merge window with a little time in linux-next.
> -----Original Message-----
> From: Christoph Hellwig <[email protected]>
...
> Switch hexagon to use the generic code for dma_alloc_coherent from
> a global pre-filled pool.
>
> Signed-off-by: Christoph Hellwig <[email protected]>
Reviewed-by: Brian Cain <[email protected]>
Any more comments?
Otherwise I'm going to pull this into the dma-mapping tree next week.
Thanks,
I've pulled the whole series into the dma-mapping for-next tree.