A few architectures support uncached kernel segments. In that case we get
an uncached mapping for a given physica address by using an offset in the
uncached segement. Implement support for this scheme in the generic
dma-direct code instead of duplicating it in arch hooks.
Signed-off-by: Christoph Hellwig <[email protected]>
---
arch/Kconfig | 8 ++++++++
include/linux/dma-noncoherent.h | 3 +++
kernel/dma/direct.c | 18 ++++++++++++++++--
3 files changed, 27 insertions(+), 2 deletions(-)
diff --git a/arch/Kconfig b/arch/Kconfig
index 33687dddd86a..ea22a8c894ec 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -249,6 +249,14 @@ config ARCH_HAS_FORTIFY_SOURCE
config ARCH_HAS_SET_MEMORY
bool
+#
+# Select if arch has an uncached kernel segment and provides the
+# uncached_kernel_address / cached_kernel_address symbols to use it
+#
+config ARCH_HAS_UNCACHED_SEGMENT
+ select ARCH_HAS_DMA_PREP_COHERENT
+ bool
+
# Select if arch init_task must go in the __init_task_data section
config ARCH_TASK_STRUCT_ON_STACK
bool
diff --git a/include/linux/dma-noncoherent.h b/include/linux/dma-noncoherent.h
index 9741767e400f..7e0126a04e02 100644
--- a/include/linux/dma-noncoherent.h
+++ b/include/linux/dma-noncoherent.h
@@ -80,4 +80,7 @@ static inline void arch_dma_prep_coherent(struct page *page, size_t size)
}
#endif /* CONFIG_ARCH_HAS_DMA_PREP_COHERENT */
+void *uncached_kernel_address(void *addr);
+void *cached_kernel_address(void *addr);
+
#endif /* _LINUX_DMA_NONCOHERENT_H */
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index 2c2772e9702a..d15a535c3e67 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -164,6 +164,13 @@ void *dma_direct_alloc_pages(struct device *dev, size_t size,
}
ret = page_address(page);
+
+ if (IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) &&
+ !dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_NON_CONSISTENT)) {
+ arch_dma_prep_coherent(page, size);
+ ret = uncached_kernel_address(ret);
+ }
+
if (force_dma_unencrypted()) {
set_memory_decrypted((unsigned long)ret, 1 << get_order(size));
*dma_handle = __phys_to_dma(dev, page_to_phys(page));
@@ -171,6 +178,7 @@ void *dma_direct_alloc_pages(struct device *dev, size_t size,
*dma_handle = phys_to_dma(dev, page_to_phys(page));
}
memset(ret, 0, size);
+
return ret;
}
@@ -189,13 +197,18 @@ void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr,
if (force_dma_unencrypted())
set_memory_encrypted((unsigned long)cpu_addr, 1 << page_order);
+
+ if (IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) &&
+ !dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_NON_CONSISTENT))
+ cpu_addr = cached_kernel_address(cpu_addr);
__dma_direct_free_pages(dev, size, virt_to_page(cpu_addr));
}
void *dma_direct_alloc(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
{
- if (!dev_is_dma_coherent(dev))
+ if (!IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) &&
+ !dev_is_dma_coherent(dev))
return arch_dma_alloc(dev, size, dma_handle, gfp, attrs);
return dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs);
}
@@ -203,7 +216,8 @@ void *dma_direct_alloc(struct device *dev, size_t size,
void dma_direct_free(struct device *dev, size_t size,
void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs)
{
- if (!dev_is_dma_coherent(dev))
+ if (!IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) &&
+ !dev_is_dma_coherent(dev))
arch_dma_free(dev, size, cpu_addr, dma_addr, attrs);
else
dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs);
--
2.20.1
Hi Christoph,
On Tue, Apr 30, 2019 at 07:00:29AM -0400, Christoph Hellwig wrote:
> diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
> index 2c2772e9702a..d15a535c3e67 100644
> --- a/kernel/dma/direct.c
> +++ b/kernel/dma/direct.c
> @@ -164,6 +164,13 @@ void *dma_direct_alloc_pages(struct device *dev, size_t size,
> }
>
> ret = page_address(page);
> +
> + if (IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) &&
> + !dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_NON_CONSISTENT)) {
> + arch_dma_prep_coherent(page, size);
> + ret = uncached_kernel_address(ret);
> + }
> +
> if (force_dma_unencrypted()) {
> set_memory_decrypted((unsigned long)ret, 1 << get_order(size));
> *dma_handle = __phys_to_dma(dev, page_to_phys(page));
> @@ -171,6 +178,7 @@ void *dma_direct_alloc_pages(struct device *dev, size_t size,
> *dma_handle = phys_to_dma(dev, page_to_phys(page));
> }
> memset(ret, 0, size);
> +
> return ret;
> }
I'm not so sure about this part though.
On MIPS we currently don't clear the allocated memory with memset. Is
doing that really necessary?
If it is necessary then as-is this code will clear the allocated memory
using uncached writes which will be pretty slow. It would be much more
efficient to perform the memset before arch_dma_prep_coherent() & before
converting ret to an uncached address.
Thanks,
Paul
On Wed, May 01, 2019 at 05:18:59PM +0000, Paul Burton wrote:
> I'm not so sure about this part though.
>
> On MIPS we currently don't clear the allocated memory with memset. Is
> doing that really necessary?
We are clearling it on mips, it is inside dma_direct_alloc_pages.
> If it is necessary then as-is this code will clear the allocated memory
> using uncached writes which will be pretty slow. It would be much more
> efficient to perform the memset before arch_dma_prep_coherent() & before
> converting ret to an uncached address.
Yes, we could do that.
Hi Christoph,
On Wed, May 01, 2019 at 07:29:12PM +0200, Christoph Hellwig wrote:
> On Wed, May 01, 2019 at 05:18:59PM +0000, Paul Burton wrote:
> > I'm not so sure about this part though.
> >
> > On MIPS we currently don't clear the allocated memory with memset. Is
> > doing that really necessary?
>
> We are clearling it on mips, it is inside dma_direct_alloc_pages.
Ah, of course, I clearly require more caffeine :)
> > If it is necessary then as-is this code will clear the allocated memory
> > using uncached writes which will be pretty slow. It would be much more
> > efficient to perform the memset before arch_dma_prep_coherent() & before
> > converting ret to an uncached address.
>
> Yes, we could do that.
Great; using cached writes would match the existing MIPS behavior.
Thanks,
Paul
On Wed, May 01, 2019 at 05:40:34PM +0000, Paul Burton wrote:
> > > If it is necessary then as-is this code will clear the allocated memory
> > > using uncached writes which will be pretty slow. It would be much more
> > > efficient to perform the memset before arch_dma_prep_coherent() & before
> > > converting ret to an uncached address.
> >
> > Yes, we could do that.
>
> Great; using cached writes would match the existing MIPS behavior.
Can you test the stack with the two updated patches and ack them if
they are fine? That would allow getting at least the infrastructure
and mips in for this merge window.
Hi Christoph,
On Wed, May 01, 2019 at 07:49:05PM +0200, Christoph Hellwig wrote:
> On Wed, May 01, 2019 at 05:40:34PM +0000, Paul Burton wrote:
> > > > If it is necessary then as-is this code will clear the allocated memory
> > > > using uncached writes which will be pretty slow. It would be much more
> > > > efficient to perform the memset before arch_dma_prep_coherent() & before
> > > > converting ret to an uncached address.
> > >
> > > Yes, we could do that.
> >
> > Great; using cached writes would match the existing MIPS behavior.
>
> Can you test the stack with the two updated patches and ack them if
> they are fine? That would allow getting at least the infrastructure
> and mips in for this merge window.
Did you send a v2 of this patch?
If so it hasn't showed up in my inbox, nor on the linux-mips archive on
lore.kernel.org.
Thanks,
Paul
On Thu, May 02, 2019 at 12:08:01AM +0000, Paul Burton wrote:
> > Can you test the stack with the two updated patches and ack them if
> > they are fine? That would allow getting at least the infrastructure
> > and mips in for this merge window.
>
> Did you send a v2 of this patch?
>
> If so it hasn't showed up in my inbox, nor on the linux-mips archive on
> lore.kernel.org.
I did earlier in this thread. Here it is again:
---
From 247ca658ebeb7c8d04918747ec8a0da45c36bcb8 Mon Sep 17 00:00:00 2001
From: Christoph Hellwig <[email protected]>
Date: Sun, 28 Apr 2019 13:23:26 -0500
Subject: dma-direct: provide generic support for uncached kernel segments
A few architectures support uncached kernel segments. In that case we get
an uncached mapping for a given physica address by using an offset in the
uncached segement. Implement support for this scheme in the generic
dma-direct code instead of duplicating it in arch hooks.
Signed-off-by: Christoph Hellwig <[email protected]>
---
arch/Kconfig | 8 ++++++++
include/linux/dma-noncoherent.h | 3 +++
kernel/dma/direct.c | 17 +++++++++++++++--
3 files changed, 26 insertions(+), 2 deletions(-)
diff --git a/arch/Kconfig b/arch/Kconfig
index 33687dddd86a..ea22a8c894ec 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -249,6 +249,14 @@ config ARCH_HAS_FORTIFY_SOURCE
config ARCH_HAS_SET_MEMORY
bool
+#
+# Select if arch has an uncached kernel segment and provides the
+# uncached_kernel_address / cached_kernel_address symbols to use it
+#
+config ARCH_HAS_UNCACHED_SEGMENT
+ select ARCH_HAS_DMA_PREP_COHERENT
+ bool
+
# Select if arch init_task must go in the __init_task_data section
config ARCH_TASK_STRUCT_ON_STACK
bool
diff --git a/include/linux/dma-noncoherent.h b/include/linux/dma-noncoherent.h
index 9741767e400f..7e0126a04e02 100644
--- a/include/linux/dma-noncoherent.h
+++ b/include/linux/dma-noncoherent.h
@@ -80,4 +80,7 @@ static inline void arch_dma_prep_coherent(struct page *page, size_t size)
}
#endif /* CONFIG_ARCH_HAS_DMA_PREP_COHERENT */
+void *uncached_kernel_address(void *addr);
+void *cached_kernel_address(void *addr);
+
#endif /* _LINUX_DMA_NONCOHERENT_H */
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index 2c2772e9702a..6688e1cee7d1 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -171,6 +171,13 @@ void *dma_direct_alloc_pages(struct device *dev, size_t size,
*dma_handle = phys_to_dma(dev, page_to_phys(page));
}
memset(ret, 0, size);
+
+ if (IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) &&
+ !dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_NON_CONSISTENT)) {
+ arch_dma_prep_coherent(page, size);
+ ret = uncached_kernel_address(ret);
+ }
+
return ret;
}
@@ -189,13 +196,18 @@ void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr,
if (force_dma_unencrypted())
set_memory_encrypted((unsigned long)cpu_addr, 1 << page_order);
+
+ if (IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) &&
+ !dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_NON_CONSISTENT))
+ cpu_addr = cached_kernel_address(cpu_addr);
__dma_direct_free_pages(dev, size, virt_to_page(cpu_addr));
}
void *dma_direct_alloc(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
{
- if (!dev_is_dma_coherent(dev))
+ if (!IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) &&
+ !dev_is_dma_coherent(dev))
return arch_dma_alloc(dev, size, dma_handle, gfp, attrs);
return dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs);
}
@@ -203,7 +215,8 @@ void *dma_direct_alloc(struct device *dev, size_t size,
void dma_direct_free(struct device *dev, size_t size,
void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs)
{
- if (!dev_is_dma_coherent(dev))
+ if (!IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) &&
+ !dev_is_dma_coherent(dev))
arch_dma_free(dev, size, cpu_addr, dma_addr, attrs);
else
dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs);
--
2.20.1