__flush_dcache_icache() is only used in mem.c.
Declare it static.
Signed-off-by: Christophe Leroy <[email protected]>
---
arch/powerpc/include/asm/cacheflush.h | 1 -
arch/powerpc/mm/mem.c | 4 +++-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/arch/powerpc/include/asm/cacheflush.h b/arch/powerpc/include/asm/cacheflush.h
index f63495109f63..9110489ea411 100644
--- a/arch/powerpc/include/asm/cacheflush.h
+++ b/arch/powerpc/include/asm/cacheflush.h
@@ -40,7 +40,6 @@ void flush_icache_user_page(struct vm_area_struct *vma, struct page *page,
#define flush_icache_user_page flush_icache_user_page
void flush_dcache_icache_page(struct page *page);
-void __flush_dcache_icache(void *page);
/**
* flush_dcache_range(): Write any modified data cache blocks out to memory and
diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
index 7a59a5c9aa5d..ce6c81ce4362 100644
--- a/arch/powerpc/mm/mem.c
+++ b/arch/powerpc/mm/mem.c
@@ -472,6 +472,8 @@ void flush_dcache_page(struct page *page)
}
EXPORT_SYMBOL(flush_dcache_page);
+static void __flush_dcache_icache(void *p);
+
static void flush_dcache_icache_hugepage(struct page *page)
{
int i;
@@ -522,7 +524,7 @@ EXPORT_SYMBOL(flush_dcache_icache_page);
*
* @page: the address of the page to flush
*/
-void __flush_dcache_icache(void *p)
+static void __flush_dcache_icache(void *p)
{
unsigned long addr = (unsigned long)p;
--
2.25.0
flush_coherent_icache() can use any valid address as mentionned
by the comment.
Use PAGE_OFFSET as base address. This allows removing the
user access stuff.
Signed-off-by: Christophe Leroy <[email protected]>
---
arch/powerpc/mm/mem.c | 13 +++++--------
1 file changed, 5 insertions(+), 8 deletions(-)
diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
index ce6c81ce4362..19f807b87697 100644
--- a/arch/powerpc/mm/mem.c
+++ b/arch/powerpc/mm/mem.c
@@ -342,10 +342,9 @@ void free_initmem(void)
/**
* flush_coherent_icache() - if a CPU has a coherent icache, flush it
- * @addr: The base address to use (can be any valid address, the whole cache will be flushed)
* Return true if the cache was flushed, false otherwise
*/
-static inline bool flush_coherent_icache(unsigned long addr)
+static inline bool flush_coherent_icache(void)
{
/*
* For a snooping icache, we still need a dummy icbi to purge all the
@@ -355,9 +354,7 @@ static inline bool flush_coherent_icache(unsigned long addr)
*/
if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) {
mb(); /* sync */
- allow_read_from_user((const void __user *)addr, L1_CACHE_BYTES);
- icbi((void *)addr);
- prevent_read_from_user((const void __user *)addr, L1_CACHE_BYTES);
+ icbi((void *)PAGE_OFFSET);
mb(); /* sync */
isync();
return true;
@@ -397,7 +394,7 @@ static void invalidate_icache_range(unsigned long start, unsigned long stop)
*/
void flush_icache_range(unsigned long start, unsigned long stop)
{
- if (flush_coherent_icache(start))
+ if (flush_coherent_icache())
return;
clean_dcache_range(start, stop);
@@ -509,7 +506,7 @@ void flush_dcache_icache_page(struct page *page)
} else {
unsigned long addr = page_to_pfn(page) << PAGE_SHIFT;
- if (flush_coherent_icache(addr))
+ if (flush_coherent_icache())
return;
flush_dcache_icache_phys(addr);
}
@@ -528,7 +525,7 @@ static void __flush_dcache_icache(void *p)
{
unsigned long addr = (unsigned long)p;
- if (flush_coherent_icache(addr))
+ if (flush_coherent_icache())
return;
clean_dcache_range(addr, addr + PAGE_SIZE);
--
2.25.0
flush_coherent_icache() doesn't need the address anymore,
so it can be called immediately when entering the public
functions and doesn't need to be disseminated among
lower level functions.
And use page_to_phys() instead of open coding the calculation
of phys address to call flush_dcache_icache_phys().
Signed-off-by: Christophe Leroy <[email protected]>
---
arch/powerpc/mm/mem.c | 11 +++--------
1 file changed, 3 insertions(+), 8 deletions(-)
diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
index 19f807b87697..29ce215e491f 100644
--- a/arch/powerpc/mm/mem.c
+++ b/arch/powerpc/mm/mem.c
@@ -491,6 +491,8 @@ static void flush_dcache_icache_hugepage(struct page *page)
void flush_dcache_icache_page(struct page *page)
{
+ if (flush_coherent_icache())
+ return;
if (PageCompound(page))
return flush_dcache_icache_hugepage(page);
@@ -504,11 +506,7 @@ void flush_dcache_icache_page(struct page *page)
__flush_dcache_icache(start);
kunmap_atomic(start);
} else {
- unsigned long addr = page_to_pfn(page) << PAGE_SHIFT;
-
- if (flush_coherent_icache())
- return;
- flush_dcache_icache_phys(addr);
+ flush_dcache_icache_phys(page_to_phys(page));
}
#endif
}
@@ -525,9 +523,6 @@ static void __flush_dcache_icache(void *p)
{
unsigned long addr = (unsigned long)p;
- if (flush_coherent_icache())
- return;
-
clean_dcache_range(addr, addr + PAGE_SIZE);
/*
--
2.25.0
flush_dcache_page() is only a few lines, it is worth
inlining.
ia64, csky, mips, openrisc and riscv have a similar
flush_dcache_page() and inline it.
On pmac32_defconfig, we get a small size reduction.
On ppc64_defconfig, we get a very small size increase.
In both case that's in the noise (less than 0.1%).
text data bss dec hex filename
18991155 5934744 1497624 26423523 19330e3 vmlinux64.before
18994829 5936732 1497624 26429185 1934701 vmlinux64.after
9150963 2467502 184548 11803013 b41985 vmlinux32.before
9149689 2467302 184548 11801539 b413c3 vmlinux32.after
Signed-off-by: Christophe Leroy <[email protected]>
---
arch/powerpc/include/asm/cacheflush.h | 14 +++++++++++++-
arch/powerpc/mm/mem.c | 15 ---------------
2 files changed, 13 insertions(+), 16 deletions(-)
diff --git a/arch/powerpc/include/asm/cacheflush.h b/arch/powerpc/include/asm/cacheflush.h
index 9110489ea411..7564dd4fd12b 100644
--- a/arch/powerpc/include/asm/cacheflush.h
+++ b/arch/powerpc/include/asm/cacheflush.h
@@ -30,7 +30,19 @@ static inline void flush_cache_vmap(unsigned long start, unsigned long end)
#endif /* CONFIG_PPC_BOOK3S_64 */
#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1
-extern void flush_dcache_page(struct page *page);
+/*
+ * This is called when a page has been modified by the kernel.
+ * It just marks the page as not i-cache clean. We do the i-cache
+ * flush later when the page is given to a user process, if necessary.
+ */
+static inline void flush_dcache_page(struct page *page)
+{
+ if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
+ return;
+ /* avoid an atomic op if possible */
+ if (test_bit(PG_dcache_clean, &page->flags))
+ clear_bit(PG_dcache_clean, &page->flags);
+}
void flush_icache_range(unsigned long start, unsigned long stop);
#define flush_icache_range flush_icache_range
diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
index 460ab5000a3f..65b2205839fe 100644
--- a/arch/powerpc/mm/mem.c
+++ b/arch/powerpc/mm/mem.c
@@ -458,21 +458,6 @@ static void flush_dcache_icache_phys(unsigned long physaddr)
}
#endif
-/*
- * This is called when a page has been modified by the kernel.
- * It just marks the page as not i-cache clean. We do the i-cache
- * flush later when the page is given to a user process, if necessary.
- */
-void flush_dcache_page(struct page *page)
-{
- if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
- return;
- /* avoid an atomic op if possible */
- if (test_bit(PG_dcache_clean, &page->flags))
- clear_bit(PG_dcache_clean, &page->flags);
-}
-EXPORT_SYMBOL(flush_dcache_page);
-
static void __flush_dcache_icache(void *p);
static void flush_dcache_icache_hugepage(struct page *page)
--
2.25.0
Flushing functions don't rely on preemption being disabled, so
use kmap_local_page() instead of kmap_atomic().
Signed-off-by: Christophe Leroy <[email protected]>
---
arch/powerpc/mm/mem.c | 19 ++++++++++---------
1 file changed, 10 insertions(+), 9 deletions(-)
diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
index 65b2205839fe..1895bd64191a 100644
--- a/arch/powerpc/mm/mem.c
+++ b/arch/powerpc/mm/mem.c
@@ -464,16 +464,16 @@ static void flush_dcache_icache_hugepage(struct page *page)
{
int i;
int nr = compound_nr(page);
- void *start;
if (!PageHighMem(page)) {
for (i = 0; i < nr; i++)
__flush_dcache_icache(lowmem_page_address(page + i));
} else {
for (i = 0; i < nr; i++) {
- start = kmap_atomic(page+i);
+ void *start = kmap_local_page(page + i);
+
__flush_dcache_icache(start);
- kunmap_atomic(start);
+ kunmap_local(start);
}
}
}
@@ -489,9 +489,10 @@ void flush_dcache_icache_page(struct page *page)
if (!PageHighMem(page)) {
__flush_dcache_icache(lowmem_page_address(page));
} else if (IS_ENABLED(CONFIG_BOOKE) || sizeof(phys_addr_t) > sizeof(void *)) {
- void *start = kmap_atomic(page);
+ void *start = kmap_local_page(page);
+
__flush_dcache_icache(start);
- kunmap_atomic(start);
+ kunmap_local(start);
} else {
flush_dcache_icache_phys(page_to_phys(page));
}
@@ -564,11 +565,11 @@ void copy_user_page(void *vto, void *vfrom, unsigned long vaddr,
void flush_icache_user_page(struct vm_area_struct *vma, struct page *page,
unsigned long addr, int len)
{
- unsigned long maddr;
+ void *maddr;
- maddr = (unsigned long) kmap(page) + (addr & ~PAGE_MASK);
- flush_icache_range(maddr, maddr + len);
- kunmap(page);
+ maddr = kmap_local_page(page) + (addr & ~PAGE_MASK);
+ flush_icache_range((unsigned long)maddr, (unsigned long)maddr + len);
+ kunmap_local(maddr);
}
/*
--
2.25.0
'And' the given page address with PAGE_MASK to help GCC.
With the patch:
00000024 <__flush_dcache_icache>:
24: 54 63 00 26 rlwinm r3,r3,0,0,19
28: 39 40 00 40 li r10,64
2c: 7c 69 1b 78 mr r9,r3
30: 7d 49 03 a6 mtctr r10
34: 7c 00 48 6c dcbst 0,r9
38: 39 29 00 20 addi r9,r9,32
3c: 7c 00 48 6c dcbst 0,r9
40: 39 29 00 20 addi r9,r9,32
44: 42 00 ff f0 bdnz 34 <__flush_dcache_icache+0x10>
48: 7c 00 04 ac hwsync
4c: 39 20 00 40 li r9,64
50: 7d 29 03 a6 mtctr r9
54: 7c 00 1f ac icbi 0,r3
58: 38 63 00 20 addi r3,r3,32
5c: 7c 00 1f ac icbi 0,r3
60: 38 63 00 20 addi r3,r3,32
64: 42 00 ff f0 bdnz 54 <__flush_dcache_icache+0x30>
68: 7c 00 04 ac hwsync
6c: 4c 00 01 2c isync
70: 4e 80 00 20 blr
Without the patch:
00000024 <__flush_dcache_icache>:
24: 54 6a 00 34 rlwinm r10,r3,0,0,26
28: 39 23 10 1f addi r9,r3,4127
2c: 7d 2a 48 50 subf r9,r10,r9
30: 55 29 d9 7f rlwinm. r9,r9,27,5,31
34: 41 82 00 94 beq c8 <__flush_dcache_icache+0xa4>
38: 71 28 00 01 andi. r8,r9,1
3c: 38 c9 ff ff addi r6,r9,-1
40: 7d 48 53 78 mr r8,r10
44: 7d 27 4b 78 mr r7,r9
48: 40 82 00 6c bne b4 <__flush_dcache_icache+0x90>
4c: 54 e7 f8 7e rlwinm r7,r7,31,1,31
50: 7c e9 03 a6 mtctr r7
54: 7c 00 40 6c dcbst 0,r8
58: 39 08 00 20 addi r8,r8,32
5c: 7c 00 40 6c dcbst 0,r8
60: 39 08 00 20 addi r8,r8,32
64: 42 00 ff f0 bdnz 54 <__flush_dcache_icache+0x30>
68: 7c 00 04 ac hwsync
6c: 71 28 00 01 andi. r8,r9,1
70: 39 09 ff ff addi r8,r9,-1
74: 40 82 00 2c bne a0 <__flush_dcache_icache+0x7c>
78: 55 29 f8 7e rlwinm r9,r9,31,1,31
7c: 7d 29 03 a6 mtctr r9
80: 7c 00 57 ac icbi 0,r10
84: 39 4a 00 20 addi r10,r10,32
88: 7c 00 57 ac icbi 0,r10
8c: 39 4a 00 20 addi r10,r10,32
90: 42 00 ff f0 bdnz 80 <__flush_dcache_icache+0x5c>
94: 7c 00 04 ac hwsync
98: 4c 00 01 2c isync
9c: 4e 80 00 20 blr
a0: 7c 00 57 ac icbi 0,r10
a4: 2c 08 00 00 cmpwi r8,0
a8: 39 4a 00 20 addi r10,r10,32
ac: 40 82 ff cc bne 78 <__flush_dcache_icache+0x54>
b0: 4b ff ff e4 b 94 <__flush_dcache_icache+0x70>
b4: 7c 00 50 6c dcbst 0,r10
b8: 2c 06 00 00 cmpwi r6,0
bc: 39 0a 00 20 addi r8,r10,32
c0: 40 82 ff 8c bne 4c <__flush_dcache_icache+0x28>
c4: 4b ff ff a4 b 68 <__flush_dcache_icache+0x44>
c8: 7c 00 04 ac hwsync
cc: 7c 00 04 ac hwsync
d0: 4c 00 01 2c isync
d4: 4e 80 00 20 blr
Signed-off-by: Christophe Leroy <[email protected]>
---
arch/powerpc/mm/mem.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
index 9a5542f4de92..460ab5000a3f 100644
--- a/arch/powerpc/mm/mem.c
+++ b/arch/powerpc/mm/mem.c
@@ -522,7 +522,7 @@ EXPORT_SYMBOL(flush_dcache_icache_page);
*/
static void __flush_dcache_icache(void *p)
{
- unsigned long addr = (unsigned long)p;
+ unsigned long addr = (unsigned long)p & PAGE_MASK;
clean_dcache_range(addr, addr + PAGE_SIZE);
--
2.25.0
flush_dcache_icache_hugepage() is a static function, with
only one caller. That caller calls it when PageCompound() is true,
so bugging on !PageCompound() is useless if we can trust the
compiler a little. Remove the BUG_ON(!PageCompound()).
The number of elements of a page won't change over time, but
GCC doesn't know about it, so it gets the value at every iteration.
To avoid that, call compound_nr() outside the loop and save it in
a local variable.
Whether the page is a HIGHMEM page or not doesn't change over time.
But GCC doesn't know it so it does the test on every iteration.
Do the test outside the loop.
When the page is not a HIGHMEM page, page_address() will fallback on
lowmem_page_address(), so call lowmem_page_address() directly and
don't suffer the call to page_address() on every iteration.
Signed-off-by: Christophe Leroy <[email protected]>
---
arch/powerpc/mm/mem.c | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
index 29ce215e491f..d2c66827d9fd 100644
--- a/arch/powerpc/mm/mem.c
+++ b/arch/powerpc/mm/mem.c
@@ -474,14 +474,14 @@ static void __flush_dcache_icache(void *p);
static void flush_dcache_icache_hugepage(struct page *page)
{
int i;
+ int nr = compound_nr(page);
void *start;
- BUG_ON(!PageCompound(page));
-
- for (i = 0; i < compound_nr(page); i++) {
- if (!PageHighMem(page)) {
- __flush_dcache_icache(page_address(page+i));
- } else {
+ if (!PageHighMem(page)) {
+ for (i = 0; i < nr; i++)
+ __flush_dcache_icache(lowmem_page_address(page + i));
+ } else {
+ for (i = 0; i < nr; i++) {
start = kmap_atomic(page+i);
__flush_dcache_icache(start);
kunmap_atomic(start);
--
2.25.0
__flush_dcache_icache() is usable for non HIGHMEM pages on
every platform.
It is only for HIGHMEM pages that BOOKE needs kmap() and
BOOK3S needs flush_dcache_icache_phys().
So make flush_dcache_icache_phys() dependent on CONFIG_HIGHMEM and
call it only when it is a HIGHMEM page.
We could make flush_dcache_icache_phys() available at all time,
but as it is declared NOKPROBE_SYMBOL(), GCC doesn't optimise
it out when it is not used.
So define a stub for !CONFIG_HIGHMEM in order to remove the #ifdef in
flush_dcache_icache_page() and use IS_ENABLED() instead.
Signed-off-by: Christophe Leroy <[email protected]>
---
arch/powerpc/mm/mem.c | 17 +++++++++--------
1 file changed, 9 insertions(+), 8 deletions(-)
diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
index d2c66827d9fd..9a5542f4de92 100644
--- a/arch/powerpc/mm/mem.c
+++ b/arch/powerpc/mm/mem.c
@@ -413,7 +413,7 @@ void flush_icache_range(unsigned long start, unsigned long stop)
}
EXPORT_SYMBOL(flush_icache_range);
-#if !defined(CONFIG_PPC_8xx) && !defined(CONFIG_PPC64)
+#ifdef CONFIG_HIGHMEM
/**
* flush_dcache_icache_phys() - Flush a page by it's physical address
* @physaddr: the physical address of the page
@@ -452,7 +452,11 @@ static void flush_dcache_icache_phys(unsigned long physaddr)
: "ctr", "memory");
}
NOKPROBE_SYMBOL(flush_dcache_icache_phys)
-#endif // !defined(CONFIG_PPC_8xx) && !defined(CONFIG_PPC64)
+#else
+static void flush_dcache_icache_phys(unsigned long physaddr)
+{
+}
+#endif
/*
* This is called when a page has been modified by the kernel.
@@ -497,18 +501,15 @@ void flush_dcache_icache_page(struct page *page)
if (PageCompound(page))
return flush_dcache_icache_hugepage(page);
-#if defined(CONFIG_PPC_8xx) || defined(CONFIG_PPC64)
- /* On 8xx there is no need to kmap since highmem is not supported */
- __flush_dcache_icache(page_address(page));
-#else
- if (IS_ENABLED(CONFIG_BOOKE) || sizeof(phys_addr_t) > sizeof(void *)) {
+ if (!PageHighMem(page)) {
+ __flush_dcache_icache(lowmem_page_address(page));
+ } else if (IS_ENABLED(CONFIG_BOOKE) || sizeof(phys_addr_t) > sizeof(void *)) {
void *start = kmap_atomic(page);
__flush_dcache_icache(start);
kunmap_atomic(start);
} else {
flush_dcache_icache_phys(page_to_phys(page));
}
-#endif
}
EXPORT_SYMBOL(flush_dcache_icache_page);
--
2.25.0
Christophe Leroy <[email protected]> writes:
> flush_coherent_icache() can use any valid address as mentionned
> by the comment.
>
> Use PAGE_OFFSET as base address. This allows removing the
> user access stuff.
>
> Signed-off-by: Christophe Leroy <[email protected]>
> ---
> arch/powerpc/mm/mem.c | 13 +++++--------
> 1 file changed, 5 insertions(+), 8 deletions(-)
>
> diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
> index ce6c81ce4362..19f807b87697 100644
> --- a/arch/powerpc/mm/mem.c
> +++ b/arch/powerpc/mm/mem.c
> @@ -342,10 +342,9 @@ void free_initmem(void)
>
> /**
> * flush_coherent_icache() - if a CPU has a coherent icache, flush it
> - * @addr: The base address to use (can be any valid address, the whole cache will be flushed)
> * Return true if the cache was flushed, false otherwise
> */
> -static inline bool flush_coherent_icache(unsigned long addr)
> +static inline bool flush_coherent_icache(void)
> {
> /*
> * For a snooping icache, we still need a dummy icbi to purge all the
> @@ -355,9 +354,7 @@ static inline bool flush_coherent_icache(unsigned long addr)
> */
> if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) {
> mb(); /* sync */
> - allow_read_from_user((const void __user *)addr, L1_CACHE_BYTES);
> - icbi((void *)addr);
> - prevent_read_from_user((const void __user *)addr, L1_CACHE_BYTES);
> + icbi((void *)PAGE_OFFSET);
> mb(); /* sync */
> isync();
> return true;
do we need that followup sync? Usermanual suggest sync; icbi(any address);
isync sequence.
-aneesh
Le 08/04/2021 à 10:50, Aneesh Kumar K.V a écrit :
> Christophe Leroy <[email protected]> writes:
>
>> flush_coherent_icache() can use any valid address as mentionned
>> by the comment.
>>
>> Use PAGE_OFFSET as base address. This allows removing the
>> user access stuff.
>>
>> Signed-off-by: Christophe Leroy <[email protected]>
>> ---
>> arch/powerpc/mm/mem.c | 13 +++++--------
>> 1 file changed, 5 insertions(+), 8 deletions(-)
>>
>> diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
>> index ce6c81ce4362..19f807b87697 100644
>> --- a/arch/powerpc/mm/mem.c
>> +++ b/arch/powerpc/mm/mem.c
>> @@ -342,10 +342,9 @@ void free_initmem(void)
>>
>> /**
>> * flush_coherent_icache() - if a CPU has a coherent icache, flush it
>> - * @addr: The base address to use (can be any valid address, the whole cache will be flushed)
>> * Return true if the cache was flushed, false otherwise
>> */
>> -static inline bool flush_coherent_icache(unsigned long addr)
>> +static inline bool flush_coherent_icache(void)
>> {
>> /*
>> * For a snooping icache, we still need a dummy icbi to purge all the
>> @@ -355,9 +354,7 @@ static inline bool flush_coherent_icache(unsigned long addr)
>> */
>> if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) {
>> mb(); /* sync */
>> - allow_read_from_user((const void __user *)addr, L1_CACHE_BYTES);
>> - icbi((void *)addr);
>> - prevent_read_from_user((const void __user *)addr, L1_CACHE_BYTES);
>> + icbi((void *)PAGE_OFFSET);
>> mb(); /* sync */
>> isync();
>> return true;
>
> do we need that followup sync? Usermanual suggest sync; icbi(any address);
> isync sequence.
I don't know.
The original implementation is here: https://github.com/linuxppc/linux/commit/0ce636700
Christophe