This patchset provides a further optimisation of TLB handling in the 8xx.
Changes are:
- Not saving registers like CR when not needed
- Adding support to any TASK_SIZE
Only the last patch of the set is changed compared to v4
Christophe Leroy (5):
powerpc/8xx: macro for handling CPU15 errata
powerpc/8xx: Handle CR out of exception PROLOG/EPILOG
powerpc/8xx: dont save CR in SCRATCH registers
powerpc/8xx: Use SPRG2 instead of DAR for saving r3
powerpc/8xx: Add support for TASK_SIZE greater than 0x80000000
arch/powerpc/kernel/head_8xx.S | 79 +++++++++++++++++++++++++++---------------
1 file changed, 51 insertions(+), 28 deletions(-)
--
2.1.0
Having a macro will help keep clear code.
Signed-off-by: Christophe Leroy <[email protected]>
---
arch/powerpc/kernel/head_8xx.S | 18 ++++++++++++------
1 file changed, 12 insertions(+), 6 deletions(-)
diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index 9b53fe1..1279018 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -297,6 +297,17 @@ SystemCall:
* We have to use the MD_xxx registers for the tablewalk because the
* equivalent MI_xxx registers only perform the attribute functions.
*/
+
+#ifdef CONFIG_8xx_CPU15
+#define INVALIDATE_ADJACENT_PAGES_CPU15(tmp, addr) \
+ addi tmp, addr, PAGE_SIZE; \
+ tlbie tmp; \
+ addi tmp, addr, -PAGE_SIZE; \
+ tlbie tmp
+#else
+#define INVALIDATE_ADJACENT_PAGES_CPU15(tmp, addr)
+#endif
+
InstructionTLBMiss:
#ifdef CONFIG_8xx_CPU6
mtspr SPRN_DAR, r3
@@ -304,12 +315,7 @@ InstructionTLBMiss:
EXCEPTION_PROLOG_0
mtspr SPRN_SPRG_SCRATCH2, r10
mfspr r10, SPRN_SRR0 /* Get effective address of fault */
-#ifdef CONFIG_8xx_CPU15
- addi r11, r10, PAGE_SIZE
- tlbie r11
- addi r11, r10, -PAGE_SIZE
- tlbie r11
-#endif
+ INVALIDATE_ADJACENT_PAGES_CPU15(r11, r10)
/* If we are faulting a kernel address, we have to use the
* kernel page tables.
--
2.1.0
In order to be able to reduce scope during which CR is saved, we take
CR saving/restoring out of exception PROLOG and EPILOG
Signed-off-by: Christophe Leroy <[email protected]>
---
arch/powerpc/kernel/head_8xx.S | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index 1279018..5a69c5e 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -116,13 +116,13 @@ turn_on_mmu:
*/
#define EXCEPTION_PROLOG \
EXCEPTION_PROLOG_0; \
+ mfcr r10; \
EXCEPTION_PROLOG_1; \
EXCEPTION_PROLOG_2
#define EXCEPTION_PROLOG_0 \
mtspr SPRN_SPRG_SCRATCH0,r10; \
- mtspr SPRN_SPRG_SCRATCH1,r11; \
- mfcr r10
+ mtspr SPRN_SPRG_SCRATCH1,r11
#define EXCEPTION_PROLOG_1 \
mfspr r11,SPRN_SRR1; /* check whether user or kernel */ \
@@ -162,7 +162,6 @@ turn_on_mmu:
* Exception exit code.
*/
#define EXCEPTION_EPILOG_0 \
- mtcr r10; \
mfspr r10,SPRN_SPRG_SCRATCH0; \
mfspr r11,SPRN_SPRG_SCRATCH1
@@ -313,6 +312,7 @@ InstructionTLBMiss:
mtspr SPRN_DAR, r3
#endif
EXCEPTION_PROLOG_0
+ mfcr r10
mtspr SPRN_SPRG_SCRATCH2, r10
mfspr r10, SPRN_SRR0 /* Get effective address of fault */
INVALIDATE_ADJACENT_PAGES_CPU15(r11, r10)
@@ -363,6 +363,7 @@ InstructionTLBMiss:
mtspr SPRN_DAR, r11 /* Tag DAR */
#endif
mfspr r10, SPRN_SPRG_SCRATCH2
+ mtcr r10
EXCEPTION_EPILOG_0
rfi
@@ -372,6 +373,7 @@ DataStoreTLBMiss:
mtspr SPRN_DAR, r3
#endif
EXCEPTION_PROLOG_0
+ mfcr r10
mtspr SPRN_SPRG_SCRATCH2, r10
mfspr r10, SPRN_MD_EPN
@@ -437,6 +439,7 @@ DataStoreTLBMiss:
#endif
mtspr SPRN_DAR, r11 /* Tag DAR */
mfspr r10, SPRN_SPRG_SCRATCH2
+ mtcr r10
EXCEPTION_EPILOG_0
rfi
@@ -462,6 +465,7 @@ InstructionTLBError:
. = 0x1400
DataTLBError:
EXCEPTION_PROLOG_0
+ mfcr r10
mfspr r11, SPRN_DAR
cmpwi cr0, r11, RPN_PATTERN
--
2.1.0
CR only needs to be preserved when checking if we are handling a kernel address.
So we can preserve CR in a register:
- In ITLBMiss, check is done only when CONFIG_MODULES is defined. Otherwise we
don't need to do anything at all with CR.
- We use r10, then we reload SRR0/MD_EPN into r10 when CR is restored
Signed-off-by: Christophe Leroy <[email protected]>
---
arch/powerpc/kernel/head_8xx.S | 29 +++++++++++++++--------------
1 file changed, 15 insertions(+), 14 deletions(-)
diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index 5a69c5e..150d03f 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -312,10 +312,6 @@ InstructionTLBMiss:
mtspr SPRN_DAR, r3
#endif
EXCEPTION_PROLOG_0
- mfcr r10
- mtspr SPRN_SPRG_SCRATCH2, r10
- mfspr r10, SPRN_SRR0 /* Get effective address of fault */
- INVALIDATE_ADJACENT_PAGES_CPU15(r11, r10)
/* If we are faulting a kernel address, we have to use the
* kernel page tables.
@@ -323,13 +319,20 @@ InstructionTLBMiss:
#ifdef CONFIG_MODULES
/* Only modules will cause ITLB Misses as we always
* pin the first 8MB of kernel memory */
- andis. r11, r10, 0x8000 /* Address >= 0x80000000 */
-#endif
+ mfspr r11, SPRN_SRR0 /* Get effective address of fault */
+ INVALIDATE_ADJACENT_PAGES_CPU15(r10, r11)
+ mfcr r10
+ andis. r11, r11, 0x8000 /* Address >= 0x80000000 */
mfspr r11, SPRN_M_TW /* Get level 1 table */
-#ifdef CONFIG_MODULES
beq 3f
lis r11, (swapper_pg_dir-PAGE_OFFSET)@ha
3:
+ mtcr r10
+ mfspr r10, SPRN_SRR0 /* Get effective address of fault */
+#else
+ mfspr r10, SPRN_SRR0 /* Get effective address of fault */
+ INVALIDATE_ADJACENT_PAGES_CPU15(r11, r10)
+ mfspr r11, SPRN_M_TW /* Get level 1 table base address */
#endif
/* Insert level 1 index */
rlwimi r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29
@@ -362,8 +365,6 @@ InstructionTLBMiss:
mfspr r3, SPRN_DAR
mtspr SPRN_DAR, r11 /* Tag DAR */
#endif
- mfspr r10, SPRN_SPRG_SCRATCH2
- mtcr r10
EXCEPTION_EPILOG_0
rfi
@@ -374,17 +375,19 @@ DataStoreTLBMiss:
#endif
EXCEPTION_PROLOG_0
mfcr r10
- mtspr SPRN_SPRG_SCRATCH2, r10
- mfspr r10, SPRN_MD_EPN
/* If we are faulting a kernel address, we have to use the
* kernel page tables.
*/
- andis. r11, r10, 0x8000
+ mfspr r11, SPRN_MD_EPN
+ andis. r11, r11, 0x8000
mfspr r11, SPRN_M_TW /* Get level 1 table */
beq 3f
lis r11, (swapper_pg_dir-PAGE_OFFSET)@ha
3:
+ mtcr r10
+ mfspr r10, SPRN_MD_EPN
+
/* Insert level 1 index */
rlwimi r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29
lwz r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11) /* Get the level 1 entry */
@@ -438,8 +441,6 @@ DataStoreTLBMiss:
mfspr r3, SPRN_DAR
#endif
mtspr SPRN_DAR, r11 /* Tag DAR */
- mfspr r10, SPRN_SPRG_SCRATCH2
- mtcr r10
EXCEPTION_EPILOG_0
rfi
--
2.1.0
We now have SPRG2 available as in it not used anymore for saving CR, so we don't
need to crash DAR anymore for saving r3 for CPU6 ERRATA handling.
Signed-off-by: Christophe Leroy <[email protected]>
---
arch/powerpc/kernel/head_8xx.S | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index 150d03f..ba2dc53 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -309,7 +309,7 @@ SystemCall:
InstructionTLBMiss:
#ifdef CONFIG_8xx_CPU6
- mtspr SPRN_DAR, r3
+ mtspr SPRN_SPRG_SCRATCH2, r3
#endif
EXCEPTION_PROLOG_0
@@ -362,8 +362,7 @@ InstructionTLBMiss:
/* Restore registers */
#ifdef CONFIG_8xx_CPU6
- mfspr r3, SPRN_DAR
- mtspr SPRN_DAR, r11 /* Tag DAR */
+ mfspr r3, SPRN_SPRG_SCRATCH2
#endif
EXCEPTION_EPILOG_0
rfi
@@ -371,7 +370,7 @@ InstructionTLBMiss:
. = 0x1200
DataStoreTLBMiss:
#ifdef CONFIG_8xx_CPU6
- mtspr SPRN_DAR, r3
+ mtspr SPRN_SPRG_SCRATCH2, r3
#endif
EXCEPTION_PROLOG_0
mfcr r10
@@ -438,7 +437,7 @@ DataStoreTLBMiss:
/* Restore registers */
#ifdef CONFIG_8xx_CPU6
- mfspr r3, SPRN_DAR
+ mfspr r3, SPRN_SPRG_SCRATCH2
#endif
mtspr SPRN_DAR, r11 /* Tag DAR */
EXCEPTION_EPILOG_0
--
2.1.0
By default, TASK_SIZE is set to 0x80000000 for PPC_8xx, which is most
likely sufficient for most cases. However, kernel configuration allows
to set TASK_SIZE to another value, so the 8xx shall handle it.
This patch also takes into account the case of PAGE_OFFSET lower than
0x80000000, allthought most of the time it is equal to 0xC0000000
Signed-off-by: Christophe Leroy <[email protected]>
---
arch/powerpc/kernel/head_8xx.S | 25 +++++++++++++++++++------
1 file changed, 19 insertions(+), 6 deletions(-)
diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index ba2dc53..c640bbb 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -48,6 +48,19 @@
mtspr spr, reg
#endif
+/* Macro to test if an address is a kernel address */
+#if CONFIG_TASK_SIZE <= 0x80000000 && CONFIG_PAGE_OFFSET >= 0x80000000
+#define IS_KERNEL(tmp, addr) \
+ andis. tmp, addr, 0x8000 /* Address >= 0x80000000 */
+#define BRANCH_UNLESS_KERNEL(label) beq label
+#else
+#define IS_KERNEL(tmp, addr) \
+ rlwinm tmp, addr, 16, 16, 31; \
+ cmpli cr0, tmp, PAGE_OFFSET >> 16
+#define BRANCH_UNLESS_KERNEL(label) blt label
+#endif
+
+
/*
* Value for the bits that have fixed value in RPN entries.
* Also used for tagging DAR for DTLBerror.
@@ -322,9 +335,9 @@ InstructionTLBMiss:
mfspr r11, SPRN_SRR0 /* Get effective address of fault */
INVALIDATE_ADJACENT_PAGES_CPU15(r10, r11)
mfcr r10
- andis. r11, r11, 0x8000 /* Address >= 0x80000000 */
+ IS_KERNEL(r11, r11)
mfspr r11, SPRN_M_TW /* Get level 1 table */
- beq 3f
+ BRANCH_UNLESS_KERNEL(3f)
lis r11, (swapper_pg_dir-PAGE_OFFSET)@ha
3:
mtcr r10
@@ -379,9 +392,9 @@ DataStoreTLBMiss:
* kernel page tables.
*/
mfspr r11, SPRN_MD_EPN
- andis. r11, r11, 0x8000
+ IS_KERNEL(r11, r11)
mfspr r11, SPRN_M_TW /* Get level 1 table */
- beq 3f
+ BRANCH_UNLESS_KERNEL(3f)
lis r11, (swapper_pg_dir-PAGE_OFFSET)@ha
3:
mtcr r10
@@ -513,9 +526,9 @@ FixupDAR:/* Entry point for dcbx workaround. */
mtspr SPRN_SPRG_SCRATCH2, r10
/* fetch instruction from memory. */
mfspr r10, SPRN_SRR0
- andis. r11, r10, 0x8000 /* Address >= 0x80000000 */
+ IS_KERNEL(r11, r10)
mfspr r11, SPRN_M_TW /* Get level 1 table */
- beq 3f
+ BRANCH_UNLESS_KERNEL(3f)
lis r11, (swapper_pg_dir-PAGE_OFFSET)@ha
/* Insert level 1 index */
3: rlwimi r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29
--
2.1.0
On Fri, 2015-04-17 at 18:32 +0200, root wrote:
> This patchset provides a further optimisation of TLB handling in the 8xx.
> Changes are:
> - Not saving registers like CR when not needed
> - Adding support to any TASK_SIZE
>
> Only the last patch of the set is changed compared to v4
>
> Christophe Leroy (5):
> powerpc/8xx: macro for handling CPU15 errata
> powerpc/8xx: Handle CR out of exception PROLOG/EPILOG
> powerpc/8xx: dont save CR in SCRATCH registers
> powerpc/8xx: Use SPRG2 instead of DAR for saving r3
> powerpc/8xx: Add support for TASK_SIZE greater than 0x80000000
>
> arch/powerpc/kernel/head_8xx.S | 79 +++++++++++++++++++++++++++---------------
> 1 file changed, 51 insertions(+), 28 deletions(-)
>
Do you really want your name in the git history to be "root"?
-Scott