The purpose of this serie is to enable CONFIG_VMAP_STACK on the 8xx.
I'm sending it now to get early feedback if any.
For the time being, there is no proper handling of stack overflow.
This serie applies on the top of the serie "powerpc: Switch to CONFIG_THREAD_INFO_IN_TASK"
Christophe Leroy (3):
powerpc/32: prepare for CONFIG_VMAP_STACK
powerpc/8xx: Use alternative scratch registers in DTLB miss handler
powerpc/8xx: Enable CONFIG_VMAP_STACK
arch/powerpc/Kconfig | 1 +
arch/powerpc/include/asm/processor.h | 5 ++
arch/powerpc/include/asm/thread_info.h | 5 ++
arch/powerpc/kernel/asm-offsets.c | 5 ++
arch/powerpc/kernel/entry_32.S | 16 +++++
arch/powerpc/kernel/head_8xx.S | 114 ++++++++++++++++++++++++++-------
arch/powerpc/perf/8xx-pmu.c | 12 ++--
7 files changed, 132 insertions(+), 26 deletions(-)
--
2.13.3
This patch enables CONFIG_VMAP_STACK. For that, a few changes are
done in head_8xx.S to re-activation DATA MMU Translation before
accessing to the stack.
Due to the growing of exception prolog, a few rearrangement is also
done in a few exception handlers.
Signed-off-by: Christophe Leroy <[email protected]>
---
arch/powerpc/Kconfig | 1 +
arch/powerpc/kernel/head_8xx.S | 87 +++++++++++++++++++++++++++++++++++++-----
2 files changed, 79 insertions(+), 9 deletions(-)
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index 94b46624068d..323b8a1efb3e 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -180,6 +180,7 @@ config PPC
select HAVE_ARCH_MMAP_RND_COMPAT_BITS if COMPAT
select HAVE_ARCH_SECCOMP_FILTER
select HAVE_ARCH_TRACEHOOK
+ select HAVE_ARCH_VMAP_STACK if PPC_8xx
select HAVE_CBPF_JIT if !PPC64
select HAVE_STACKPROTECTOR if PPC64 && $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=r13)
select HAVE_STACKPROTECTOR if PPC32 && $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=r2)
diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index 48996a424075..ded66a6fdfeb 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -136,6 +136,53 @@ instruction_counter:
EXCEPTION_PROLOG_1; \
EXCEPTION_PROLOG_2
+#ifdef CONFIG_VMAP_STACK
+#define EXCEPTION_PROLOG_1 \
+ mtspr SPRN_SPRG_SCRATCH2, r12; \
+ mfspr r12, SPRN_SPRG_THREAD; \
+ mfspr r11, SPRN_SRR0; \
+ stw r11, SRR0(r12); \
+ mfspr r11, SPRN_DAR; \
+ stw r11, DAR(r12); \
+ mfspr r11,SPRN_SRR1; /* check whether user or kernel */ \
+ stw r11, SRR1(r12); \
+ andi. r11,r11,MSR_PR
+
+#define EXCEPTION_PROLOG_2 \
+ li r11, MSR_KERNEL & ~(MSR_IR | MSR_RI); /* can take DTLB miss */ \
+ mtmsr r11; \
+ tovirt(r12, r12); \
+ subi r11, r1, INT_FRAME_SIZE; /* use r1 if kernel */ \
+ beq 1f; \
+ lwz r11, TASK_STACK-THREAD(r12); \
+ addi r11, r11, THREAD_SIZE - INT_FRAME_SIZE; \
+1: stw r10,_CCR(r11); /* save registers */ \
+ stw r9,GPR9(r11); \
+ mfspr r10,SPRN_SPRG_SCRATCH0; \
+ stw r10,GPR10(r11); \
+ mfspr r10,SPRN_SPRG_SCRATCH1; \
+ stw r10,GPR11(r11); \
+ mfspr r10,SPRN_SPRG_SCRATCH2; \
+ stw r10,GPR12(r11); \
+ mflr r10; \
+ stw r10,_LINK(r11); \
+ lwz r10, DAR(r12); \
+ stw r10, _DAR(r11); \
+ lwz r9, SRR1(r12); \
+ lwz r12, SRR0(r12); \
+ stw r1,GPR1(r11); \
+ stw r1,0(r11); \
+ mr r1, r11; /* set new kernel sp */ \
+ li r10,MSR_KERNEL & ~MSR_IR; /* can take exceptions */ \
+ mtmsr r10; \
+ stw r0,GPR0(r11); \
+ lis r10, STACK_FRAME_REGS_MARKER@ha; /* exception frame marker */ \
+ addi r10, r10, STACK_FRAME_REGS_MARKER@l; \
+ stw r10, 8(r11); \
+ SAVE_4GPRS(3, r11); \
+ SAVE_2GPRS(7, r11)
+
+#else
#define EXCEPTION_PROLOG_1 \
mfspr r11,SPRN_SRR1; /* check whether user or kernel */ \
andi. r11,r11,MSR_PR; \
@@ -172,6 +219,8 @@ instruction_counter:
SAVE_4GPRS(3, r11); \
SAVE_2GPRS(7, r11)
+#endif
+
/*
* Note: code which follows this uses cr0.eq (set if from kernel),
* r11, r12 (SRR0), and r9 (SRR1).
@@ -226,8 +275,12 @@ i##n: \
. = 0x200
MachineCheck:
EXCEPTION_PROLOG
+#ifdef CONFIG_VMAP_STACK
+ lwz r4, _DAR(r11)
+#else
mfspr r4,SPRN_DAR
stw r4,_DAR(r11)
+#endif
li r5,RPN_PATTERN
mtspr SPRN_DAR,r5 /* Tag DAR, to be used in DTLB Error */
mfspr r5,SPRN_DSISR
@@ -254,8 +307,12 @@ InstructionAccess:
. = 0x600
Alignment:
EXCEPTION_PROLOG
+#ifdef CONFIG_VMAP_STACK
+ lwz r4, _DAR(r11)
+#else
mfspr r4,SPRN_DAR
stw r4,_DAR(r11)
+#endif
li r5,RPN_PATTERN
mtspr SPRN_DAR,r5 /* Tag DAR, to be used in DTLB Error */
mfspr r5,SPRN_DSISR
@@ -573,20 +630,31 @@ DataTLBError:
beq- FixupDAR /* must be a buggy dcbX, icbi insn. */
DARFixed:/* Return from dcbx instruction bug workaround */
EXCEPTION_PROLOG_1
+#ifdef CONFIG_VMAP_STACK
+ li r11, RPN_PATTERN
+ mtspr SPRN_DAR, r11 /* Tag DAR, to be used in DTLB Error */
+#endif
EXCEPTION_PROLOG_2
mfspr r5,SPRN_DSISR
stw r5,_DSISR(r11)
+#ifdef CONFIG_VMAP_STACK
+ lwz r4, _DAR(r11)
+#else
mfspr r4,SPRN_DAR
+#endif
andis. r10,r5,DSISR_NOHPTE@h
beq+ 1f
tlbie r4
dtlbie:
-1: li r10,RPN_PATTERN
+1:
+#ifndef CONFIG_VMAP_STACK
+ li r10, RPN_PATTERN
mtspr SPRN_DAR,r10 /* Tag DAR, to be used in DTLB Error */
+#endif
/* 0x300 is DataAccess exception, needed by bad_page_fault() */
EXC_XFER_LITE(0x300, handle_page_fault)
- EXCEPTION(0x1500, Trap_15, unknown_exception, EXC_XFER_EE)
+/* EXCEPTION(0x1500, Trap_15, unknown_exception, EXC_XFER_EE)*/
EXCEPTION(0x1600, Trap_16, unknown_exception, EXC_XFER_EE)
EXCEPTION(0x1700, Trap_17, unknown_exception, EXC_XFER_EE)
EXCEPTION(0x1800, Trap_18, unknown_exception, EXC_XFER_EE)
@@ -598,6 +666,12 @@ dtlbie:
* support of breakpoints and such. Someday I will get around to
* using them.
*/
+11:
+ mtcr r10
+ mfspr r10, SPRN_SPRG_SCRATCH0
+ mfspr r11, SPRN_SPRG_SCRATCH1
+ rfi
+
. = 0x1c00
DataBreakpoint:
mtspr SPRN_SPRG_SCRATCH0, r10
@@ -606,8 +680,8 @@ DataBreakpoint:
mfspr r11, SPRN_SRR0
cmplwi cr0, r11, (dtlbie - PAGE_OFFSET)@l
cmplwi cr7, r11, (itlbie - PAGE_OFFSET)@l
- beq- cr0, 11f
- beq- cr7, 11f
+ beq- cr0, 11b
+ beq- cr7, 11b
EXCEPTION_PROLOG_1
EXCEPTION_PROLOG_2
addi r3,r1,STACK_FRAME_OVERHEAD
@@ -615,11 +689,6 @@ DataBreakpoint:
stw r4,_DAR(r11)
mfspr r5,SPRN_DSISR
EXC_XFER_EE(0x1c00, do_break)
-11:
- mtcr r10
- mfspr r10, SPRN_SPRG_SCRATCH0
- mfspr r11, SPRN_SPRG_SCRATCH1
- rfi
#ifdef CONFIG_PERF_EVENTS
. = 0x1d00
--
2.13.3
To support CONFIG_VMAP_STACK, the kernel must be able to activate
Data MMU Translation for accessing the stack. Before doing that
it must save SRR0, SRR1 and DAR in order to not loose them in
case there is a Data TLB Miss once the translation is reactivated.
This patch defines fields in the thread struct for saving those
registers. It also prepares entry_32.S to handle exception entry
with Data MMU Translation enabled.
Signed-off-by: Christophe Leroy <[email protected]>
---
arch/powerpc/include/asm/processor.h | 5 +++++
arch/powerpc/include/asm/thread_info.h | 5 +++++
arch/powerpc/kernel/asm-offsets.c | 5 +++++
arch/powerpc/kernel/entry_32.S | 16 ++++++++++++++++
4 files changed, 31 insertions(+)
diff --git a/arch/powerpc/include/asm/processor.h b/arch/powerpc/include/asm/processor.h
index 8179b64871ed..e839a1231b17 100644
--- a/arch/powerpc/include/asm/processor.h
+++ b/arch/powerpc/include/asm/processor.h
@@ -218,6 +218,11 @@ struct thread_struct {
#ifdef CONFIG_PPC32
void *pgdir; /* root of page-table tree */
unsigned long ksp_limit; /* if ksp <= ksp_limit stack overflow */
+#ifdef CONFIG_VMAP_STACK
+ unsigned long dar;
+ unsigned long srr0;
+ unsigned long srr1;
+#endif
#endif
/* Debug Registers */
struct debug_reg debug;
diff --git a/arch/powerpc/include/asm/thread_info.h b/arch/powerpc/include/asm/thread_info.h
index 8e1d0195ac36..488d5c4670ff 100644
--- a/arch/powerpc/include/asm/thread_info.h
+++ b/arch/powerpc/include/asm/thread_info.h
@@ -10,10 +10,15 @@
#define _ASM_POWERPC_THREAD_INFO_H
#include <asm/asm-const.h>
+#include <asm/page.h>
#ifdef __KERNEL__
+#if defined(CONFIG_VMAP_STACK) && CONFIG_THREAD_SHIFT < PAGE_SHIFT
+#define THREAD_SHIFT PAGE_SHIFT
+#else
#define THREAD_SHIFT CONFIG_THREAD_SHIFT
+#endif
#define THREAD_SIZE (1 << THREAD_SHIFT)
diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index 03439785c2ea..985523ef23e8 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -128,6 +128,11 @@ int main(void)
OFFSET(KSP_VSID, thread_struct, ksp_vsid);
#else /* CONFIG_PPC64 */
OFFSET(PGDIR, thread_struct, pgdir);
+#ifdef CONFIG_VMAP_STACK
+ OFFSET(SRR0, thread_struct, srr0);
+ OFFSET(SRR1, thread_struct, srr1);
+ OFFSET(DAR, thread_struct, dar);
+#endif
#ifdef CONFIG_SPE
OFFSET(THREAD_EVR0, thread_struct, evr[0]);
OFFSET(THREAD_ACC, thread_struct, acc);
diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S
index 52a061f14c7d..6e2c45fdd2c0 100644
--- a/arch/powerpc/kernel/entry_32.S
+++ b/arch/powerpc/kernel/entry_32.S
@@ -150,8 +150,13 @@ transfer_to_handler:
stw r12,_CTR(r11)
stw r2,_XER(r11)
mfspr r12,SPRN_SPRG_THREAD
+#ifdef CONFIG_VMAP_STACK
+ tovirt(r12, r12)
+#endif
addi r2,r12,-THREAD
+#ifndef CONFIG_VMAP_STACK
tovirt(r2,r2) /* set r2 to current */
+#endif
beq 2f /* if from user, fix up THREAD.regs */
addi r11,r1,STACK_FRAME_OVERHEAD
stw r11,PT_REGS(r12)
@@ -179,9 +184,13 @@ transfer_to_handler:
stw r12,4(r11)
#endif
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
+#ifdef CONFIG_VMAP_STACK
+ ACCOUNT_CPU_USER_ENTRY(r2, r11, r12)
+#else
tophys(r9, r2)
ACCOUNT_CPU_USER_ENTRY(r9, r11, r12)
#endif
+#endif
b 3f
@@ -193,8 +202,12 @@ transfer_to_handler:
ble- stack_ovf /* then the kernel stack overflowed */
5:
#if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_E500)
+#ifdef CONFIG_VMAP_STACK
+ lwz r12, TI_LOCAL_FLAGS(r2)
+#else
tophys(r9,r2) /* check local flags */
lwz r12,TI_LOCAL_FLAGS(r9)
+#endif
mtcrf 0x01,r12
bt- 31-TLF_NAPPING,4f
bt- 31-TLF_SLEEPING,7f
@@ -203,6 +216,9 @@ transfer_to_handler:
transfer_to_handler_cont:
3:
mflr r9
+#ifdef CONFIG_VMAP_STACK
+ tovirt(r9,r9)
+#endif
lwz r11,0(r9) /* virtual address of handler */
lwz r9,4(r9) /* where to go when done */
#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
--
2.13.3
In preparation of handling CONFIG_VMAP_STACK, we need DTLB miss handler
to use different scratch registers than other exception handlers in
order to not jeopardise exception entry on stack DTLB misses.
Signed-off-by: Christophe Leroy <[email protected]>
---
arch/powerpc/kernel/head_8xx.S | 27 ++++++++++++++-------------
arch/powerpc/perf/8xx-pmu.c | 12 ++++++++----
2 files changed, 22 insertions(+), 17 deletions(-)
diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index 5f5f89e87e3a..48996a424075 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -421,8 +421,8 @@ ITLBMissLinear:
. = 0x1200
DataStoreTLBMiss:
- mtspr SPRN_SPRG_SCRATCH0, r10
- mtspr SPRN_SPRG_SCRATCH1, r11
+ mtspr SPRN_DAR, r10
+ mtspr SPRN_M_TW, r11
mfcr r11
/* If we are faulting a kernel address, we have to use the
@@ -487,10 +487,10 @@ DataStoreTLBMiss:
mtspr SPRN_MD_RPN, r10 /* Update TLB entry */
/* Restore registers */
- mtspr SPRN_DAR, r11 /* Tag DAR */
-0: mfspr r10, SPRN_SPRG_SCRATCH0
- mfspr r11, SPRN_SPRG_SCRATCH1
+0: mfspr r10, SPRN_DAR
+ mtspr SPRN_DAR, r11 /* Tag DAR */
+ mfspr r11, SPRN_M_TW
rfi
patch_site 0b, patch__dtlbmiss_exit_1
@@ -499,8 +499,9 @@ DataStoreTLBMiss:
0: lwz r10, (dtlb_miss_counter - PAGE_OFFSET)@l(0)
addi r10, r10, 1
stw r10, (dtlb_miss_counter - PAGE_OFFSET)@l(0)
- mfspr r10, SPRN_SPRG_SCRATCH0
- mfspr r11, SPRN_SPRG_SCRATCH1
+ mfspr r10, SPRN_DAR
+ mtspr SPRN_DAR, r11 /* Tag DAR */
+ mfspr r11, SPRN_M_TW
rfi
#endif
@@ -516,10 +517,10 @@ DTLBMissIMMR:
mtspr SPRN_MD_RPN, r10 /* Update TLB entry */
li r11, RPN_PATTERN
- mtspr SPRN_DAR, r11 /* Tag DAR */
-0: mfspr r10, SPRN_SPRG_SCRATCH0
- mfspr r11, SPRN_SPRG_SCRATCH1
+0: mfspr r10, SPRN_DAR
+ mtspr SPRN_DAR, r11 /* Tag DAR */
+ mfspr r11, SPRN_M_TW
rfi
patch_site 0b, patch__dtlbmiss_exit_2
@@ -534,10 +535,10 @@ DTLBMissLinear:
mtspr SPRN_MD_RPN, r10 /* Update TLB entry */
li r11, RPN_PATTERN
- mtspr SPRN_DAR, r11 /* Tag DAR */
-0: mfspr r10, SPRN_SPRG_SCRATCH0
- mfspr r11, SPRN_SPRG_SCRATCH1
+0: mfspr r10, SPRN_DAR
+ mtspr SPRN_DAR, r11 /* Tag DAR */
+ mfspr r11, SPRN_M_TW
rfi
patch_site 0b, patch__dtlbmiss_exit_3
diff --git a/arch/powerpc/perf/8xx-pmu.c b/arch/powerpc/perf/8xx-pmu.c
index e38f74e9e7a4..4556c8837575 100644
--- a/arch/powerpc/perf/8xx-pmu.c
+++ b/arch/powerpc/perf/8xx-pmu.c
@@ -161,10 +161,6 @@ static void mpc8xx_pmu_read(struct perf_event *event)
static void mpc8xx_pmu_del(struct perf_event *event, int flags)
{
- /* mfspr r10, SPRN_SPRG_SCRATCH0 */
- unsigned int insn = PPC_INST_MFSPR | __PPC_RS(R10) |
- __PPC_SPR(SPRN_SPRG_SCRATCH0);
-
mpc8xx_pmu_read(event);
/* If it was the last user, stop counting to avoid useles overhead */
@@ -177,6 +173,10 @@ static void mpc8xx_pmu_del(struct perf_event *event, int flags)
break;
case PERF_8xx_ID_ITLB_LOAD_MISS:
if (atomic_dec_return(&itlb_miss_ref) == 0) {
+ /* mfspr r10, SPRN_SPRG_SCRATCH0 */
+ unsigned int insn = PPC_INST_MFSPR | __PPC_RS(R10) |
+ __PPC_SPR(SPRN_SPRG_SCRATCH0);
+
patch_instruction_site(&patch__itlbmiss_exit_1, insn);
#ifndef CONFIG_PIN_TLB_TEXT
patch_instruction_site(&patch__itlbmiss_exit_2, insn);
@@ -185,6 +185,10 @@ static void mpc8xx_pmu_del(struct perf_event *event, int flags)
break;
case PERF_8xx_ID_DTLB_LOAD_MISS:
if (atomic_dec_return(&dtlb_miss_ref) == 0) {
+ /* mfspr r10, SPRN_DAR */
+ unsigned int insn = PPC_INST_MFSPR | __PPC_RS(R10) |
+ __PPC_SPR(SPRN_DAR);
+
patch_instruction_site(&patch__dtlbmiss_exit_1, insn);
patch_instruction_site(&patch__dtlbmiss_exit_2, insn);
patch_instruction_site(&patch__dtlbmiss_exit_3, insn);
--
2.13.3