This series drops the per-node-mode exception handlers.
The UnixBench test results are as follows, Loongson-3C5000L-LL, CLFS7.1,
a, DYNAMIC_FTRACE enabled, KASLR disabled, no-pernode
b, DYNAMIC_FTRACE enabled, KASLR disabled, pernode
c, DYNAMIC_FTRACE enabled, KASLR enabled, no-pernode
1 cpu 16 cpus
a-1, 782.6 4025.1
a-2, 783.3 3994.5
a-3, 784.2 4045.5
b-1, 775.5 3913.0
b-2, 763.9 3833.8
b-3, 777.7 3930.4
c-1, 811.0 3950.4
c-2, 795.7 3952.9
The results were almost identical, as Huacai said, the performance
difference between pernode and nopernode is negligible
v2:
Directly call rather than using jirl.
Thanks for Ruoyao.
Jinyang He (6):
LoongArch: Rename handle_syscall to handle_sys
LoongArch: Move the content defined by SYM_DATA to data or rodata
section
LoongArch: Create a exception handlers section
LoongArch: Drop pernode exception handlers
LoongArch: Fix up the prologue unwinder unwind exception frame
LoongArch: Clean up la_abs macro
arch/loongarch/include/asm/asmmacro.h | 17 -----
arch/loongarch/include/asm/setup.h | 14 ----
arch/loongarch/include/asm/stackframe.h | 2 +-
arch/loongarch/include/asm/traps.h | 81 ++++++++++++++++++++++
arch/loongarch/include/asm/unwind.h | 2 +-
arch/loongarch/kernel/entry.S | 15 ++--
arch/loongarch/kernel/genex.S | 75 ++++++++++++--------
arch/loongarch/kernel/head.S | 2 +
arch/loongarch/kernel/mcount_dyn.S | 2 +
arch/loongarch/kernel/relocate.c | 28 --------
arch/loongarch/kernel/traps.c | 74 +-------------------
arch/loongarch/kernel/unwind_prologue.c | 91 +++----------------------
arch/loongarch/kernel/vmlinux.lds.S | 37 +++++++---
arch/loongarch/mm/cache.c | 6 --
arch/loongarch/mm/tlb.c | 40 -----------
arch/loongarch/mm/tlbex.S | 71 +++++++++++--------
arch/loongarch/power/suspend.c | 5 +-
17 files changed, 230 insertions(+), 332 deletions(-)
create mode 100644 arch/loongarch/include/asm/traps.h
--
2.34.3
The original handle_sys() is a trampline for handler_syscall. Drop it
and rename handle_syscall to handle_sys for invoking it directly.
Signed-off-by: Jinyang He <[email protected]>
---
arch/loongarch/kernel/entry.S | 13 +++++++------
arch/loongarch/kernel/genex.S | 5 -----
2 files changed, 7 insertions(+), 11 deletions(-)
diff --git a/arch/loongarch/kernel/entry.S b/arch/loongarch/kernel/entry.S
index d737e3cf42d3..d6e9572eb279 100644
--- a/arch/loongarch/kernel/entry.S
+++ b/arch/loongarch/kernel/entry.S
@@ -18,9 +18,9 @@
.text
.cfi_sections .debug_frame
.align 5
-SYM_FUNC_START(handle_syscall)
+SYM_FUNC_START(handle_sys)
csrrd t0, PERCPU_BASE_KS
- la.pcrel t1, kernelsp
+ la_abs t1, kernelsp
add.d t1, t1, t0
move t2, sp
ld.d sp, t1, 0
@@ -62,12 +62,13 @@ SYM_FUNC_START(handle_syscall)
li.d tp, ~_THREAD_MASK
and tp, tp, sp
- move a0, sp
- bl do_syscall
+ move a0, sp
+ la_abs ra, do_syscall
+ jirl ra, ra, 0
RESTORE_ALL_AND_RET
-SYM_FUNC_END(handle_syscall)
-_ASM_NOKPROBE(handle_syscall)
+SYM_FUNC_END(handle_sys)
+_ASM_NOKPROBE(handle_sys)
SYM_CODE_START(ret_from_fork)
bl schedule_tail # a0 = struct task_struct *prev
diff --git a/arch/loongarch/kernel/genex.S b/arch/loongarch/kernel/genex.S
index 44ff1ff64260..1af169ca3734 100644
--- a/arch/loongarch/kernel/genex.S
+++ b/arch/loongarch/kernel/genex.S
@@ -91,8 +91,3 @@ SYM_FUNC_END(except_vec_cex)
BUILD_HANDLER ri ri none
BUILD_HANDLER watch watch none
BUILD_HANDLER reserved reserved none /* others */
-
-SYM_FUNC_START(handle_sys)
- la_abs t0, handle_syscall
- jr t0
-SYM_FUNC_END(handle_sys)
--
2.34.3
Due to the SYM_DATA do not give the section notes, manually set it.
Reported-by: Youling Tang <[email protected]>
Signed-off-by: Jinyang He <[email protected]>
---
arch/loongarch/kernel/genex.S | 2 ++
arch/loongarch/kernel/head.S | 2 ++
2 files changed, 4 insertions(+)
diff --git a/arch/loongarch/kernel/genex.S b/arch/loongarch/kernel/genex.S
index 1af169ca3734..de591a9270bd 100644
--- a/arch/loongarch/kernel/genex.S
+++ b/arch/loongarch/kernel/genex.S
@@ -77,7 +77,9 @@ SYM_FUNC_END(except_vec_cex)
668:
RESTORE_ALL_AND_RET
SYM_FUNC_END(handle_\exception)
+ .pushsection .rodata
SYM_DATA(unwind_hint_\exception, .word 668b - 666b)
+ .popsection
.endm
BUILD_HANDLER ade ade badv
diff --git a/arch/loongarch/kernel/head.S b/arch/loongarch/kernel/head.S
index aa64b179744f..7cfa1f850e54 100644
--- a/arch/loongarch/kernel/head.S
+++ b/arch/loongarch/kernel/head.S
@@ -32,9 +32,11 @@ _head:
pe_header:
__EFI_PE_HEADER
+.pushsection .rodata
SYM_DATA(kernel_asize, .long _end - _text);
SYM_DATA(kernel_fsize, .long _edata - _text);
SYM_DATA(kernel_offset, .long kernel_offset - _text);
+.popsection
#endif
--
2.34.3
Create a section named .ex_handlers to collect exception handlers.
Use strict alignment for using it later. As some handlers is shared
to other exception, we can not define symbol directly. Correct these
symbols by adding suffix or by using number label.
Signed-off-by: Jinyang He <[email protected]>
---
arch/loongarch/include/asm/traps.h | 39 +++++++++++++++++
arch/loongarch/kernel/entry.S | 5 +++
arch/loongarch/kernel/genex.S | 65 ++++++++++++++++++++++-------
arch/loongarch/kernel/vmlinux.lds.S | 19 +++++++++
arch/loongarch/mm/tlbex.S | 53 ++++++++++++++++-------
5 files changed, 149 insertions(+), 32 deletions(-)
create mode 100644 arch/loongarch/include/asm/traps.h
diff --git a/arch/loongarch/include/asm/traps.h b/arch/loongarch/include/asm/traps.h
new file mode 100644
index 000000000000..b348d66c16a1
--- /dev/null
+++ b/arch/loongarch/include/asm/traps.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2023 Loongson Technology Corporation Limited
+ */
+#ifndef _ASM_TRAPS_H
+#define _ASM_TRAPS_H
+
+#include <asm/loongarch.h> // For EXCCODES
+
+#ifdef __ASSEMBLY__
+/* Used for create exception handlers. */
+#define EXCCODE_UNDEF 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, \
+ 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, \
+ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, \
+ 59, 60, 61, 62, 63
+#define EXCCODE_NOHND EXCCODE_OOB, EXCCODE_BTE, EXCCODE_PSI, \
+ EXCCODE_HYP, EXCCODE_GCM, EXCCODE_SE
+
+/* As merrentry is set to the same value as eentry, the merr handler can
+ * be thought as the first exception handler.
+ */
+#define EXCCODE_MERR EXCCODE_RSV
+
+#define _SET_EXCEPTION_HANDLER(EXCCODE, ...) \
+ .pushsection .exception_handlers.##EXCCODE, "ax"; \
+ .align 9; \
+ __VA_ARGS__; \
+ .popsection;
+
+#define SET_EXCEPTION_HANDLER(EXCCODE, ...) \
+ _SET_EXCEPTION_HANDLER(EXCCODE, __VA_ARGS__)
+
+#define SET_TLBR_HANDLER(...) \
+ .pushsection .tlbrhandler, "ax"; \
+ __VA_ARGS__; \
+ .popsection;
+#endif /* __ASSEMBLY__ */
+
+#endif /* _ASM_TRAPS_H */
diff --git a/arch/loongarch/kernel/entry.S b/arch/loongarch/kernel/entry.S
index d6e9572eb279..ca4651f91e73 100644
--- a/arch/loongarch/kernel/entry.S
+++ b/arch/loongarch/kernel/entry.S
@@ -14,9 +14,11 @@
#include <asm/regdef.h>
#include <asm/stackframe.h>
#include <asm/thread_info.h>
+#include <asm/traps.h>
.text
.cfi_sections .debug_frame
+ .macro _handle_sys
.align 5
SYM_FUNC_START(handle_sys)
csrrd t0, PERCPU_BASE_KS
@@ -69,6 +71,9 @@ SYM_FUNC_START(handle_sys)
RESTORE_ALL_AND_RET
SYM_FUNC_END(handle_sys)
_ASM_NOKPROBE(handle_sys)
+ .endm
+
+SET_EXCEPTION_HANDLER(EXCCODE_SYS, _handle_sys)
SYM_CODE_START(ret_from_fork)
bl schedule_tail # a0 = struct task_struct *prev
diff --git a/arch/loongarch/kernel/genex.S b/arch/loongarch/kernel/genex.S
index de591a9270bd..86abafc8b95b 100644
--- a/arch/loongarch/kernel/genex.S
+++ b/arch/loongarch/kernel/genex.S
@@ -15,6 +15,7 @@
#include <asm/fpregdef.h>
#include <asm/stackframe.h>
#include <asm/thread_info.h>
+#include <asm/traps.h>
.align 5
SYM_FUNC_START(__arch_cpu_idle)
@@ -31,7 +32,8 @@ SYM_FUNC_START(__arch_cpu_idle)
1: jr ra
SYM_FUNC_END(__arch_cpu_idle)
-SYM_FUNC_START(handle_vint)
+ .macro _handle_vint idx
+SYM_FUNC_START(handle_vint\idx)
BACKUP_T0T1
SAVE_ALL
la_abs t1, __arch_cpu_idle
@@ -46,11 +48,14 @@ SYM_FUNC_START(handle_vint)
la_abs t0, do_vint
jirl ra, t0, 0
RESTORE_ALL_AND_RET
-SYM_FUNC_END(handle_vint)
+SYM_FUNC_END(handle_vint\idx)
+ .endm
+ .macro _except_vec_cex
SYM_FUNC_START(except_vec_cex)
b cache_parity_error
SYM_FUNC_END(except_vec_cex)
+ .endm
.macro build_prep_badv
csrrd t0, LOONGARCH_CSR_BADV
@@ -66,7 +71,7 @@ SYM_FUNC_END(except_vec_cex)
.macro BUILD_HANDLER exception handler prep
.align 5
- SYM_FUNC_START(handle_\exception)
+SYM_FUNC_START(handle_\exception)
666:
BACKUP_T0T1
SAVE_ALL
@@ -76,20 +81,48 @@ SYM_FUNC_END(except_vec_cex)
jirl ra, t0, 0
668:
RESTORE_ALL_AND_RET
- SYM_FUNC_END(handle_\exception)
+SYM_FUNC_END(handle_\exception)
.pushsection .rodata
- SYM_DATA(unwind_hint_\exception, .word 668b - 666b)
+SYM_DATA(unwind_hint_\exception, .word 668b - 666b)
.popsection
.endm
- BUILD_HANDLER ade ade badv
- BUILD_HANDLER ale ale badv
- BUILD_HANDLER bp bp none
- BUILD_HANDLER fpe fpe fcsr
- BUILD_HANDLER fpu fpu none
- BUILD_HANDLER lsx lsx none
- BUILD_HANDLER lasx lasx none
- BUILD_HANDLER lbt lbt none
- BUILD_HANDLER ri ri none
- BUILD_HANDLER watch watch none
- BUILD_HANDLER reserved reserved none /* others */
+SET_EXCEPTION_HANDLER(EXCCODE_SIP0, _handle_vint)
+SET_EXCEPTION_HANDLER(EXCCODE_SIP1, _handle_vint 1)
+SET_EXCEPTION_HANDLER(EXCCODE_IP0, _handle_vint 2)
+SET_EXCEPTION_HANDLER(EXCCODE_IP1, _handle_vint 3)
+SET_EXCEPTION_HANDLER(EXCCODE_IP2, _handle_vint 4)
+SET_EXCEPTION_HANDLER(EXCCODE_IP3, _handle_vint 5)
+SET_EXCEPTION_HANDLER(EXCCODE_IP4, _handle_vint 6)
+SET_EXCEPTION_HANDLER(EXCCODE_IP5, _handle_vint 7)
+SET_EXCEPTION_HANDLER(EXCCODE_IP6, _handle_vint 8)
+SET_EXCEPTION_HANDLER(EXCCODE_IP7, _handle_vint 9)
+SET_EXCEPTION_HANDLER(EXCCODE_PMC, _handle_vint 10)
+SET_EXCEPTION_HANDLER(EXCCODE_TIMER, _handle_vint 11)
+SET_EXCEPTION_HANDLER(EXCCODE_IPI, _handle_vint 12)
+SET_EXCEPTION_HANDLER(EXCCODE_NMI, _handle_vint 13)
+
+SET_EXCEPTION_HANDLER(EXCCODE_MERR, _except_vec_cex)
+
+SET_EXCEPTION_HANDLER(EXCCODE_ADE, BUILD_HANDLER ade ade badv)
+SET_EXCEPTION_HANDLER(EXCCODE_ALE, BUILD_HANDLER ale ale badv)
+SET_EXCEPTION_HANDLER(EXCCODE_BP, BUILD_HANDLER bp bp none)
+SET_EXCEPTION_HANDLER(EXCCODE_FPE, BUILD_HANDLER fpe fpe fcsr)
+SET_EXCEPTION_HANDLER(EXCCODE_FPDIS, BUILD_HANDLER fpu fpu none)
+SET_EXCEPTION_HANDLER(EXCCODE_LSXDIS, BUILD_HANDLER lsx lsx none)
+SET_EXCEPTION_HANDLER(EXCCODE_LASXDIS, BUILD_HANDLER lasx lasx none)
+SET_EXCEPTION_HANDLER(EXCCODE_BTDIS, BUILD_HANDLER lbt lbt none)
+SET_EXCEPTION_HANDLER(EXCCODE_INE, BUILD_HANDLER ri ri none)
+SET_EXCEPTION_HANDLER(EXCCODE_IPE, BUILD_HANDLER ri1 ri none)
+SET_EXCEPTION_HANDLER(EXCCODE_WATCH, BUILD_HANDLER watch watch none)
+
+.irp i, EXCCODE_UNDEF
+SET_EXCEPTION_HANDLER(\i, BUILD_HANDLER reserved\i reserved none)
+.endr
+
+.irp i, EXCCODE_NOHND
+SET_EXCEPTION_HANDLER(\i, BUILD_HANDLER reserved\i reserved none)
+.endr
+
+/* Create handle_reserved for temporary build. */
+BUILD_HANDLER reserved reserved none
diff --git a/arch/loongarch/kernel/vmlinux.lds.S b/arch/loongarch/kernel/vmlinux.lds.S
index ad2ce1a0908e..e99b50359900 100644
--- a/arch/loongarch/kernel/vmlinux.lds.S
+++ b/arch/loongarch/kernel/vmlinux.lds.S
@@ -21,6 +21,11 @@
*/
PECOFF_FILE_ALIGN = 0x200;
PECOFF_SEGMENT_ALIGN = 0x10000;
+EX_HANDLER_ALIGN = SZ_512;
+EX_HANDLER_SIZE = SZ_512;
+EX_HANDLER_TOTAL = 78; /* 64 Exceptions + 14 Interrupts */
+EXCEPTION_ENTRY_ALIGN = SZ_64K;
+TLBR_ENTRY_ALIGN = SZ_4K;
OUTPUT_ARCH(loongarch)
ENTRY(kernel_entry)
@@ -54,6 +59,17 @@ SECTIONS
. = ALIGN(PECOFF_SEGMENT_ALIGN);
_etext = .;
+ . = ALIGN(EXCEPTION_ENTRY_ALIGN);
+ .ex_handlers : {
+ __ex_handlers = .;
+ *(SORT_BY_INIT_PRIORITY(.exception_handlers.*))
+ . = ALIGN(EX_HANDLER_ALIGN);
+ __ex_handlers_end = .;
+ . = ALIGN(TLBR_ENTRY_ALIGN);
+ __tlbr_entry = .;
+ *(.tlbrhandler)
+ }
+
/*
* struct alt_inst entries. From the header (alternative.h):
* "Alternative instructions for different CPU types or capabilities"
@@ -154,3 +170,6 @@ SECTIONS
*(.eh_frame)
}
}
+
+ASSERT(__ex_handlers_end == (__ex_handlers + EX_HANDLER_SIZE * EX_HANDLER_TOTAL),
+ "The amount of exception handlers does not match!");
diff --git a/arch/loongarch/mm/tlbex.S b/arch/loongarch/mm/tlbex.S
index 244e2f5aeee5..53321d3447a2 100644
--- a/arch/loongarch/mm/tlbex.S
+++ b/arch/loongarch/mm/tlbex.S
@@ -9,6 +9,7 @@
#include <asm/pgtable.h>
#include <asm/regdef.h>
#include <asm/stackframe.h>
+#include <asm/traps.h>
#define INVTLB_ADDR_GFALSE_AND_ASID 5
@@ -32,7 +33,8 @@
tlb_do_page_fault 0
tlb_do_page_fault 1
-SYM_FUNC_START(handle_tlb_protect)
+ .macro _handle_tlb_protect idx
+SYM_FUNC_START(handle_tlb_protect\idx)
BACKUP_T0T1
SAVE_ALL
move a0, sp
@@ -42,9 +44,11 @@ SYM_FUNC_START(handle_tlb_protect)
la_abs t0, do_page_fault
jirl ra, t0, 0
RESTORE_ALL_AND_RET
-SYM_FUNC_END(handle_tlb_protect)
+SYM_FUNC_END(handle_tlb_protect\idx)
+ .endm
-SYM_FUNC_START(handle_tlb_load)
+ .macro _handle_tlb_load idx
+SYM_FUNC_START(handle_tlb_load\idx)
csrwr t0, EXCEPTION_KS0
csrwr t1, EXCEPTION_KS1
csrwr ra, EXCEPTION_KS2
@@ -53,10 +57,10 @@ SYM_FUNC_START(handle_tlb_load)
* The vmalloc handling is not in the hotpath.
*/
csrrd t0, LOONGARCH_CSR_BADV
- bltz t0, vmalloc_load
+ bltz t0, 3f
csrrd t1, LOONGARCH_CSR_PGDL
-vmalloc_done_load:
+1: /* vmalloc_done_load: */
/* Get PGD offset in bytes */
bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT
alsl.d t1, ra, t1, 3
@@ -78,25 +82,25 @@ vmalloc_done_load:
* see if we need to jump to huge tlb processing.
*/
rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1
- bltz ra, tlb_huge_update_load
+ bltz ra, 4f
rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1)
bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT
alsl.d t1, t0, ra, _PTE_T_LOG2
#ifdef CONFIG_SMP
-smp_pgtable_change_load:
+2: /* smp_pgtable_change_load: */
ll.d t0, t1, 0
#else
ld.d t0, t1, 0
#endif
andi ra, t0, _PAGE_PRESENT
- beqz ra, nopage_tlb_load
+ beqz ra, 5f
ori t0, t0, _PAGE_VALID
#ifdef CONFIG_SMP
sc.d t0, t1, 0
- beqz t0, smp_pgtable_change_load
+ beqz t0, 2b
#else
st.d t0, t1, 0
#endif
@@ -114,23 +118,23 @@ smp_pgtable_change_load:
ertn
#ifdef CONFIG_64BIT
-vmalloc_load:
+3: /* vmalloc_load: */
la_abs t1, swapper_pg_dir
- b vmalloc_done_load
+ b 1b
#endif
/* This is the entry point of a huge page. */
-tlb_huge_update_load:
+4: /* tlb_huge_update_load: */
#ifdef CONFIG_SMP
ll.d ra, t1, 0
#endif
andi t0, ra, _PAGE_PRESENT
- beqz t0, nopage_tlb_load
+ beqz t0, 5f
#ifdef CONFIG_SMP
ori t0, ra, _PAGE_VALID
sc.d t0, t1, 0
- beqz t0, tlb_huge_update_load
+ beqz t0, 4b
ori t0, ra, _PAGE_VALID
#else
rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1)
@@ -183,13 +187,15 @@ tlb_huge_update_load:
csrrd ra, EXCEPTION_KS2
ertn
-nopage_tlb_load:
+5: /* nopage_tlb_load: */
dbar 0
csrrd ra, EXCEPTION_KS2
la_abs t0, tlb_do_page_fault_0
jr t0
-SYM_FUNC_END(handle_tlb_load)
+SYM_FUNC_END(handle_tlb_load\idx)
+ .endm
+ .macro _handle_tlb_store
SYM_FUNC_START(handle_tlb_store)
csrwr t0, EXCEPTION_KS0
csrwr t1, EXCEPTION_KS1
@@ -338,7 +344,9 @@ nopage_tlb_store:
la_abs t0, tlb_do_page_fault_1
jr t0
SYM_FUNC_END(handle_tlb_store)
+ .endm
+ .macro _handle_tlb_modify
SYM_FUNC_START(handle_tlb_modify)
csrwr t0, EXCEPTION_KS0
csrwr t1, EXCEPTION_KS1
@@ -485,7 +493,9 @@ nopage_tlb_modify:
la_abs t0, tlb_do_page_fault_1
jr t0
SYM_FUNC_END(handle_tlb_modify)
+ .endm
+ .macro _handle_tlb_refill
SYM_FUNC_START(handle_tlb_refill)
csrwr t0, LOONGARCH_CSR_TLBRSAVE
csrrd t0, LOONGARCH_CSR_PGD
@@ -502,3 +512,14 @@ SYM_FUNC_START(handle_tlb_refill)
csrrd t0, LOONGARCH_CSR_TLBRSAVE
ertn
SYM_FUNC_END(handle_tlb_refill)
+ .endm
+
+SET_EXCEPTION_HANDLER(EXCCODE_TLBI, _handle_tlb_load)
+SET_EXCEPTION_HANDLER(EXCCODE_TLBL, _handle_tlb_load 1)
+SET_EXCEPTION_HANDLER(EXCCODE_TLBS, _handle_tlb_store)
+SET_EXCEPTION_HANDLER(EXCCODE_TLBM, _handle_tlb_modify)
+SET_EXCEPTION_HANDLER(EXCCODE_TLBNR, _handle_tlb_protect)
+SET_EXCEPTION_HANDLER(EXCCODE_TLBNX, _handle_tlb_protect 1)
+SET_EXCEPTION_HANDLER(EXCCODE_TLBPE, _handle_tlb_protect 2)
+
+SET_TLBR_HANDLER(_handle_tlb_refill)
--
2.34.3
1, set the value of CSR.EENTRY to &__ex_handlers.
2, set the value of CSR.TLBRENTRY to &__tlbr_entry.
3, set the value of CSR.MERRENTRY to &__ex_handlers.
Then, we can drop the pernode exception handlers.
Signed-off-by: Jinyang He <[email protected]>
---
arch/loongarch/include/asm/setup.h | 7 ---
arch/loongarch/include/asm/traps.h | 29 ++++++++++
arch/loongarch/kernel/genex.S | 8 ---
arch/loongarch/kernel/traps.c | 74 +------------------------
arch/loongarch/kernel/unwind_prologue.c | 62 +--------------------
arch/loongarch/mm/cache.c | 6 --
arch/loongarch/mm/tlb.c | 40 -------------
arch/loongarch/power/suspend.c | 5 +-
8 files changed, 33 insertions(+), 198 deletions(-)
diff --git a/arch/loongarch/include/asm/setup.h b/arch/loongarch/include/asm/setup.h
index be05c0e706a2..4074225339ec 100644
--- a/arch/loongarch/include/asm/setup.h
+++ b/arch/loongarch/include/asm/setup.h
@@ -9,17 +9,10 @@
#include <linux/types.h>
#include <uapi/asm/setup.h>
-#define VECSIZE 0x200
-
-extern unsigned long eentry;
-extern unsigned long tlbrentry;
extern char init_command_line[COMMAND_LINE_SIZE];
extern void tlb_init(int cpu);
extern void cpu_cache_init(void);
-extern void cache_error_setup(void);
extern void per_cpu_trap_init(int cpu);
-extern void set_handler(unsigned long offset, void *addr, unsigned long len);
-extern void set_merr_handler(unsigned long offset, void *addr, unsigned long len);
#ifdef CONFIG_RELOCATABLE
diff --git a/arch/loongarch/include/asm/traps.h b/arch/loongarch/include/asm/traps.h
index b348d66c16a1..8f276253f145 100644
--- a/arch/loongarch/include/asm/traps.h
+++ b/arch/loongarch/include/asm/traps.h
@@ -34,6 +34,35 @@
.pushsection .tlbrhandler, "ax"; \
__VA_ARGS__; \
.popsection;
+
+#else /* __ASSEMBLY__ */
+
+#define VECSIZE 0x200
+extern void *__ex_handlers;
+extern void *__tlbr_entry;
+
+static inline void set_eentry(void *entry)
+{
+ csr_write64((unsigned long)entry, LOONGARCH_CSR_EENTRY);
+}
+
+static inline void set_tlbrentry(void *entry)
+{
+ csr_write64((unsigned long)entry, LOONGARCH_CSR_TLBRENTRY);
+}
+
+static inline void set_merrentry(void *entry)
+{
+ csr_write64((unsigned long)entry, LOONGARCH_CSR_MERRENTRY);
+}
+
+static inline void configure_exception_vector(void)
+{
+ set_eentry(&__ex_handlers);
+ set_tlbrentry(&__tlbr_entry);
+ set_merrentry(&__ex_handlers);
+}
+
#endif /* __ASSEMBLY__ */
#endif /* _ASM_TRAPS_H */
diff --git a/arch/loongarch/kernel/genex.S b/arch/loongarch/kernel/genex.S
index 86abafc8b95b..256e2e5b83d4 100644
--- a/arch/loongarch/kernel/genex.S
+++ b/arch/loongarch/kernel/genex.S
@@ -72,19 +72,14 @@ SYM_FUNC_END(except_vec_cex)
.macro BUILD_HANDLER exception handler prep
.align 5
SYM_FUNC_START(handle_\exception)
- 666:
BACKUP_T0T1
SAVE_ALL
build_prep_\prep
move a0, sp
la_abs t0, do_\handler
jirl ra, t0, 0
- 668:
RESTORE_ALL_AND_RET
SYM_FUNC_END(handle_\exception)
- .pushsection .rodata
-SYM_DATA(unwind_hint_\exception, .word 668b - 666b)
- .popsection
.endm
SET_EXCEPTION_HANDLER(EXCCODE_SIP0, _handle_vint)
@@ -123,6 +118,3 @@ SET_EXCEPTION_HANDLER(\i, BUILD_HANDLER reserved\i reserved none)
.irp i, EXCCODE_NOHND
SET_EXCEPTION_HANDLER(\i, BUILD_HANDLER reserved\i reserved none)
.endr
-
-/* Create handle_reserved for temporary build. */
-BUILD_HANDLER reserved reserved none
diff --git a/arch/loongarch/kernel/traps.c b/arch/loongarch/kernel/traps.c
index c8b3bd76c941..84ac78bc8c57 100644
--- a/arch/loongarch/kernel/traps.c
+++ b/arch/loongarch/kernel/traps.c
@@ -43,6 +43,7 @@
#include <asm/siginfo.h>
#include <asm/stacktrace.h>
#include <asm/tlb.h>
+#include <asm/traps.h>
#include <asm/types.h>
#include <asm/unwind.h>
@@ -798,25 +799,8 @@ asmlinkage void noinstr do_vint(struct pt_regs *regs, unsigned long sp)
irqentry_exit(regs, state);
}
-unsigned long eentry;
-unsigned long tlbrentry;
-
-long exception_handlers[VECSIZE * 128 / sizeof(long)] __aligned(SZ_64K);
-
-static void configure_exception_vector(void)
-{
- eentry = (unsigned long)exception_handlers;
- tlbrentry = (unsigned long)exception_handlers + 80*VECSIZE;
-
- csr_write64(eentry, LOONGARCH_CSR_EENTRY);
- csr_write64(eentry, LOONGARCH_CSR_MERRENTRY);
- csr_write64(tlbrentry, LOONGARCH_CSR_TLBRENTRY);
-}
-
void per_cpu_trap_init(int cpu)
{
- unsigned int i;
-
setup_vint_size(VECSIZE);
configure_exception_vector();
@@ -829,62 +813,6 @@ void per_cpu_trap_init(int cpu)
BUG_ON(current->mm);
enter_lazy_tlb(&init_mm, current);
- /* Initialise exception handlers */
- if (cpu == 0)
- for (i = 0; i < 64; i++)
- set_handler(i * VECSIZE, handle_reserved, VECSIZE);
-
tlb_init(cpu);
cpu_cache_init();
}
-
-/* Install CPU exception handler */
-void set_handler(unsigned long offset, void *addr, unsigned long size)
-{
- memcpy((void *)(eentry + offset), addr, size);
- local_flush_icache_range(eentry + offset, eentry + offset + size);
-}
-
-static const char panic_null_cerr[] =
- "Trying to set NULL cache error exception handler\n";
-
-/*
- * Install uncached CPU exception handler.
- * This is suitable only for the cache error exception which is the only
- * exception handler that is being run uncached.
- */
-void set_merr_handler(unsigned long offset, void *addr, unsigned long size)
-{
- unsigned long uncached_eentry = TO_UNCACHE(__pa(eentry));
-
- if (!addr)
- panic(panic_null_cerr);
-
- memcpy((void *)(uncached_eentry + offset), addr, size);
-}
-
-void __init trap_init(void)
-{
- long i;
-
- /* Set interrupt vector handler */
- for (i = EXCCODE_INT_START; i < EXCCODE_INT_END; i++)
- set_handler(i * VECSIZE, handle_vint, VECSIZE);
-
- set_handler(EXCCODE_ADE * VECSIZE, handle_ade, VECSIZE);
- set_handler(EXCCODE_ALE * VECSIZE, handle_ale, VECSIZE);
- set_handler(EXCCODE_SYS * VECSIZE, handle_sys, VECSIZE);
- set_handler(EXCCODE_BP * VECSIZE, handle_bp, VECSIZE);
- set_handler(EXCCODE_INE * VECSIZE, handle_ri, VECSIZE);
- set_handler(EXCCODE_IPE * VECSIZE, handle_ri, VECSIZE);
- set_handler(EXCCODE_FPDIS * VECSIZE, handle_fpu, VECSIZE);
- set_handler(EXCCODE_LSXDIS * VECSIZE, handle_lsx, VECSIZE);
- set_handler(EXCCODE_LASXDIS * VECSIZE, handle_lasx, VECSIZE);
- set_handler(EXCCODE_FPE * VECSIZE, handle_fpe, VECSIZE);
- set_handler(EXCCODE_BTDIS * VECSIZE, handle_lbt, VECSIZE);
- set_handler(EXCCODE_WATCH * VECSIZE, handle_watch, VECSIZE);
-
- cache_error_setup();
-
- local_flush_icache_range(eentry, eentry + 0x400);
-}
diff --git a/arch/loongarch/kernel/unwind_prologue.c b/arch/loongarch/kernel/unwind_prologue.c
index 9095fde8e55d..de18335c6ba6 100644
--- a/arch/loongarch/kernel/unwind_prologue.c
+++ b/arch/loongarch/kernel/unwind_prologue.c
@@ -12,69 +12,9 @@
#include <asm/setup.h>
#include <asm/unwind.h>
-extern const int unwind_hint_ade;
-extern const int unwind_hint_ale;
-extern const int unwind_hint_bp;
-extern const int unwind_hint_fpe;
-extern const int unwind_hint_fpu;
-extern const int unwind_hint_lsx;
-extern const int unwind_hint_lasx;
-extern const int unwind_hint_lbt;
-extern const int unwind_hint_ri;
-extern const int unwind_hint_watch;
-extern unsigned long eentry;
-#ifdef CONFIG_NUMA
-extern unsigned long pcpu_handlers[NR_CPUS];
-#endif
-
-static inline bool scan_handlers(unsigned long entry_offset)
-{
- int idx, offset;
-
- if (entry_offset >= EXCCODE_INT_START * VECSIZE)
- return false;
-
- idx = entry_offset / VECSIZE;
- offset = entry_offset % VECSIZE;
- switch (idx) {
- case EXCCODE_ADE:
- return offset == unwind_hint_ade;
- case EXCCODE_ALE:
- return offset == unwind_hint_ale;
- case EXCCODE_BP:
- return offset == unwind_hint_bp;
- case EXCCODE_FPE:
- return offset == unwind_hint_fpe;
- case EXCCODE_FPDIS:
- return offset == unwind_hint_fpu;
- case EXCCODE_LSXDIS:
- return offset == unwind_hint_lsx;
- case EXCCODE_LASXDIS:
- return offset == unwind_hint_lasx;
- case EXCCODE_BTDIS:
- return offset == unwind_hint_lbt;
- case EXCCODE_INE:
- return offset == unwind_hint_ri;
- case EXCCODE_WATCH:
- return offset == unwind_hint_watch;
- default:
- return false;
- }
-}
-
static inline bool fix_exception(unsigned long pc)
{
-#ifdef CONFIG_NUMA
- int cpu;
-
- for_each_possible_cpu(cpu) {
- if (!pcpu_handlers[cpu])
- continue;
- if (scan_handlers(pc - pcpu_handlers[cpu]))
- return true;
- }
-#endif
- return scan_handlers(pc - eentry);
+ return false;
}
/*
diff --git a/arch/loongarch/mm/cache.c b/arch/loongarch/mm/cache.c
index 72685a48eaf0..d8e53702d61f 100644
--- a/arch/loongarch/mm/cache.c
+++ b/arch/loongarch/mm/cache.c
@@ -25,12 +25,6 @@
#include <asm/processor.h>
#include <asm/setup.h>
-void cache_error_setup(void)
-{
- extern char __weak except_vec_cex;
- set_merr_handler(0x0, &except_vec_cex, 0x80);
-}
-
/*
* LoongArch maintains ICache/DCache coherency by hardware,
* we just need "ibar" to avoid instruction hazard here.
diff --git a/arch/loongarch/mm/tlb.c b/arch/loongarch/mm/tlb.c
index 8bad6b0cff59..dda35eae1c49 100644
--- a/arch/loongarch/mm/tlb.c
+++ b/arch/loongarch/mm/tlb.c
@@ -250,50 +250,10 @@ static void output_pgtable_bits_defines(void)
pr_debug("\n");
}
-#ifdef CONFIG_NUMA
-unsigned long pcpu_handlers[NR_CPUS];
-#endif
-extern long exception_handlers[VECSIZE * 128 / sizeof(long)];
-
void setup_tlb_handler(int cpu)
{
setup_ptwalker();
local_flush_tlb_all();
-
- /* The tlb handlers are generated only once */
- if (cpu == 0) {
- memcpy((void *)tlbrentry, handle_tlb_refill, 0x80);
- local_flush_icache_range(tlbrentry, tlbrentry + 0x80);
- set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load, VECSIZE);
- set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load, VECSIZE);
- set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store, VECSIZE);
- set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify, VECSIZE);
- set_handler(EXCCODE_TLBNR * VECSIZE, handle_tlb_protect, VECSIZE);
- set_handler(EXCCODE_TLBNX * VECSIZE, handle_tlb_protect, VECSIZE);
- set_handler(EXCCODE_TLBPE * VECSIZE, handle_tlb_protect, VECSIZE);
- }
-#ifdef CONFIG_NUMA
- else {
- void *addr;
- struct page *page;
- const int vec_sz = sizeof(exception_handlers);
-
- if (pcpu_handlers[cpu])
- return;
-
- page = alloc_pages_node(cpu_to_node(cpu), GFP_ATOMIC, get_order(vec_sz));
- if (!page)
- return;
-
- addr = page_address(page);
- pcpu_handlers[cpu] = (unsigned long)addr;
- memcpy((void *)addr, (void *)eentry, vec_sz);
- local_flush_icache_range((unsigned long)addr, (unsigned long)addr + vec_sz);
- csr_write64(pcpu_handlers[cpu], LOONGARCH_CSR_EENTRY);
- csr_write64(pcpu_handlers[cpu], LOONGARCH_CSR_MERRENTRY);
- csr_write64(pcpu_handlers[cpu] + 80*VECSIZE, LOONGARCH_CSR_TLBRENTRY);
- }
-#endif
}
void tlb_init(int cpu)
diff --git a/arch/loongarch/power/suspend.c b/arch/loongarch/power/suspend.c
index 5e19733e5e05..b2a10a33b372 100644
--- a/arch/loongarch/power/suspend.c
+++ b/arch/loongarch/power/suspend.c
@@ -14,6 +14,7 @@
#include <asm/setup.h>
#include <asm/time.h>
#include <asm/tlbflush.h>
+#include <asm/traps.h>
u64 loongarch_suspend_addr;
@@ -45,9 +46,7 @@ static void arch_common_resume(void)
sync_counter();
local_flush_tlb_all();
csr_write64(per_cpu_offset(0), PERCPU_BASE_KS);
- csr_write64(eentry, LOONGARCH_CSR_EENTRY);
- csr_write64(eentry, LOONGARCH_CSR_MERRENTRY);
- csr_write64(tlbrentry, LOONGARCH_CSR_TLBRENTRY);
+ configure_exception_vector();
csr_write64(saved_regs.pgd, LOONGARCH_CSR_PGDL);
csr_write64(saved_regs.kpgd, LOONGARCH_CSR_PGDH);
--
2.34.3
It is a simply way to correct the unwind info when the special functions
influence the normal prologue analysis. We find out the position where
should unwind by PT_REGS, and mark it UNW_NEED_RESET. Linkers will
collect them and the prologue unwinder will compare them to pc.
Signed-off-by: Jinyang He <[email protected]>
---
arch/loongarch/include/asm/traps.h | 13 +++++++++
arch/loongarch/include/asm/unwind.h | 2 +-
arch/loongarch/kernel/genex.S | 1 +
arch/loongarch/kernel/mcount_dyn.S | 2 ++
arch/loongarch/kernel/unwind_prologue.c | 37 +++++++++----------------
arch/loongarch/kernel/vmlinux.lds.S | 9 ++++++
6 files changed, 39 insertions(+), 25 deletions(-)
diff --git a/arch/loongarch/include/asm/traps.h b/arch/loongarch/include/asm/traps.h
index 8f276253f145..0c30a024a9e5 100644
--- a/arch/loongarch/include/asm/traps.h
+++ b/arch/loongarch/include/asm/traps.h
@@ -5,6 +5,7 @@
#ifndef _ASM_TRAPS_H
#define _ASM_TRAPS_H
+#include <asm/asm.h>
#include <asm/loongarch.h> // For EXCCODES
#ifdef __ASSEMBLY__
@@ -35,11 +36,23 @@
__VA_ARGS__; \
.popsection;
+#ifdef CONFIG_UNWINDER_PROLOGUE
+#define UNW_NEED_RESET \
+ 668: \
+ .pushsection .unw_need_reset, "a"; \
+ PTR 668b; \
+ .popsection;
+#else /* CONFIG_UNWINDER_PROLOGUE */
+#define UNW_NEED_RESET
+#endif
+
#else /* __ASSEMBLY__ */
#define VECSIZE 0x200
extern void *__ex_handlers;
extern void *__tlbr_entry;
+extern void *__unw_need_reset;
+extern void *__unw_need_reset_end;
static inline void set_eentry(void *entry)
{
diff --git a/arch/loongarch/include/asm/unwind.h b/arch/loongarch/include/asm/unwind.h
index b9dce87afd2e..d9a10e264bdd 100644
--- a/arch/loongarch/include/asm/unwind.h
+++ b/arch/loongarch/include/asm/unwind.h
@@ -22,7 +22,7 @@ struct unwind_state {
char type; /* UNWINDER_XXX */
struct stack_info stack_info;
struct task_struct *task;
- bool first, error, reset;
+ bool first, error, need_reset;
int graph_idx;
unsigned long sp, pc, ra;
};
diff --git a/arch/loongarch/kernel/genex.S b/arch/loongarch/kernel/genex.S
index 256e2e5b83d4..8705a7661ce9 100644
--- a/arch/loongarch/kernel/genex.S
+++ b/arch/loongarch/kernel/genex.S
@@ -78,6 +78,7 @@ SYM_FUNC_START(handle_\exception)
move a0, sp
la_abs t0, do_\handler
jirl ra, t0, 0
+ UNW_NEED_RESET
RESTORE_ALL_AND_RET
SYM_FUNC_END(handle_\exception)
.endm
diff --git a/arch/loongarch/kernel/mcount_dyn.S b/arch/loongarch/kernel/mcount_dyn.S
index bbabf06244c2..3550bab52ff0 100644
--- a/arch/loongarch/kernel/mcount_dyn.S
+++ b/arch/loongarch/kernel/mcount_dyn.S
@@ -7,6 +7,7 @@
#include <asm/ftrace.h>
#include <asm/regdef.h>
#include <asm/stackframe.h>
+#include <asm/traps.h>
.text
/*
@@ -81,6 +82,7 @@ SYM_CODE_START(ftrace_common)
SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
bl ftrace_stub
+ UNW_NEED_RESET
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL)
nop /* b ftrace_graph_caller */
diff --git a/arch/loongarch/kernel/unwind_prologue.c b/arch/loongarch/kernel/unwind_prologue.c
index de18335c6ba6..aa01c881481c 100644
--- a/arch/loongarch/kernel/unwind_prologue.c
+++ b/arch/loongarch/kernel/unwind_prologue.c
@@ -10,33 +10,22 @@
#include <asm/loongson.h>
#include <asm/ptrace.h>
#include <asm/setup.h>
+#include <asm/traps.h>
#include <asm/unwind.h>
-static inline bool fix_exception(unsigned long pc)
-{
- return false;
-}
-
-/*
- * As we meet ftrace_regs_entry, reset first flag like first doing
- * tracing. Prologue analysis will stop soon because PC is at entry.
- */
-static inline bool fix_ftrace(unsigned long pc)
-{
-#ifdef CONFIG_DYNAMIC_FTRACE
- return pc == (unsigned long)ftrace_call + LOONGARCH_INSN_SIZE;
-#else
- return false;
-#endif
-}
-
static inline bool unwind_state_fixup(struct unwind_state *state)
{
- if (!fix_exception(state->pc) && !fix_ftrace(state->pc))
- return false;
+ unsigned long *p = (unsigned long *)&__unw_need_reset;
+ unsigned long *q = (unsigned long *)&__unw_need_reset_end;
- state->reset = true;
- return true;
+ for (; p < q; p++) {
+ if (*p != state->pc)
+ continue;
+ state->need_reset = true;
+ return true;
+ }
+
+ return false;
}
/*
@@ -59,10 +48,10 @@ static bool unwind_by_prologue(struct unwind_state *state)
if (state->sp >= info->end || state->sp < info->begin)
return false;
- if (state->reset) {
+ if (state->need_reset) {
regs = (struct pt_regs *)state->sp;
state->first = true;
- state->reset = false;
+ state->need_reset = false;
state->pc = regs->csr_era;
state->ra = regs->regs[1];
state->sp = regs->regs[3];
diff --git a/arch/loongarch/kernel/vmlinux.lds.S b/arch/loongarch/kernel/vmlinux.lds.S
index e99b50359900..00f1f9061961 100644
--- a/arch/loongarch/kernel/vmlinux.lds.S
+++ b/arch/loongarch/kernel/vmlinux.lds.S
@@ -70,6 +70,15 @@ SECTIONS
*(.tlbrhandler)
}
+#ifdef CONFIG_UNWINDER_PROLOGUE
+ . = ALIGN(8);
+ .unw_need_reset : {
+ __unw_need_reset = .;
+ *(.unw_need_reset)
+ __unw_need_reset_end = .;
+ }
+#endif
+
/*
* struct alt_inst entries. From the header (alternative.h):
* "Alternative instructions for different CPU types or capabilities"
--
2.34.3
Now we can replace la_abs with la.pcrel. Clean up the la_abs macro.
Signed-off-by: Jinyang He <[email protected]>
---
arch/loongarch/include/asm/asmmacro.h | 17 ---------------
arch/loongarch/include/asm/setup.h | 7 -------
arch/loongarch/include/asm/stackframe.h | 2 +-
arch/loongarch/kernel/entry.S | 5 ++---
arch/loongarch/kernel/genex.S | 8 +++----
arch/loongarch/kernel/relocate.c | 28 -------------------------
arch/loongarch/kernel/vmlinux.lds.S | 9 --------
arch/loongarch/mm/tlbex.S | 18 +++++++---------
8 files changed, 13 insertions(+), 81 deletions(-)
diff --git a/arch/loongarch/include/asm/asmmacro.h b/arch/loongarch/include/asm/asmmacro.h
index 79e1d53fea89..328bb956f241 100644
--- a/arch/loongarch/include/asm/asmmacro.h
+++ b/arch/loongarch/include/asm/asmmacro.h
@@ -667,21 +667,4 @@
nor \dst, \src, zero
.endm
-.macro la_abs reg, sym
-#ifndef CONFIG_RELOCATABLE
- la.abs \reg, \sym
-#else
- 766:
- lu12i.w \reg, 0
- ori \reg, \reg, 0
- lu32i.d \reg, 0
- lu52i.d \reg, \reg, 0
- .pushsection ".la_abs", "aw", %progbits
- 768:
- .dword 768b-766b
- .dword \sym
- .popsection
-#endif
-.endm
-
#endif /* _ASM_ASMMACRO_H */
diff --git a/arch/loongarch/include/asm/setup.h b/arch/loongarch/include/asm/setup.h
index 4074225339ec..84d131f29e0c 100644
--- a/arch/loongarch/include/asm/setup.h
+++ b/arch/loongarch/include/asm/setup.h
@@ -16,13 +16,6 @@ extern void per_cpu_trap_init(int cpu);
#ifdef CONFIG_RELOCATABLE
-struct rela_la_abs {
- long offset;
- long symvalue;
-};
-
-extern long __la_abs_begin;
-extern long __la_abs_end;
extern long __rela_dyn_begin;
extern long __rela_dyn_end;
diff --git a/arch/loongarch/include/asm/stackframe.h b/arch/loongarch/include/asm/stackframe.h
index bb5e0e2bd3a9..fff4a0ce7750 100644
--- a/arch/loongarch/include/asm/stackframe.h
+++ b/arch/loongarch/include/asm/stackframe.h
@@ -86,7 +86,7 @@
* new value in sp.
*/
.macro get_saved_sp docfi=0
- la_abs t1, kernelsp
+ la.pcrel t1, kernelsp
#ifdef CONFIG_SMP
csrrd t0, PERCPU_BASE_KS
LONG_ADD t1, t1, t0
diff --git a/arch/loongarch/kernel/entry.S b/arch/loongarch/kernel/entry.S
index ca4651f91e73..87b620a0aa13 100644
--- a/arch/loongarch/kernel/entry.S
+++ b/arch/loongarch/kernel/entry.S
@@ -22,7 +22,7 @@
.align 5
SYM_FUNC_START(handle_sys)
csrrd t0, PERCPU_BASE_KS
- la_abs t1, kernelsp
+ la.pcrel t1, kernelsp
add.d t1, t1, t0
move t2, sp
ld.d sp, t1, 0
@@ -65,8 +65,7 @@ SYM_FUNC_START(handle_sys)
and tp, tp, sp
move a0, sp
- la_abs ra, do_syscall
- jirl ra, ra, 0
+ bl do_syscall
RESTORE_ALL_AND_RET
SYM_FUNC_END(handle_sys)
diff --git a/arch/loongarch/kernel/genex.S b/arch/loongarch/kernel/genex.S
index 8705a7661ce9..061727164ec2 100644
--- a/arch/loongarch/kernel/genex.S
+++ b/arch/loongarch/kernel/genex.S
@@ -36,7 +36,7 @@ SYM_FUNC_END(__arch_cpu_idle)
SYM_FUNC_START(handle_vint\idx)
BACKUP_T0T1
SAVE_ALL
- la_abs t1, __arch_cpu_idle
+ la.pcrel t1, __arch_cpu_idle
LONG_L t0, sp, PT_ERA
/* 32 byte rollback region */
ori t0, t0, 0x1f
@@ -45,8 +45,7 @@ SYM_FUNC_START(handle_vint\idx)
LONG_S t0, sp, PT_ERA
1: move a0, sp
move a1, sp
- la_abs t0, do_vint
- jirl ra, t0, 0
+ bl do_vint
RESTORE_ALL_AND_RET
SYM_FUNC_END(handle_vint\idx)
.endm
@@ -76,8 +75,7 @@ SYM_FUNC_START(handle_\exception)
SAVE_ALL
build_prep_\prep
move a0, sp
- la_abs t0, do_\handler
- jirl ra, t0, 0
+ bl do_\handler
UNW_NEED_RESET
RESTORE_ALL_AND_RET
SYM_FUNC_END(handle_\exception)
diff --git a/arch/loongarch/kernel/relocate.c b/arch/loongarch/kernel/relocate.c
index 070b9fb87ebc..5ecfd7791268 100644
--- a/arch/loongarch/kernel/relocate.c
+++ b/arch/loongarch/kernel/relocate.c
@@ -12,7 +12,6 @@
#include <linux/start_kernel.h>
#include <asm/bootinfo.h>
#include <asm/early_ioremap.h>
-#include <asm/inst.h>
#include <asm/sections.h>
#include <asm/setup.h>
@@ -41,31 +40,6 @@ static inline __init void relocate_relative(void)
}
}
-static inline void __init relocate_la_abs(long random_offset)
-{
- void *begin, *end;
- struct rela_la_abs *p;
-
- begin = RELOCATED_KASLR(&__la_abs_begin);
- end = RELOCATED_KASLR(&__la_abs_end);
-
- for (p = begin; (void *)p < end; p++) {
- long v = p->symvalue;
- uint32_t lu12iw, ori, lu32id, lu52id;
- union loongarch_instruction *insn = (void *)p - p->offset;
-
- lu12iw = (v >> 12) & 0xfffff;
- ori = v & 0xfff;
- lu32id = (v >> 32) & 0xfffff;
- lu52id = v >> 52;
-
- insn[0].reg1i20_format.immediate = lu12iw;
- insn[1].reg2i12_format.immediate = ori;
- insn[2].reg1i20_format.immediate = lu32id;
- insn[3].reg2i12_format.immediate = lu52id;
- }
-}
-
#ifdef CONFIG_RANDOMIZE_BASE
static inline __init unsigned long rotate_xor(unsigned long hash,
const void *area, size_t size)
@@ -202,8 +176,6 @@ void * __init relocate_kernel(void)
if (reloc_offset)
relocate_relative();
- relocate_la_abs(random_offset);
-
return kernel_entry;
}
diff --git a/arch/loongarch/kernel/vmlinux.lds.S b/arch/loongarch/kernel/vmlinux.lds.S
index 00f1f9061961..8ae5f245eb2f 100644
--- a/arch/loongarch/kernel/vmlinux.lds.S
+++ b/arch/loongarch/kernel/vmlinux.lds.S
@@ -91,15 +91,6 @@ SECTIONS
__alt_instructions_end = .;
}
-#ifdef CONFIG_RELOCATABLE
- . = ALIGN(8);
- .la_abs : AT(ADDR(.la_abs) - LOAD_OFFSET) {
- __la_abs_begin = .;
- *(.la_abs)
- __la_abs_end = .;
- }
-#endif
-
.got : ALIGN(16) { *(.got) }
.plt : ALIGN(16) { *(.plt) }
.got.plt : ALIGN(16) { *(.got.plt) }
diff --git a/arch/loongarch/mm/tlbex.S b/arch/loongarch/mm/tlbex.S
index 53321d3447a2..3ee16fc61788 100644
--- a/arch/loongarch/mm/tlbex.S
+++ b/arch/loongarch/mm/tlbex.S
@@ -41,8 +41,7 @@ SYM_FUNC_START(handle_tlb_protect\idx)
move a1, zero
csrrd a2, LOONGARCH_CSR_BADV
REG_S a2, sp, PT_BVADDR
- la_abs t0, do_page_fault
- jirl ra, t0, 0
+ bl do_page_fault
RESTORE_ALL_AND_RET
SYM_FUNC_END(handle_tlb_protect\idx)
.endm
@@ -119,7 +118,7 @@ SYM_FUNC_START(handle_tlb_load\idx)
#ifdef CONFIG_64BIT
3: /* vmalloc_load: */
- la_abs t1, swapper_pg_dir
+ la.pcrel t1, swapper_pg_dir
b 1b
#endif
@@ -190,8 +189,7 @@ SYM_FUNC_START(handle_tlb_load\idx)
5: /* nopage_tlb_load: */
dbar 0
csrrd ra, EXCEPTION_KS2
- la_abs t0, tlb_do_page_fault_0
- jr t0
+ b tlb_do_page_fault_0
SYM_FUNC_END(handle_tlb_load\idx)
.endm
@@ -268,7 +266,7 @@ smp_pgtable_change_store:
#ifdef CONFIG_64BIT
vmalloc_store:
- la_abs t1, swapper_pg_dir
+ la.pcrel t1, swapper_pg_dir
b vmalloc_done_store
#endif
@@ -341,8 +339,7 @@ tlb_huge_update_store:
nopage_tlb_store:
dbar 0
csrrd ra, EXCEPTION_KS2
- la_abs t0, tlb_do_page_fault_1
- jr t0
+ b tlb_do_page_fault_1
SYM_FUNC_END(handle_tlb_store)
.endm
@@ -418,7 +415,7 @@ smp_pgtable_change_modify:
#ifdef CONFIG_64BIT
vmalloc_modify:
- la_abs t1, swapper_pg_dir
+ la.pcrel t1, swapper_pg_dir
b vmalloc_done_modify
#endif
@@ -490,8 +487,7 @@ tlb_huge_update_modify:
nopage_tlb_modify:
dbar 0
csrrd ra, EXCEPTION_KS2
- la_abs t0, tlb_do_page_fault_1
- jr t0
+ b tlb_do_page_fault_1
SYM_FUNC_END(handle_tlb_modify)
.endm
--
2.34.3
Hi, Jinyang
On 02/28/2023 04:02 PM, Jinyang He wrote:
> diff --git a/arch/loongarch/kernel/vmlinux.lds.S b/arch/loongarch/kernel/vmlinux.lds.S
> index ad2ce1a0908e..e99b50359900 100644
> --- a/arch/loongarch/kernel/vmlinux.lds.S
> +++ b/arch/loongarch/kernel/vmlinux.lds.S
> @@ -21,6 +21,11 @@
> */
> PECOFF_FILE_ALIGN = 0x200;
> PECOFF_SEGMENT_ALIGN = 0x10000;
> +EX_HANDLER_ALIGN = SZ_512;
> +EX_HANDLER_SIZE = SZ_512;
> +EX_HANDLER_TOTAL = 78; /* 64 Exceptions + 14 Interrupts */
You can define EXCCODE_NUM as 78 in loongarch.h, and remove the unused
EXCCODE_INT_END and EXCCODE_INT_NUM in PATCH4.
Youling.
> +EXCEPTION_ENTRY_ALIGN = SZ_64K;
> +TLBR_ENTRY_ALIGN = SZ_4K;
>
> OUTPUT_ARCH(loongarch)
> ENTRY(kernel_entry)
> @@ -54,6 +59,17 @@ SECTIONS
> . = ALIGN(PECOFF_SEGMENT_ALIGN);
> _etext = .;
>
> + . = ALIGN(EXCEPTION_ENTRY_ALIGN);
> + .ex_handlers : {
> + __ex_handlers = .;
> + *(SORT_BY_INIT_PRIORITY(.exception_handlers.*))
> + . = ALIGN(EX_HANDLER_ALIGN);
> + __ex_handlers_end = .;
> + . = ALIGN(TLBR_ENTRY_ALIGN);
> + __tlbr_entry = .;
> + *(.tlbrhandler)
> + }
> +
> /*
> * struct alt_inst entries. From the header (alternative.h):
> * "Alternative instructions for different CPU types or capabilities"
> @@ -154,3 +170,6 @@ SECTIONS
> *(.eh_frame)
> }
> }
> +
> +ASSERT(__ex_handlers_end == (__ex_handlers + EX_HANDLER_SIZE * EX_HANDLER_TOTAL),
> + "The amount of exception handlers does not match!");
On 02/28/2023 04:02 PM, Jinyang He wrote:
> diff --git a/arch/loongarch/mm/tlb.c b/arch/loongarch/mm/tlb.c
> index 8bad6b0cff59..dda35eae1c49 100644
> --- a/arch/loongarch/mm/tlb.c
> +++ b/arch/loongarch/mm/tlb.c
> @@ -250,50 +250,10 @@ static void output_pgtable_bits_defines(void)
> pr_debug("\n");
> }
>
> -#ifdef CONFIG_NUMA
> -unsigned long pcpu_handlers[NR_CPUS];
> -#endif
> -extern long exception_handlers[VECSIZE * 128 / sizeof(long)];
> -
> void setup_tlb_handler(int cpu)
It may be necessary to rename the function after modification.
Youling.
> {
> setup_ptwalker();
> local_flush_tlb_all();
> -
> - /* The tlb handlers are generated only once */
> - if (cpu == 0) {
> - memcpy((void *)tlbrentry, handle_tlb_refill, 0x80);
> - local_flush_icache_range(tlbrentry, tlbrentry + 0x80);
> - set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load, VECSIZE);
> - set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load, VECSIZE);
> - set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store, VECSIZE);
> - set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify, VECSIZE);
> - set_handler(EXCCODE_TLBNR * VECSIZE, handle_tlb_protect, VECSIZE);
> - set_handler(EXCCODE_TLBNX * VECSIZE, handle_tlb_protect, VECSIZE);
> - set_handler(EXCCODE_TLBPE * VECSIZE, handle_tlb_protect, VECSIZE);
> - }
> -#ifdef CONFIG_NUMA
> - else {
> - void *addr;
> - struct page *page;
> - const int vec_sz = sizeof(exception_handlers);
> -
> - if (pcpu_handlers[cpu])
> - return;
> -
> - page = alloc_pages_node(cpu_to_node(cpu), GFP_ATOMIC, get_order(vec_sz));
> - if (!page)
> - return;
> -
> - addr = page_address(page);
> - pcpu_handlers[cpu] = (unsigned long)addr;
> - memcpy((void *)addr, (void *)eentry, vec_sz);
> - local_flush_icache_range((unsigned long)addr, (unsigned long)addr + vec_sz);
> - csr_write64(pcpu_handlers[cpu], LOONGARCH_CSR_EENTRY);
> - csr_write64(pcpu_handlers[cpu], LOONGARCH_CSR_MERRENTRY);
> - csr_write64(pcpu_handlers[cpu] + 80*VECSIZE, LOONGARCH_CSR_TLBRENTRY);
> - }
> -#endif
> }
>