[Patch 1-2]: Fix build errors and simplify code.
[Patch 3-4]:
This series adds DYNAMIC_FTRACE_WITH_DIRECT_CALLS support for LoongArch.
SAMPLE_FTRACE_DIRECT and SAMPLE_FTRACE_DIRECT_MULTI are also included
here as the samples for testing DIRECT_CALLS related interface.
Part of the code refers to arm64 and riscv.
The following tests have been passed in my local 3A5000 machine.
- ftrace*.ko
- tools/testing/selftests/ftrace/
eg:
[loongson@linux linux]$ sudo insmod samples/ftrace/ftrace-direct-too.ko
[root@linux linux]# cat /sys/kernel/debug/tracing/trace | tail
rmmod-4552 [001] ..... 3166.654540: 0xffff800002094040: handle mm fault vma=00000000d0dbdb16 address=7ffff2a793e8 flags=254
rmmod-4552 [001] ..... 3166.654550: 0xffff800002094040: handle mm fault vma=0000000047694d8b address=555570700008 flags=255
rmmod-4552 [001] ..... 3166.654557: 0xffff800002094040: handle mm fault vma=00000000d0dbdb16 address=7ffff29e6eb0 flags=254
rmmod-4552 [001] ..... 3166.654561: 0xffff800002094040: handle mm fault vma=00000000d0dbdb16 address=7ffff297d480 flags=254
[Patch 5]: Abstract DYNAMIC_FTRACE_WITH_ARGS accesses.
Note: This patch depends on regs_set_return_value() in the patch
"LoongArch: Add support for function error injection".
Qing Zhang (1):
LoongArch: Abstract DYNAMIC_FTRACE_WITH_ARGS accesses
Youling Tang (4):
LoongArch: Fix build error if CONFIG_DYNAMIC_FTRACE_WITH_REGS is not
set
LoongArch: ftrace: Implement ftrace_find_callable_addr() to simplify
code
LoongArch: ftrace: Add direct call support
LoongArch: ftrace: Add direct call trampoline samples support
arch/loongarch/Kconfig | 3 +
arch/loongarch/include/asm/ftrace.h | 37 ++++++
arch/loongarch/kernel/ftrace_dyn.c | 132 +++++++++++---------
arch/loongarch/kernel/mcount_dyn.S | 7 +-
samples/ftrace/ftrace-direct-modify.c | 34 +++++
samples/ftrace/ftrace-direct-multi-modify.c | 41 ++++++
samples/ftrace/ftrace-direct-multi.c | 25 ++++
samples/ftrace/ftrace-direct-too.c | 27 ++++
samples/ftrace/ftrace-direct.c | 23 ++++
9 files changed, 266 insertions(+), 63 deletions(-)
--
2.37.1
We can see the following build error on LoongArch if CONFIG_DYNAMIC_FTRACE_WITH_REGS
is not set:
arch/loongarch/kernel/ftrace_dyn.c: In function ‘ftrace_make_call’:
arch/loongarch/kernel/ftrace_dyn.c:167:23: error: implicit declaration of function ‘__get_mod’
167 | ret = __get_mod(&mod, pc);
| ^~~~~~~~~
arch/loongarch/kernel/ftrace_dyn.c:171:24: error: implicit declaration of function ‘get_plt_addr’
171 | addr = get_plt_addr(mod, addr);
| ^~~~~~~~~~~~
The reason is that the __get_mod and get_plt_addr functions will be called
in ftrace_make_{call,nop}.
Signed-off-by: Youling Tang <[email protected]>
---
arch/loongarch/kernel/ftrace_dyn.c | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/arch/loongarch/kernel/ftrace_dyn.c b/arch/loongarch/kernel/ftrace_dyn.c
index 4a3ef8516ccc..c5f4b4681ddc 100644
--- a/arch/loongarch/kernel/ftrace_dyn.c
+++ b/arch/loongarch/kernel/ftrace_dyn.c
@@ -30,8 +30,6 @@ static int ftrace_modify_code(unsigned long pc, u32 old, u32 new, bool validate)
return 0;
}
-#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
-
#ifdef CONFIG_MODULES
static inline int __get_mod(struct module **mod, unsigned long addr)
{
@@ -72,6 +70,7 @@ static unsigned long get_plt_addr(struct module *mod, unsigned long addr)
}
#endif
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr)
{
u32 old, new;
@@ -102,7 +101,6 @@ int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned
return ftrace_modify_code(pc, old, new, true);
}
-
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
int ftrace_update_ftrace_func(ftrace_func_t func)
--
2.37.1
In the module processing part, the same code is reused by implementing
ftrace_find_callable_addr().
Signed-off-by: Youling Tang <[email protected]>
---
arch/loongarch/kernel/ftrace_dyn.c | 120 +++++++++++++++--------------
1 file changed, 61 insertions(+), 59 deletions(-)
diff --git a/arch/loongarch/kernel/ftrace_dyn.c b/arch/loongarch/kernel/ftrace_dyn.c
index c5f4b4681ddc..3cc4f8159f48 100644
--- a/arch/loongarch/kernel/ftrace_dyn.c
+++ b/arch/loongarch/kernel/ftrace_dyn.c
@@ -31,16 +31,11 @@ static int ftrace_modify_code(unsigned long pc, u32 old, u32 new, bool validate)
}
#ifdef CONFIG_MODULES
-static inline int __get_mod(struct module **mod, unsigned long addr)
+static bool reachable_by_bl(unsigned long addr, unsigned long pc)
{
- preempt_disable();
- *mod = __module_text_address(addr);
- preempt_enable();
+ long offset = (long)addr - (long)pc;
- if (WARN_ON(!(*mod)))
- return -EINVAL;
-
- return 0;
+ return offset >= -SZ_128M && offset < SZ_128M;
}
static struct plt_entry *get_ftrace_plt(struct module *mod, unsigned long addr)
@@ -56,18 +51,63 @@ static struct plt_entry *get_ftrace_plt(struct module *mod, unsigned long addr)
return NULL;
}
-static unsigned long get_plt_addr(struct module *mod, unsigned long addr)
+/*
+ * Find the address the callsite must branch to in order to reach '*addr'.
+ *
+ * Due to the limited range of 'bl' instructions, modules may be placed too far
+ * away to branch directly and must use a PLT.
+ *
+ * Returns true when '*addr' contains a reachable target address, or has been
+ * modified to contain a PLT address. Returns false otherwise.
+ */
+static bool ftrace_find_callable_addr(struct dyn_ftrace *rec, struct module *mod,
+ unsigned long *addr)
{
+ unsigned long pc = rec->ip + LOONGARCH_INSN_SIZE;
struct plt_entry *plt;
- plt = get_ftrace_plt(mod, addr);
+ /*
+ * When the target is within range of the 'bl' instruction, use 'addr'
+ * as-is and branch to that directly.
+ */
+ if (reachable_by_bl(*addr, pc))
+ return true;
+
+ /*
+ * 'mod' is only set at module load time, but if we end up
+ * dealing with an out-of-range condition, we can assume it
+ * is due to a module being loaded far away from the kernel.
+ *
+ * NOTE: __module_text_address() must be called with preemption
+ * disabled, but we can rely on ftrace_lock to ensure that 'mod'
+ * retains its validity throughout the remainder of this code.
+ */
+ if (!mod) {
+ preempt_disable();
+ mod = __module_text_address(pc);
+ preempt_enable();
+ }
+
+ if (WARN_ON(!mod))
+ return false;
+
+ plt = get_ftrace_plt(mod, *addr);
if (!plt) {
- pr_err("ftrace: no module PLT for %ps\n", (void *)addr);
- return -EINVAL;
+ pr_err("ftrace: no module PLT for %ps\n", (void *)*addr);
+ return false;
}
- return (unsigned long)plt;
+ *addr = (unsigned long)plt;
+ return true;
}
+
+#else /* !CONFIG_MODULES */
+static bool ftrace_find_callable_addr(struct dyn_ftrace *rec, struct module *mod,
+ unsigned long *addr)
+{
+ return true;
+}
+
#endif
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
@@ -75,26 +115,14 @@ int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned
{
u32 old, new;
unsigned long pc;
- long offset __maybe_unused;
pc = rec->ip + LOONGARCH_INSN_SIZE;
-#ifdef CONFIG_MODULES
- offset = (long)pc - (long)addr;
-
- if (offset < -SZ_128M || offset >= SZ_128M) {
- int ret;
- struct module *mod;
-
- ret = __get_mod(&mod, pc);
- if (ret)
- return ret;
-
- addr = get_plt_addr(mod, addr);
+ if (!ftrace_find_callable_addr(rec, NULL, &addr))
+ return -EINVAL;
- old_addr = get_plt_addr(mod, old_addr);
- }
-#endif
+ if (!ftrace_find_callable_addr(rec, NULL, &old_addr))
+ return -EINVAL;
new = larch_insn_gen_bl(pc, addr);
old = larch_insn_gen_bl(pc, old_addr);
@@ -151,24 +179,11 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
{
u32 old, new;
unsigned long pc;
- long offset __maybe_unused;
pc = rec->ip + LOONGARCH_INSN_SIZE;
-#ifdef CONFIG_MODULES
- offset = (long)pc - (long)addr;
-
- if (offset < -SZ_128M || offset >= SZ_128M) {
- int ret;
- struct module *mod;
-
- ret = __get_mod(&mod, pc);
- if (ret)
- return ret;
-
- addr = get_plt_addr(mod, addr);
- }
-#endif
+ if (!ftrace_find_callable_addr(rec, NULL, &addr))
+ return -EINVAL;
old = larch_insn_gen_nop();
new = larch_insn_gen_bl(pc, addr);
@@ -180,24 +195,11 @@ int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long ad
{
u32 old, new;
unsigned long pc;
- long offset __maybe_unused;
pc = rec->ip + LOONGARCH_INSN_SIZE;
-#ifdef CONFIG_MODULES
- offset = (long)pc - (long)addr;
-
- if (offset < -SZ_128M || offset >= SZ_128M) {
- int ret;
- struct module *mod;
-
- ret = __get_mod(&mod, pc);
- if (ret)
- return ret;
-
- addr = get_plt_addr(mod, addr);
- }
-#endif
+ if (!ftrace_find_callable_addr(rec, NULL, &addr))
+ return -EINVAL;
new = larch_insn_gen_nop();
old = larch_insn_gen_bl(pc, addr);
--
2.37.1
select the DYNAMIC_FTRACE_WITH_DIRECT_CALLS to provide the
register_ftrace_direct[_multi] interfaces allowing users to register
the customed trampoline (direct_caller) as the mcount for one or
more target functions. And modify_ftrace_direct[_multi] are also
provided for modifying direct_caller.
There are a few cases to distinguish:
- If a direct call ops is the only one tracing a function:
- If the direct called trampoline is within the reach of a 'bl'
instruction
-> the ftrace patchsite jumps to the trampoline
- Else
-> the ftrace patchsite jumps to the ftrace_regs_caller trampoline
which reads the ops pointer in the patchsite and jumps to the
direct call address stored in the ops
- Else
-> the ftrace patchsite jumps to the ftrace_regs_caller trampoline and
its ops literal points to ftrace_list_ops so it iterates over all
registered ftrace ops, including the direct call ops and calls its
call_direct_funcs handler which stores the direct called trampoline's
address in the ftrace_regs and the ftrace_regs_caller trampoline will
return to that address instead of returning to the traced function
Signed-off-by: Qing Zhang <[email protected]>
Signed-off-by: Youling Tang <[email protected]>
---
arch/loongarch/Kconfig | 1 +
arch/loongarch/include/asm/ftrace.h | 12 ++++++++++++
arch/loongarch/kernel/ftrace_dyn.c | 8 ++++++++
arch/loongarch/kernel/mcount_dyn.S | 7 ++++++-
4 files changed, 27 insertions(+), 1 deletion(-)
diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig
index 3ddde336e6a5..d6068a88d53f 100644
--- a/arch/loongarch/Kconfig
+++ b/arch/loongarch/Kconfig
@@ -92,6 +92,7 @@ config LOONGARCH
select HAVE_DMA_CONTIGUOUS
select HAVE_DYNAMIC_FTRACE
select HAVE_DYNAMIC_FTRACE_WITH_ARGS
+ select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
select HAVE_DYNAMIC_FTRACE_WITH_REGS
select HAVE_EBPF_JIT
select HAVE_EFFICIENT_UNALIGNED_ACCESS if !ARCH_STRICT_ALIGN
diff --git a/arch/loongarch/include/asm/ftrace.h b/arch/loongarch/include/asm/ftrace.h
index 3418d32d4fc7..f789e680f633 100644
--- a/arch/loongarch/include/asm/ftrace.h
+++ b/arch/loongarch/include/asm/ftrace.h
@@ -57,6 +57,18 @@ static __always_inline struct pt_regs *arch_ftrace_get_regs(struct ftrace_regs *
#define ftrace_graph_func ftrace_graph_func
void ftrace_graph_func(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *op, struct ftrace_regs *fregs);
+
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
+static inline void
+__arch_ftrace_set_direct_caller(struct pt_regs *regs, unsigned long addr)
+{
+ regs->regs[13] = addr; /* t1 */
+}
+
+#define arch_ftrace_set_direct_caller(fregs, addr) \
+ __arch_ftrace_set_direct_caller(&(fregs)->regs, addr)
+#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
+
#endif
#endif /* __ASSEMBLY__ */
diff --git a/arch/loongarch/kernel/ftrace_dyn.c b/arch/loongarch/kernel/ftrace_dyn.c
index 3cc4f8159f48..4062a7e63137 100644
--- a/arch/loongarch/kernel/ftrace_dyn.c
+++ b/arch/loongarch/kernel/ftrace_dyn.c
@@ -66,6 +66,14 @@ static bool ftrace_find_callable_addr(struct dyn_ftrace *rec, struct module *mod
unsigned long pc = rec->ip + LOONGARCH_INSN_SIZE;
struct plt_entry *plt;
+ /*
+ * If a custom trampoline is unreachable, rely on the ftrace_regs_caller
+ * trampoline which knows how to indirectly reach that trampoline
+ * through ops->direct_call.
+ */
+ if (*addr != FTRACE_ADDR && *addr != FTRACE_REGS_ADDR && !reachable_by_bl(*addr, pc))
+ *addr = FTRACE_REGS_ADDR;
+
/*
* When the target is within range of the 'bl' instruction, use 'addr'
* as-is and branch to that directly.
diff --git a/arch/loongarch/kernel/mcount_dyn.S b/arch/loongarch/kernel/mcount_dyn.S
index bbabf06244c2..1008fbc3cabc 100644
--- a/arch/loongarch/kernel/mcount_dyn.S
+++ b/arch/loongarch/kernel/mcount_dyn.S
@@ -42,7 +42,6 @@
.if \allregs
PTR_S tp, sp, PT_R2
PTR_S t0, sp, PT_R12
- PTR_S t1, sp, PT_R13
PTR_S t2, sp, PT_R14
PTR_S t3, sp, PT_R15
PTR_S t4, sp, PT_R16
@@ -64,6 +63,8 @@
PTR_S zero, sp, PT_R0
.endif
PTR_S ra, sp, PT_ERA /* Save trace function ra at PT_ERA */
+ move t1, zero
+ PTR_S t1, sp, PT_R13
PTR_ADDI t8, sp, PT_SIZE
PTR_S t8, sp, PT_R3
.endm
@@ -104,8 +105,12 @@ ftrace_common_return:
PTR_L a7, sp, PT_R11
PTR_L fp, sp, PT_R22
PTR_L t0, sp, PT_ERA
+ PTR_L t1, sp, PT_R13
PTR_ADDI sp, sp, PT_SIZE
+ bnez t1,.Ldirect
jr t0
+.Ldirect:
+ jr t1
SYM_CODE_END(ftrace_common)
SYM_CODE_START(ftrace_caller)
--
2.37.1
The ftrace samples need per-architecture trampoline implementations
to save and restore argument registers around the calls to
my_direct_func* and to restore polluted registers (eg: ra).
Signed-off-by: Qing Zhang <[email protected]>
Signed-off-by: Youling Tang <[email protected]>
---
arch/loongarch/Kconfig | 2 +
samples/ftrace/ftrace-direct-modify.c | 34 +++++++++++++++++
samples/ftrace/ftrace-direct-multi-modify.c | 41 +++++++++++++++++++++
samples/ftrace/ftrace-direct-multi.c | 25 +++++++++++++
samples/ftrace/ftrace-direct-too.c | 27 ++++++++++++++
samples/ftrace/ftrace-direct.c | 23 ++++++++++++
6 files changed, 152 insertions(+)
diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig
index d6068a88d53f..322c320fe57e 100644
--- a/arch/loongarch/Kconfig
+++ b/arch/loongarch/Kconfig
@@ -118,6 +118,8 @@ config LOONGARCH
select HAVE_PERF_USER_STACK_DUMP
select HAVE_REGS_AND_STACK_ACCESS_API
select HAVE_RSEQ
+ select HAVE_SAMPLE_FTRACE_DIRECT
+ select HAVE_SAMPLE_FTRACE_DIRECT_MULTI
select HAVE_SETUP_PER_CPU_AREA if NUMA
select HAVE_STACKPROTECTOR
select HAVE_SYSCALL_TRACEPOINTS
diff --git a/samples/ftrace/ftrace-direct-modify.c b/samples/ftrace/ftrace-direct-modify.c
index d93abbcb1f4c..ca72c3b710eb 100644
--- a/samples/ftrace/ftrace-direct-modify.c
+++ b/samples/ftrace/ftrace-direct-modify.c
@@ -96,6 +96,40 @@ asm (
#endif /* CONFIG_S390 */
+#ifdef CONFIG_LOONGARCH
+
+asm (
+" .pushsection .text, \"ax\", @progbits\n"
+" .type my_tramp1, @function\n"
+" .globl my_tramp1\n"
+" my_tramp1:\n"
+" addi.d $sp, $sp, -16\n"
+" st.d $t0, $sp, 0\n"
+" st.d $ra, $sp, 8\n"
+" bl my_direct_func1\n"
+" ld.d $t0, $sp, 0\n"
+" ld.d $ra, $sp, 8\n"
+" addi.d $sp, $sp, 16\n"
+" jr $t0\n"
+" .size my_tramp1, .-my_tramp1\n"
+
+" .type my_tramp2, @function\n"
+" .globl my_tramp2\n"
+" my_tramp2:\n"
+" addi.d $sp, $sp, -16\n"
+" st.d $t0, $sp, 0\n"
+" st.d $ra, $sp, 8\n"
+" bl my_direct_func2\n"
+" ld.d $t0, $sp, 0\n"
+" ld.d $ra, $sp, 8\n"
+" addi.d $sp, $sp, 16\n"
+" jr $t0\n"
+" .size my_tramp2, .-my_tramp2\n"
+" .popsection\n"
+);
+
+#endif /* CONFIG_LOONGARCH */
+
static unsigned long my_tramp = (unsigned long)my_tramp1;
static unsigned long tramps[2] = {
(unsigned long)my_tramp1,
diff --git a/samples/ftrace/ftrace-direct-multi-modify.c b/samples/ftrace/ftrace-direct-multi-modify.c
index b58c594efb51..4708c24d47c6 100644
--- a/samples/ftrace/ftrace-direct-multi-modify.c
+++ b/samples/ftrace/ftrace-direct-multi-modify.c
@@ -103,6 +103,47 @@ asm (
#endif /* CONFIG_S390 */
+#ifdef CONFIG_LOONGARCH
+#include <asm/asm.h>
+
+asm (
+" .pushsection .text, \"ax\", @progbits\n"
+" .type my_tramp1, @function\n"
+" .globl my_tramp1\n"
+" my_tramp1:\n"
+" addi.d $sp, $sp, -32\n"
+" st.d $a0, $sp, 0\n"
+" st.d $t0, $sp, 8\n"
+" st.d $ra, $sp, 16\n"
+" move $a0, $t0\n"
+" bl my_direct_func1\n"
+" ld.d $a0, $sp, 0\n"
+" ld.d $t0, $sp, 8\n"
+" ld.d $ra, $sp, 16\n"
+" addi.d $sp, $sp, 32\n"
+" jr $t0\n"
+" .size my_tramp1, .-my_tramp1\n"
+
+" .type my_tramp2, @function\n"
+" .globl my_tramp2\n"
+" my_tramp2:\n"
+" addi.d $sp, $sp, -32\n"
+" st.d $a0, $sp, 0\n"
+" st.d $t0, $sp, 8\n"
+" st.d $ra, $sp, 16\n"
+" move $a0, $t0\n"
+" bl my_direct_func2\n"
+" ld.d $a0, $sp, 0\n"
+" ld.d $t0, $sp, 8\n"
+" ld.d $ra, $sp, 16\n"
+" addi.d $sp, $sp, 32\n"
+" jr $t0\n"
+" .size my_tramp2, .-my_tramp2\n"
+" .popsection\n"
+);
+
+#endif /* CONFIG_LOONGARCH */
+
static unsigned long my_tramp = (unsigned long)my_tramp1;
static unsigned long tramps[2] = {
(unsigned long)my_tramp1,
diff --git a/samples/ftrace/ftrace-direct-multi.c b/samples/ftrace/ftrace-direct-multi.c
index c27cf130c319..c2f1652c67bc 100644
--- a/samples/ftrace/ftrace-direct-multi.c
+++ b/samples/ftrace/ftrace-direct-multi.c
@@ -66,6 +66,31 @@ asm (
#endif /* CONFIG_S390 */
+#ifdef CONFIG_LOONGARCH
+
+#include <asm/asm.h>
+asm (
+" .pushsection .text, \"ax\", @progbits\n"
+" .type my_tramp, @function\n"
+" .globl my_tramp\n"
+" my_tramp:\n"
+" addi.d $sp, $sp, -32\n"
+" st.d $a0, $sp, 0\n"
+" st.d $t0, $sp, 8\n"
+" st.d $ra, $sp, 16\n"
+" move $a0, $t0\n"
+" bl my_direct_func\n"
+" ld.d $a0, $sp, 0\n"
+" ld.d $t0, $sp, 8\n"
+" ld.d $ra, $sp, 16\n"
+" addi.d $sp, $sp, 32\n"
+" jr $t0\n"
+" .size my_tramp, .-my_tramp\n"
+" .popsection\n"
+);
+
+#endif /* CONFIG_LOONGARCH */
+
static struct ftrace_ops direct;
static int __init ftrace_direct_multi_init(void)
diff --git a/samples/ftrace/ftrace-direct-too.c b/samples/ftrace/ftrace-direct-too.c
index 8139dce2a31c..ef64d7509773 100644
--- a/samples/ftrace/ftrace-direct-too.c
+++ b/samples/ftrace/ftrace-direct-too.c
@@ -70,6 +70,33 @@ asm (
#endif /* CONFIG_S390 */
+#ifdef CONFIG_LOONGARCH
+
+asm (
+" .pushsection .text, \"ax\", @progbits\n"
+" .type my_tramp, @function\n"
+" .globl my_tramp\n"
+" my_tramp:\n"
+" addi.d $sp, $sp, -48\n"
+" st.d $a0, $sp, 0\n"
+" st.d $a1, $sp, 8\n"
+" st.d $a2, $sp, 16\n"
+" st.d $t0, $sp, 24\n"
+" st.d $ra, $sp, 32\n"
+" bl my_direct_func\n"
+" ld.d $a0, $sp, 0\n"
+" ld.d $a1, $sp, 8\n"
+" ld.d $a2, $sp, 16\n"
+" ld.d $t0, $sp, 24\n"
+" ld.d $ra, $sp, 32\n"
+" addi.d $sp, $sp, 48\n"
+" jr $t0\n"
+" .size my_tramp, .-my_tramp\n"
+" .popsection\n"
+);
+
+#endif /* CONFIG_LOONGARCH */
+
static int __init ftrace_direct_init(void)
{
return register_ftrace_direct((unsigned long)handle_mm_fault,
diff --git a/samples/ftrace/ftrace-direct.c b/samples/ftrace/ftrace-direct.c
index 1d3d307ca33d..9be720957bf8 100644
--- a/samples/ftrace/ftrace-direct.c
+++ b/samples/ftrace/ftrace-direct.c
@@ -63,6 +63,29 @@ asm (
#endif /* CONFIG_S390 */
+#ifdef CONFIG_LOONGARCH
+
+asm (
+" .pushsection .text, \"ax\", @progbits\n"
+" .type my_tramp, @function\n"
+" .globl my_tramp\n"
+" my_tramp:\n"
+" addi.d $sp, $sp, -32\n"
+" st.d $a0, $sp, 0\n"
+" st.d $t0, $sp, 8\n"
+" st.d $ra, $sp, 16\n"
+" bl my_direct_func\n"
+" ld.d $a0, $sp, 0\n"
+" ld.d $t0, $sp, 8\n"
+" ld.d $ra, $sp, 16\n"
+" addi.d $sp, $sp, 32\n"
+" jr $t0\n"
+" .size my_tramp, .-my_tramp\n"
+" .popsection\n"
+);
+
+#endif /* CONFIG_LOONGARCH */
+
static int __init ftrace_direct_init(void)
{
return register_ftrace_direct((unsigned long)wake_up_process,
--
2.37.1
From: Qing Zhang <[email protected]>
1.Adds new ftrace_regs_{get,set}_*() helpers which can be used to manipulate
ftrace_regs. When CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS=y, these can always
be used on any ftrace_regs, and when CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS=n
these can be used when regs are available. A new ftrace_regs_has_args(fregs)
helper is added which code can use to check when these are usable.
2.Prepare ftrace_regs_set_instruction_pointer support in advance.
Signed-off-by: Qing Zhang <[email protected]>
---
arch/loongarch/include/asm/ftrace.h | 25 +++++++++++++++++++++++++
1 file changed, 25 insertions(+)
diff --git a/arch/loongarch/include/asm/ftrace.h b/arch/loongarch/include/asm/ftrace.h
index f789e680f633..30ee01243416 100644
--- a/arch/loongarch/include/asm/ftrace.h
+++ b/arch/loongarch/include/asm/ftrace.h
@@ -54,6 +54,31 @@ static __always_inline struct pt_regs *arch_ftrace_get_regs(struct ftrace_regs *
return &fregs->regs;
}
+static __always_inline void
+ftrace_regs_set_instruction_pointer(struct ftrace_regs *fregs,
+ unsigned long ip)
+{
+ regs_set_return_value(&fregs->regs, ip);
+}
+
+static __always_inline unsigned long
+ftrace_regs_get_instruction_pointer(struct ftrace_regs *fregs)
+{
+ return instruction_pointer(&fregs->regs);
+}
+
+#define ftrace_regs_get_argument(fregs, n) \
+ regs_get_kernel_argument(&(fregs)->regs, n)
+#define ftrace_regs_get_stack_pointer(fregs) \
+ kernel_stack_pointer(&(fregs)->regs)
+#define ftrace_regs_return_value(fregs) \
+ regs_return_value(&(fregs)->regs)
+#define ftrace_regs_set_return_value(fregs, ret) \
+ regs_set_return_value(&(fregs)->regs, ret)
+#define ftrace_override_function_with_return(fregs) \
+ override_function_with_return(&(fregs)->regs)
+#define ftrace_regs_query_register_offset(name) \
+
#define ftrace_graph_func ftrace_graph_func
void ftrace_graph_func(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *op, struct ftrace_regs *fregs);
--
2.37.1
Hi,
On Wed, Apr 26, 2023 at 09:49:12AM +0800, Youling Tang wrote:
> select the DYNAMIC_FTRACE_WITH_DIRECT_CALLS to provide the
> register_ftrace_direct[_multi] interfaces allowing users to register
> the customed trampoline (direct_caller) as the mcount for one or
> more target functions. And modify_ftrace_direct[_multi] are also
> provided for modifying direct_caller.
>
> There are a few cases to distinguish:
> - If a direct call ops is the only one tracing a function:
> - If the direct called trampoline is within the reach of a 'bl'
> instruction
> -> the ftrace patchsite jumps to the trampoline
> - Else
> -> the ftrace patchsite jumps to the ftrace_regs_caller trampoline
> which reads the ops pointer in the patchsite and jumps to the
> direct call address stored in the ops
I think you forgot to update this wording; there's no ops pointer in the
patchsite as you don't implement DYNAMIC_FTRACE_WITH_CALL_OPS.
I think you can delete the "Else" case here, and replace the above with:
- If a direct call ops is the only one tracing a function AND the direct called
trampoline is within the reach of a 'bl' instruction
-> the ftrace patchsite jumps to the trampoline
> - Else
> -> the ftrace patchsite jumps to the ftrace_regs_caller trampoline and
> its ops literal points to ftrace_list_ops so it iterates over all
> registered ftrace ops, including the direct call ops and calls its
> call_direct_funcs handler which stores the direct called trampoline's
> address in the ftrace_regs and the ftrace_regs_caller trampoline will
Likewise here, there's no "ops literal" associated with the patchsite.
Otherwise, this looks sane to me.
Mark.
> return to that address instead of returning to the traced function
>
> Signed-off-by: Qing Zhang <[email protected]>
> Signed-off-by: Youling Tang <[email protected]>
> ---
> arch/loongarch/Kconfig | 1 +
> arch/loongarch/include/asm/ftrace.h | 12 ++++++++++++
> arch/loongarch/kernel/ftrace_dyn.c | 8 ++++++++
> arch/loongarch/kernel/mcount_dyn.S | 7 ++++++-
> 4 files changed, 27 insertions(+), 1 deletion(-)
>
> diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig
> index 3ddde336e6a5..d6068a88d53f 100644
> --- a/arch/loongarch/Kconfig
> +++ b/arch/loongarch/Kconfig
> @@ -92,6 +92,7 @@ config LOONGARCH
> select HAVE_DMA_CONTIGUOUS
> select HAVE_DYNAMIC_FTRACE
> select HAVE_DYNAMIC_FTRACE_WITH_ARGS
> + select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
> select HAVE_DYNAMIC_FTRACE_WITH_REGS
> select HAVE_EBPF_JIT
> select HAVE_EFFICIENT_UNALIGNED_ACCESS if !ARCH_STRICT_ALIGN
> diff --git a/arch/loongarch/include/asm/ftrace.h b/arch/loongarch/include/asm/ftrace.h
> index 3418d32d4fc7..f789e680f633 100644
> --- a/arch/loongarch/include/asm/ftrace.h
> +++ b/arch/loongarch/include/asm/ftrace.h
> @@ -57,6 +57,18 @@ static __always_inline struct pt_regs *arch_ftrace_get_regs(struct ftrace_regs *
> #define ftrace_graph_func ftrace_graph_func
> void ftrace_graph_func(unsigned long ip, unsigned long parent_ip,
> struct ftrace_ops *op, struct ftrace_regs *fregs);
> +
> +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
> +static inline void
> +__arch_ftrace_set_direct_caller(struct pt_regs *regs, unsigned long addr)
> +{
> + regs->regs[13] = addr; /* t1 */
> +}
> +
> +#define arch_ftrace_set_direct_caller(fregs, addr) \
> + __arch_ftrace_set_direct_caller(&(fregs)->regs, addr)
> +#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
> +
> #endif
>
> #endif /* __ASSEMBLY__ */
> diff --git a/arch/loongarch/kernel/ftrace_dyn.c b/arch/loongarch/kernel/ftrace_dyn.c
> index 3cc4f8159f48..4062a7e63137 100644
> --- a/arch/loongarch/kernel/ftrace_dyn.c
> +++ b/arch/loongarch/kernel/ftrace_dyn.c
> @@ -66,6 +66,14 @@ static bool ftrace_find_callable_addr(struct dyn_ftrace *rec, struct module *mod
> unsigned long pc = rec->ip + LOONGARCH_INSN_SIZE;
> struct plt_entry *plt;
>
> + /*
> + * If a custom trampoline is unreachable, rely on the ftrace_regs_caller
> + * trampoline which knows how to indirectly reach that trampoline
> + * through ops->direct_call.
> + */
> + if (*addr != FTRACE_ADDR && *addr != FTRACE_REGS_ADDR && !reachable_by_bl(*addr, pc))
> + *addr = FTRACE_REGS_ADDR;
> +
> /*
> * When the target is within range of the 'bl' instruction, use 'addr'
> * as-is and branch to that directly.
> diff --git a/arch/loongarch/kernel/mcount_dyn.S b/arch/loongarch/kernel/mcount_dyn.S
> index bbabf06244c2..1008fbc3cabc 100644
> --- a/arch/loongarch/kernel/mcount_dyn.S
> +++ b/arch/loongarch/kernel/mcount_dyn.S
> @@ -42,7 +42,6 @@
> .if \allregs
> PTR_S tp, sp, PT_R2
> PTR_S t0, sp, PT_R12
> - PTR_S t1, sp, PT_R13
> PTR_S t2, sp, PT_R14
> PTR_S t3, sp, PT_R15
> PTR_S t4, sp, PT_R16
> @@ -64,6 +63,8 @@
> PTR_S zero, sp, PT_R0
> .endif
> PTR_S ra, sp, PT_ERA /* Save trace function ra at PT_ERA */
> + move t1, zero
> + PTR_S t1, sp, PT_R13
> PTR_ADDI t8, sp, PT_SIZE
> PTR_S t8, sp, PT_R3
> .endm
> @@ -104,8 +105,12 @@ ftrace_common_return:
> PTR_L a7, sp, PT_R11
> PTR_L fp, sp, PT_R22
> PTR_L t0, sp, PT_ERA
> + PTR_L t1, sp, PT_R13
> PTR_ADDI sp, sp, PT_SIZE
> + bnez t1,.Ldirect
> jr t0
> +.Ldirect:
> + jr t1
> SYM_CODE_END(ftrace_common)
>
> SYM_CODE_START(ftrace_caller)
> --
> 2.37.1
>
Hi, Mark
On 2023/4/26 21:43, Mark Rutland wrote:
> Hi,
>
> On Wed, Apr 26, 2023 at 09:49:12AM +0800, Youling Tang wrote:
>> select the DYNAMIC_FTRACE_WITH_DIRECT_CALLS to provide the
>> register_ftrace_direct[_multi] interfaces allowing users to register
>> the customed trampoline (direct_caller) as the mcount for one or
>> more target functions. And modify_ftrace_direct[_multi] are also
>> provided for modifying direct_caller.
>>
>> There are a few cases to distinguish:
>> - If a direct call ops is the only one tracing a function:
>> - If the direct called trampoline is within the reach of a 'bl'
>> instruction
>> -> the ftrace patchsite jumps to the trampoline
>> - Else
>> -> the ftrace patchsite jumps to the ftrace_regs_caller trampoline
>> which reads the ops pointer in the patchsite and jumps to the
>> direct call address stored in the ops
> I think you forgot to update this wording; there's no ops pointer in the
> patchsite as you don't implement DYNAMIC_FTRACE_WITH_CALL_OPS.
>
> I think you can delete the "Else" case here, and replace the above with:
>
> - If a direct call ops is the only one tracing a function AND the direct called
> trampoline is within the reach of a 'bl' instruction
> -> the ftrace patchsite jumps to the trampoline
>
>> - Else
>> -> the ftrace patchsite jumps to the ftrace_regs_caller trampoline and
>> its ops literal points to ftrace_list_ops so it iterates over all
>> registered ftrace ops, including the direct call ops and calls its
>> call_direct_funcs handler which stores the direct called trampoline's
>> address in the ftrace_regs and the ftrace_regs_caller trampoline will
> Likewise here, there's no "ops literal" associated with the patchsite.
Yes, thank you for pointing out, I will fix this description.
Thanks,
Youling.
>
> Otherwise, this looks sane to me.
>
> Mark.
>
>> return to that address instead of returning to the traced function
>>
>> Signed-off-by: Qing Zhang <[email protected]>
>> Signed-off-by: Youling Tang <[email protected]>
>> ---
>> arch/loongarch/Kconfig | 1 +
>> arch/loongarch/include/asm/ftrace.h | 12 ++++++++++++
>> arch/loongarch/kernel/ftrace_dyn.c | 8 ++++++++
>> arch/loongarch/kernel/mcount_dyn.S | 7 ++++++-
>> 4 files changed, 27 insertions(+), 1 deletion(-)
>>
>> diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig
>> index 3ddde336e6a5..d6068a88d53f 100644
>> --- a/arch/loongarch/Kconfig
>> +++ b/arch/loongarch/Kconfig
>> @@ -92,6 +92,7 @@ config LOONGARCH
>> select HAVE_DMA_CONTIGUOUS
>> select HAVE_DYNAMIC_FTRACE
>> select HAVE_DYNAMIC_FTRACE_WITH_ARGS
>> + select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
>> select HAVE_DYNAMIC_FTRACE_WITH_REGS
>> select HAVE_EBPF_JIT
>> select HAVE_EFFICIENT_UNALIGNED_ACCESS if !ARCH_STRICT_ALIGN
>> diff --git a/arch/loongarch/include/asm/ftrace.h b/arch/loongarch/include/asm/ftrace.h
>> index 3418d32d4fc7..f789e680f633 100644
>> --- a/arch/loongarch/include/asm/ftrace.h
>> +++ b/arch/loongarch/include/asm/ftrace.h
>> @@ -57,6 +57,18 @@ static __always_inline struct pt_regs *arch_ftrace_get_regs(struct ftrace_regs *
>> #define ftrace_graph_func ftrace_graph_func
>> void ftrace_graph_func(unsigned long ip, unsigned long parent_ip,
>> struct ftrace_ops *op, struct ftrace_regs *fregs);
>> +
>> +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
>> +static inline void
>> +__arch_ftrace_set_direct_caller(struct pt_regs *regs, unsigned long addr)
>> +{
>> + regs->regs[13] = addr; /* t1 */
>> +}
>> +
>> +#define arch_ftrace_set_direct_caller(fregs, addr) \
>> + __arch_ftrace_set_direct_caller(&(fregs)->regs, addr)
>> +#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
>> +
>> #endif
>>
>> #endif /* __ASSEMBLY__ */
>> diff --git a/arch/loongarch/kernel/ftrace_dyn.c b/arch/loongarch/kernel/ftrace_dyn.c
>> index 3cc4f8159f48..4062a7e63137 100644
>> --- a/arch/loongarch/kernel/ftrace_dyn.c
>> +++ b/arch/loongarch/kernel/ftrace_dyn.c
>> @@ -66,6 +66,14 @@ static bool ftrace_find_callable_addr(struct dyn_ftrace *rec, struct module *mod
>> unsigned long pc = rec->ip + LOONGARCH_INSN_SIZE;
>> struct plt_entry *plt;
>>
>> + /*
>> + * If a custom trampoline is unreachable, rely on the ftrace_regs_caller
>> + * trampoline which knows how to indirectly reach that trampoline
>> + * through ops->direct_call.
>> + */
>> + if (*addr != FTRACE_ADDR && *addr != FTRACE_REGS_ADDR && !reachable_by_bl(*addr, pc))
>> + *addr = FTRACE_REGS_ADDR;
>> +
>> /*
>> * When the target is within range of the 'bl' instruction, use 'addr'
>> * as-is and branch to that directly.
>> diff --git a/arch/loongarch/kernel/mcount_dyn.S b/arch/loongarch/kernel/mcount_dyn.S
>> index bbabf06244c2..1008fbc3cabc 100644
>> --- a/arch/loongarch/kernel/mcount_dyn.S
>> +++ b/arch/loongarch/kernel/mcount_dyn.S
>> @@ -42,7 +42,6 @@
>> .if \allregs
>> PTR_S tp, sp, PT_R2
>> PTR_S t0, sp, PT_R12
>> - PTR_S t1, sp, PT_R13
>> PTR_S t2, sp, PT_R14
>> PTR_S t3, sp, PT_R15
>> PTR_S t4, sp, PT_R16
>> @@ -64,6 +63,8 @@
>> PTR_S zero, sp, PT_R0
>> .endif
>> PTR_S ra, sp, PT_ERA /* Save trace function ra at PT_ERA */
>> + move t1, zero
>> + PTR_S t1, sp, PT_R13
>> PTR_ADDI t8, sp, PT_SIZE
>> PTR_S t8, sp, PT_R3
>> .endm
>> @@ -104,8 +105,12 @@ ftrace_common_return:
>> PTR_L a7, sp, PT_R11
>> PTR_L fp, sp, PT_R22
>> PTR_L t0, sp, PT_ERA
>> + PTR_L t1, sp, PT_R13
>> PTR_ADDI sp, sp, PT_SIZE
>> + bnez t1,.Ldirect
>> jr t0
>> +.Ldirect:
>> + jr t1
>> SYM_CODE_END(ftrace_common)
>>
>> SYM_CODE_START(ftrace_caller)
>> --
>> 2.37.1
>>