2019-08-23 10:03:00

by Mao Han

[permalink] [raw]
Subject: [PATCH V5 1/3] riscv: Add perf callchain support

This patch add support for perf callchain sampling on riscv platform.
The return address of leaf function is retrieved from pt_regs as
it is not saved in the outmost frame.

Signed-off-by: Mao Han <[email protected]>
Cc: Paul Walmsley <[email protected]>
Cc: Greentime Hu <[email protected]>
Cc: Palmer Dabbelt <[email protected]>
Cc: linux-riscv <[email protected]>
Cc: Christoph Hellwig <[email protected]>
Cc: Guo Ren <[email protected]>
---
arch/riscv/Makefile | 3 +
arch/riscv/kernel/Makefile | 3 +-
arch/riscv/kernel/perf_callchain.c | 115 +++++++++++++++++++++++++++++++++++++
3 files changed, 120 insertions(+), 1 deletion(-)
create mode 100644 arch/riscv/kernel/perf_callchain.c

diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile
index 7a117be..946565b 100644
--- a/arch/riscv/Makefile
+++ b/arch/riscv/Makefile
@@ -54,6 +54,9 @@ endif
ifeq ($(CONFIG_MODULE_SECTIONS),y)
KBUILD_LDFLAGS_MODULE += -T $(srctree)/arch/riscv/kernel/module.lds
endif
+ifeq ($(CONFIG_PERF_EVENTS),y)
+ KBUILD_CFLAGS += -fno-omit-frame-pointer
+endif

KBUILD_CFLAGS_MODULE += $(call cc-option,-mno-relax)

diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile
index 2420d37..b1bea89 100644
--- a/arch/riscv/kernel/Makefile
+++ b/arch/riscv/kernel/Makefile
@@ -38,6 +38,7 @@ obj-$(CONFIG_MODULE_SECTIONS) += module-sections.o
obj-$(CONFIG_FUNCTION_TRACER) += mcount.o ftrace.o
obj-$(CONFIG_DYNAMIC_FTRACE) += mcount-dyn.o

-obj-$(CONFIG_PERF_EVENTS) += perf_event.o
+obj-$(CONFIG_PERF_EVENTS) += perf_event.o
+obj-$(CONFIG_PERF_EVENTS) += perf_callchain.o

clean:
diff --git a/arch/riscv/kernel/perf_callchain.c b/arch/riscv/kernel/perf_callchain.c
new file mode 100644
index 0000000..d75d15c
--- /dev/null
+++ b/arch/riscv/kernel/perf_callchain.c
@@ -0,0 +1,115 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (C) 2019 Hangzhou C-SKY Microsystems co.,ltd. */
+
+#include <linux/perf_event.h>
+#include <linux/uaccess.h>
+
+/* Kernel callchain */
+struct stackframe {
+ unsigned long fp;
+ unsigned long ra;
+};
+
+static int unwind_frame_kernel(struct stackframe *frame)
+{
+ if (kstack_end((void *)frame->fp))
+ return -EPERM;
+ if (frame->fp & 0x3 || frame->fp < TASK_SIZE)
+ return -EPERM;
+ if (frame->fp < CONFIG_PAGE_OFFSET)
+ return -EPERM;
+
+ *frame = *((struct stackframe *)frame->fp - 1);
+ if (__kernel_text_address(frame->ra)) {
+ int graph = 0;
+
+ frame->ra = ftrace_graph_ret_addr(NULL, &graph, frame->ra,
+ NULL);
+ }
+ return 0;
+}
+
+static void notrace walk_stackframe(struct stackframe *fr,
+ struct perf_callchain_entry_ctx *entry)
+{
+ do {
+ perf_callchain_store(entry, fr->ra);
+ } while (unwind_frame_kernel(fr) >= 0);
+}
+
+/*
+ * Get the return address for a single stackframe and return a pointer to the
+ * next frame tail.
+ */
+static unsigned long user_backtrace(struct perf_callchain_entry_ctx *entry,
+ unsigned long fp, unsigned long reg_ra)
+{
+ struct stackframe buftail;
+ unsigned long ra = 0;
+ unsigned long *user_frame_tail =
+ (unsigned long *)(fp - sizeof(struct stackframe));
+
+ /* Check accessibility of one struct frame_tail beyond */
+ if (!access_ok(user_frame_tail, sizeof(buftail)))
+ return 0;
+ if (__copy_from_user_inatomic(&buftail, user_frame_tail,
+ sizeof(buftail)))
+ return 0;
+
+ if (reg_ra != 0)
+ ra = reg_ra;
+ else
+ ra = buftail.ra;
+
+ fp = buftail.fp;
+ perf_callchain_store(entry, ra);
+
+ return fp;
+}
+
+/*
+ * This will be called when the target is in user mode
+ * This function will only be called when we use
+ * "PERF_SAMPLE_CALLCHAIN" in
+ * kernel/events/core.c:perf_prepare_sample()
+ *
+ * How to trigger perf_callchain_[user/kernel] :
+ * $ perf record -e cpu-clock --call-graph fp ./program
+ * $ perf report --call-graph
+ *
+ * On RISC-V platform, the program being sampled and the C library
+ * need to be compiled with -fno-omit-frame-pointer, otherwise
+ * the user stack will not contain function frame.
+ */
+void perf_callchain_user(struct perf_callchain_entry_ctx *entry,
+ struct pt_regs *regs)
+{
+ unsigned long fp = 0;
+
+ /* RISC-V does not support virtualization. */
+ if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
+ return;
+
+ fp = regs->s0;
+ perf_callchain_store(entry, regs->sepc);
+
+ fp = user_backtrace(entry, fp, regs->ra);
+ while (fp && !(fp & 0x3) && entry->nr < entry->max_stack)
+ fp = user_backtrace(entry, fp, 0);
+}
+
+void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
+ struct pt_regs *regs)
+{
+ struct stackframe fr;
+
+ /* RISC-V does not support virtualization. */
+ if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
+ pr_warn("RISC-V does not support perf in guest mode!");
+ return;
+ }
+
+ fr.fp = regs->s0;
+ fr.ra = regs->ra;
+ walk_stackframe(&fr, entry);
+}
--
2.7.4


2019-08-23 11:05:14

by Greentime Hu

[permalink] [raw]
Subject: Re: [PATCH V5 1/3] riscv: Add perf callchain support

Hi Mao,

Mao Han <[email protected]> 於 2019年8月23日 週五 下午2:16寫道:

>
> This patch add support for perf callchain sampling on riscv platform.
> The return address of leaf function is retrieved from pt_regs as
> it is not saved in the outmost frame.
>
> Signed-off-by: Mao Han <[email protected]>
> Cc: Paul Walmsley <[email protected]>
> Cc: Greentime Hu <[email protected]>
> Cc: Palmer Dabbelt <[email protected]>
> Cc: linux-riscv <[email protected]>
> Cc: Christoph Hellwig <[email protected]>
> Cc: Guo Ren <[email protected]>
> ---
> arch/riscv/Makefile | 3 +
> arch/riscv/kernel/Makefile | 3 +-
> arch/riscv/kernel/perf_callchain.c | 115 +++++++++++++++++++++++++++++++++++++
> 3 files changed, 120 insertions(+), 1 deletion(-)
> create mode 100644 arch/riscv/kernel/perf_callchain.c

I just tested "./perf record -e cpu-clock --call-graph fp ls" on
Unleashed board and I got this failure.
I take a look at it. It seem failed in here. Do you have any idea?
It seems fine in Qemu.

1 *frame = *((struct stackframe *)frame->fp - 1);
ffffffe0001a198c: 00863a83 ld s5,8(a2)
ffffffe0001a1990: ff093903 ld s2,-16(s2)

./perf record -e cpu-clock --call-graph fp ls
[ 9619.423884] hrtimer: interrupt took 733000 ns
[ 9619.977017] Unable to handle kernel paging request at virtual
address ffffffffffffff94
[ 9620.214736] Oops [#1]
[ 9620.289893] Modules linked in:
[ 9620.391378] CPU: 0 PID: 264 Comm: ls Not tainted
5.3.0-rc5-00003-gb008f6bcd67c #4
[ 9620.640176] sepc: ffffffe0001a198c ra : ffffffe0001a199a sp :
ffffffe000093720
[ 9620.880366] gp : ffffffe00097dad8 tp : ffffffe000082e40 t0 : 0000000000046000
[ 9621.120564] t1 : 0000000000000002 t2 : 0000000000000007 s0 : ffffffe000093760
[ 9621.360768] s1 : ffffffe000093788 a0 : 0000000000000003 a1 : 0000000000000000
[ 9621.600991] a2 : ffffffffffffff8c a3 : 0000000000001fa0 a4 : 0000000000000010
[ 9621.841181] a5 : 0000000000000002 a6 : 0000000000000001 a7 : ffffffe079b34e10
[ 9622.081400] s2 : ffffffffffffff9c s3 : ffffffe000000000 s4 : 0000000000001ff8
[ 9622.321618] s5 : ffffffe000093da0 s6 : ffffffe00097d540 s7 : ffffffe07a1517a0
[ 9622.561811] s8 : 000008bf01c7ff60 s9 : ffffffe000235b2a s10: 0000000200000120
[ 9622.802015] s11: 0000000000000001 t3 : ffffffe079b34e00 t4 : 0000000000000001
[ 9623.042194] t5 : 0000000000000008 t6 : ffffffe0009208d0
[ 9623.218785] sstatus: 0000000200000100 sbadaddr: ffffffffffffff94
scause: 000000000000000d
[ 9623.490850] ---[ end trace 49043f28e856d84d ]---
[ 9623.644217] Kernel panic - not syncing: Fatal exception in interrupt
[ 9623.855470] SMP: stopping secondary CPUs
[ 9623.985955] ---[ end Kernel panic - not syncing: Fatal exception in
interrupt ]---

2019-08-24 00:58:46

by Guo Ren

[permalink] [raw]
Subject: Re: [PATCH V5 1/3] riscv: Add perf callchain support

Please check CONFIG_FRAME_POINTER

1 *frame = *((struct stackframe *)frame->fp - 1);
This code is origionally from riscv/kernel/stacktrace.c: walk_stackframe

In linux/Makefile it'll involve the options for gcc to definitely
store ra & prev_fp in fp pointer.
ifdef CONFIG_FRAME_POINTER
KBUILD_CFLAGS += -fno-omit-frame-pointer -fno-optimize-sibling-calls

So --call-graph fp need depends on CONFIG_FRAME_POINTER.

On Fri, Aug 23, 2019 at 4:56 PM Greentime Hu <[email protected]> wrote:
>
> Hi Mao,
>
> Mao Han <[email protected]> 於 2019年8月23日 週五 下午2:16寫道:
>
> >
> > This patch add support for perf callchain sampling on riscv platform.
> > The return address of leaf function is retrieved from pt_regs as
> > it is not saved in the outmost frame.
> >
> > Signed-off-by: Mao Han <[email protected]>
> > Cc: Paul Walmsley <[email protected]>
> > Cc: Greentime Hu <[email protected]>
> > Cc: Palmer Dabbelt <[email protected]>
> > Cc: linux-riscv <[email protected]>
> > Cc: Christoph Hellwig <[email protected]>
> > Cc: Guo Ren <[email protected]>
> > ---
> > arch/riscv/Makefile | 3 +
> > arch/riscv/kernel/Makefile | 3 +-
> > arch/riscv/kernel/perf_callchain.c | 115 +++++++++++++++++++++++++++++++++++++
> > 3 files changed, 120 insertions(+), 1 deletion(-)
> > create mode 100644 arch/riscv/kernel/perf_callchain.c
>
> I just tested "./perf record -e cpu-clock --call-graph fp ls" on
> Unleashed board and I got this failure.
> I take a look at it. It seem failed in here. Do you have any idea?
> It seems fine in Qemu.
>
> 1 *frame = *((struct stackframe *)frame->fp - 1);
> ffffffe0001a198c: 00863a83 ld s5,8(a2)
> ffffffe0001a1990: ff093903 ld s2,-16(s2)
>
> ./perf record -e cpu-clock --call-graph fp ls
> [ 9619.423884] hrtimer: interrupt took 733000 ns
> [ 9619.977017] Unable to handle kernel paging request at virtual
> address ffffffffffffff94
> [ 9620.214736] Oops [#1]
> [ 9620.289893] Modules linked in:
> [ 9620.391378] CPU: 0 PID: 264 Comm: ls Not tainted
> 5.3.0-rc5-00003-gb008f6bcd67c #4
> [ 9620.640176] sepc: ffffffe0001a198c ra : ffffffe0001a199a sp :
> ffffffe000093720
> [ 9620.880366] gp : ffffffe00097dad8 tp : ffffffe000082e40 t0 : 0000000000046000
> [ 9621.120564] t1 : 0000000000000002 t2 : 0000000000000007 s0 : ffffffe000093760
> [ 9621.360768] s1 : ffffffe000093788 a0 : 0000000000000003 a1 : 0000000000000000
> [ 9621.600991] a2 : ffffffffffffff8c a3 : 0000000000001fa0 a4 : 0000000000000010
> [ 9621.841181] a5 : 0000000000000002 a6 : 0000000000000001 a7 : ffffffe079b34e10
> [ 9622.081400] s2 : ffffffffffffff9c s3 : ffffffe000000000 s4 : 0000000000001ff8
> [ 9622.321618] s5 : ffffffe000093da0 s6 : ffffffe00097d540 s7 : ffffffe07a1517a0
> [ 9622.561811] s8 : 000008bf01c7ff60 s9 : ffffffe000235b2a s10: 0000000200000120
> [ 9622.802015] s11: 0000000000000001 t3 : ffffffe079b34e00 t4 : 0000000000000001
> [ 9623.042194] t5 : 0000000000000008 t6 : ffffffe0009208d0
> [ 9623.218785] sstatus: 0000000200000100 sbadaddr: ffffffffffffff94
> scause: 000000000000000d
> [ 9623.490850] ---[ end trace 49043f28e856d84d ]---
> [ 9623.644217] Kernel panic - not syncing: Fatal exception in interrupt
> [ 9623.855470] SMP: stopping secondary CPUs
> [ 9623.985955] ---[ end Kernel panic - not syncing: Fatal exception in
> interrupt ]---



--
Best Regards
Guo Ren

ML: https://lore.kernel.org/linux-csky/

2019-08-26 08:06:36

by Greentime Hu

[permalink] [raw]
Subject: Re: [PATCH V5 1/3] riscv: Add perf callchain support

Hi Guo,

Guo Ren <[email protected]> 於 2019年8月24日 週六 上午8:54寫道:
>
> Please check CONFIG_FRAME_POINTER
>
> 1 *frame = *((struct stackframe *)frame->fp - 1);
> This code is origionally from riscv/kernel/stacktrace.c: walk_stackframe
>
> In linux/Makefile it'll involve the options for gcc to definitely
> store ra & prev_fp in fp pointer.
> ifdef CONFIG_FRAME_POINTER
> KBUILD_CFLAGS += -fno-omit-frame-pointer -fno-optimize-sibling-calls
>
> So --call-graph fp need depends on CONFIG_FRAME_POINTER.
>

I am pretty sure CONFIG_FRAME_POINTER is Y
# zcat /proc/config.gz |grep CONFIG_FRAME_POINTER
CONFIG_FRAME_POINTER=y

This is not going to go wrong every time, the probability of error is
about one tenth or one quarter. randomly
There may be some conditions that we have not considered.

I add one more condition to check if it is a valid virtual address and
it( ./perf record -e cpu-clock --call-graph fp ls) passes 1000 times
without failure in Unleashed board based on 5.3-rc5.
Here is my patch. Please have a look at it. I am not sure if it is a
good solution.

diff --git a/arch/riscv/kernel/perf_callchain.c
b/arch/riscv/kernel/perf_callchain.c
index d75d15c13dc7..4717942435df 100644
--- a/arch/riscv/kernel/perf_callchain.c
+++ b/arch/riscv/kernel/perf_callchain.c
@@ -18,6 +18,8 @@ static int unwind_frame_kernel(struct stackframe *frame)
return -EPERM;
if (frame->fp < CONFIG_PAGE_OFFSET)
return -EPERM;
+ if (!virt_addr_valid(frame->fp))
+ return -EPERM;

*frame = *((struct stackframe *)frame->fp - 1);
if (__kernel_text_address(frame->ra)) {

It could catch cases called in this way.

[ 1381.936586] frame->fp=:ffffffff00547550
[ 1382.038542] CPU: 1 PID: 135 Comm: ls Not tainted
5.3.0-rc5-00003-gb008f6bcd67c-dirty #14
[ 1382.307440] Call Trace:
[ 1382.388947] [<ffffffe0002a2d8e>] walk_stackframe+0x0/0x9a
[ 1382.568053] [<ffffffe0002a2f5a>] show_stack+0x2a/0x34
[ 1382.735960] [<ffffffe00083dcd6>] dump_stack+0x62/0x7c
[ 1382.903863] [<ffffffe0002a49e0>] perf_callchain_kernel+0xd8/0x102
[ 1383.106558] [<ffffffe000340a6e>] get_perf_callchain+0x136/0x1f2
[ 1383.303128] [<ffffffe00033d480>] perf_callchain+0x52/0x6e
[ 1383.482553] [<ffffffe00033d50a>] perf_prepare_sample+0x6e/0x476
[ 1383.679357] [<ffffffe00033d92e>] perf_event_output_forward+0x1c/0x50
[ 1383.890633] [<ffffffe000338b4c>] __perf_event_overflow+0x6a/0xa4
[ 1384.090279] [<ffffffe000338c40>] perf_swevent_hrtimer+0xba/0x106
[ 1384.290094] [<ffffffe0002f307c>] __hrtimer_run_queues+0x84/0x108
[ 1384.489694] [<ffffffe0002f36b8>] hrtimer_interrupt+0xca/0x1ce
[ 1384.680974] [<ffffffe00072f572>] riscv_timer_interrupt+0x32/0x3a
[ 1384.880449] [<ffffffe000854b34>] do_IRQ+0x64/0xbe
[ 1385.036698] [<ffffffe0002a1c5c>] ret_from_exception+0x0/0xc

[13915.697989] frame->fp=:fffffffffffff000
[13915.799937] CPU: 2 PID: 663 Comm: ls Not tainted
5.3.0-rc5-00003-gb008f6bcd67c-dirty #14
[13916.068832] Call Trace:
[13916.150380] [<ffffffe0002a2d8e>] walk_stackframe+0x0/0x9a
[13916.329450] [<ffffffe0002a2f5a>] show_stack+0x2a/0x34
[13916.497360] [<ffffffe00083dcd6>] dump_stack+0x62/0x7c
[13916.665265] [<ffffffe0002a49e0>] perf_callchain_kernel+0xd8/0x102
[13916.867949] [<ffffffe000340a6e>] get_perf_callchain+0x136/0x1f2
[13917.064526] [<ffffffe00033d480>] perf_callchain+0x52/0x6e
[13917.243950] [<ffffffe00033d50a>] perf_prepare_sample+0x6e/0x476
[13917.440759] [<ffffffe00033d92e>] perf_event_output_forward+0x1c/0x50
[13917.652021] [<ffffffe000338b4c>] __perf_event_overflow+0x6a/0xa4
[13917.851683] [<ffffffe000338c40>] perf_swevent_hrtimer+0xba/0x106
[13918.051494] [<ffffffe0002f307c>] __hrtimer_run_queues+0x84/0x108
[13918.251094] [<ffffffe0002f36b8>] hrtimer_interrupt+0xca/0x1ce
[13918.442379] [<ffffffe00072f572>] riscv_timer_interrupt+0x32/0x3a
[13918.641840] [<ffffffe000854b34>] do_IRQ+0x64/0xbe
[13918.798082] [<ffffffe0002a1c5c>] ret_from_exception+0x0/0xc

2019-08-27 05:51:57

by Guo Ren

[permalink] [raw]
Subject: Re: [PATCH V5 1/3] riscv: Add perf callchain support

We need know the values of *regs, eg: regs->sepc, regs->ra, regs->fp,
regs->sp, regs->tp

void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
struct pt_regs *regs)
{
...
walk_stackframe(&fr, entry);

// May be we could detect error here and print the regs' value
}

On Mon, Aug 26, 2019 at 4:03 PM Greentime Hu <[email protected]> wrote:
>
> Hi Guo,
>
> Guo Ren <[email protected]> 於 2019年8月24日 週六 上午8:54寫道:
> >
> > Please check CONFIG_FRAME_POINTER
> >
> > 1 *frame = *((struct stackframe *)frame->fp - 1);
> > This code is origionally from riscv/kernel/stacktrace.c: walk_stackframe
> >
> > In linux/Makefile it'll involve the options for gcc to definitely
> > store ra & prev_fp in fp pointer.
> > ifdef CONFIG_FRAME_POINTER
> > KBUILD_CFLAGS += -fno-omit-frame-pointer -fno-optimize-sibling-calls
> >
> > So --call-graph fp need depends on CONFIG_FRAME_POINTER.
> >
>
> I am pretty sure CONFIG_FRAME_POINTER is Y
> # zcat /proc/config.gz |grep CONFIG_FRAME_POINTER
> CONFIG_FRAME_POINTER=y
>
> This is not going to go wrong every time, the probability of error is
> about one tenth or one quarter. randomly
> There may be some conditions that we have not considered.
>
> I add one more condition to check if it is a valid virtual address and
> it( ./perf record -e cpu-clock --call-graph fp ls) passes 1000 times
> without failure in Unleashed board based on 5.3-rc5.
> Here is my patch. Please have a look at it. I am not sure if it is a
> good solution.
>
> diff --git a/arch/riscv/kernel/perf_callchain.c
> b/arch/riscv/kernel/perf_callchain.c
> index d75d15c13dc7..4717942435df 100644
> --- a/arch/riscv/kernel/perf_callchain.c
> +++ b/arch/riscv/kernel/perf_callchain.c
> @@ -18,6 +18,8 @@ static int unwind_frame_kernel(struct stackframe *frame)
> return -EPERM;
> if (frame->fp < CONFIG_PAGE_OFFSET)
> return -EPERM;
> + if (!virt_addr_valid(frame->fp))
> + return -EPERM;
>
> *frame = *((struct stackframe *)frame->fp - 1);
> if (__kernel_text_address(frame->ra)) {
>
> It could catch cases called in this way.
>
> [ 1381.936586] frame->fp=:ffffffff00547550
> [ 1382.038542] CPU: 1 PID: 135 Comm: ls Not tainted
> 5.3.0-rc5-00003-gb008f6bcd67c-dirty #14
> [ 1382.307440] Call Trace:
> [ 1382.388947] [<ffffffe0002a2d8e>] walk_stackframe+0x0/0x9a
> [ 1382.568053] [<ffffffe0002a2f5a>] show_stack+0x2a/0x34
> [ 1382.735960] [<ffffffe00083dcd6>] dump_stack+0x62/0x7c
> [ 1382.903863] [<ffffffe0002a49e0>] perf_callchain_kernel+0xd8/0x102
> [ 1383.106558] [<ffffffe000340a6e>] get_perf_callchain+0x136/0x1f2
> [ 1383.303128] [<ffffffe00033d480>] perf_callchain+0x52/0x6e
> [ 1383.482553] [<ffffffe00033d50a>] perf_prepare_sample+0x6e/0x476
> [ 1383.679357] [<ffffffe00033d92e>] perf_event_output_forward+0x1c/0x50
> [ 1383.890633] [<ffffffe000338b4c>] __perf_event_overflow+0x6a/0xa4
> [ 1384.090279] [<ffffffe000338c40>] perf_swevent_hrtimer+0xba/0x106
> [ 1384.290094] [<ffffffe0002f307c>] __hrtimer_run_queues+0x84/0x108
> [ 1384.489694] [<ffffffe0002f36b8>] hrtimer_interrupt+0xca/0x1ce
> [ 1384.680974] [<ffffffe00072f572>] riscv_timer_interrupt+0x32/0x3a
> [ 1384.880449] [<ffffffe000854b34>] do_IRQ+0x64/0xbe
> [ 1385.036698] [<ffffffe0002a1c5c>] ret_from_exception+0x0/0xc
>
> [13915.697989] frame->fp=:fffffffffffff000
> [13915.799937] CPU: 2 PID: 663 Comm: ls Not tainted
> 5.3.0-rc5-00003-gb008f6bcd67c-dirty #14
> [13916.068832] Call Trace:
> [13916.150380] [<ffffffe0002a2d8e>] walk_stackframe+0x0/0x9a
> [13916.329450] [<ffffffe0002a2f5a>] show_stack+0x2a/0x34
> [13916.497360] [<ffffffe00083dcd6>] dump_stack+0x62/0x7c
> [13916.665265] [<ffffffe0002a49e0>] perf_callchain_kernel+0xd8/0x102
> [13916.867949] [<ffffffe000340a6e>] get_perf_callchain+0x136/0x1f2
> [13917.064526] [<ffffffe00033d480>] perf_callchain+0x52/0x6e
> [13917.243950] [<ffffffe00033d50a>] perf_prepare_sample+0x6e/0x476
> [13917.440759] [<ffffffe00033d92e>] perf_event_output_forward+0x1c/0x50
> [13917.652021] [<ffffffe000338b4c>] __perf_event_overflow+0x6a/0xa4
> [13917.851683] [<ffffffe000338c40>] perf_swevent_hrtimer+0xba/0x106
> [13918.051494] [<ffffffe0002f307c>] __hrtimer_run_queues+0x84/0x108
> [13918.251094] [<ffffffe0002f36b8>] hrtimer_interrupt+0xca/0x1ce
> [13918.442379] [<ffffffe00072f572>] riscv_timer_interrupt+0x32/0x3a
> [13918.641840] [<ffffffe000854b34>] do_IRQ+0x64/0xbe
> [13918.798082] [<ffffffe0002a1c5c>] ret_from_exception+0x0/0xc



--
Best Regards
Guo Ren

ML: https://lore.kernel.org/linux-csky/