Received-SPF: pass (google.com: best guess record for domain of linux-kernel-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) client-ip=209.132.180.67;
Date:   Thu, 25 Apr 2019 14:11:00 -0700 (PDT)
Subject:     Re: [PATCH 1/3] riscv: Add perf callchain support
In-Reply-To: <195185ea63240ed396026505d96d1f6449963482.1554961908.git.han_mao@c-sky.com>
CC:     linux-kernel@vger.kernel.org, han_mao@c-sky.com
From:   Palmer Dabbelt <palmer@sifive.com>
To:     han_mao@c-sky.com
Message-ID: <mhng-dd1d2b4c-122f-48d8-ac56-1c6bff93f236@palmer-si-x1e>
Mime-Version: 1.0 (MHng)
Content-Type: text/plain; charset=utf-8; format=flowed
Content-Transfer-Encoding: 8bit
Sender: linux-kernel-owner@vger.kernel.org
Precedence: bulk

On Thu, 11 Apr 2019 00:53:48 PDT (-0700), han_mao@c-sky.com wrote:
> This patch add support for perf callchain sampling on riscv platform.
> The return address of leaf function is retrieved from pt_regs as
> it is not saved in the outmost frame.
>
> Signed-off-by: Mao Han <han_mao@c-sky.com>
>
> CC: Palmer Dabbelt <palmer@sifive.com>
> ---
>  arch/riscv/kernel/Makefile         |   3 +-
>  arch/riscv/kernel/perf_callchain.c | 122 +++++++++++++++++++++++++++++++++++++
>  2 files changed, 124 insertions(+), 1 deletion(-)
>  create mode 100644 arch/riscv/kernel/perf_callchain.c
>
> diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile
> index 5985681..dd2ba44 100644
> --- a/arch/riscv/kernel/Makefile
> +++ b/arch/riscv/kernel/Makefile
> @@ -37,6 +37,7 @@ obj-$(CONFIG_MODULE_SECTIONS)	+= module-sections.o
>  obj-$(CONFIG_FUNCTION_TRACER)	+= mcount.o ftrace.o
>  obj-$(CONFIG_DYNAMIC_FTRACE)	+= mcount-dyn.o
>
> -obj-$(CONFIG_PERF_EVENTS)      += perf_event.o
> +obj-$(CONFIG_PERF_EVENTS)	+= perf_event.o
> +obj-$(CONFIG_PERF_EVENTS)	+= perf_callchain.o
>
>  clean:
> diff --git a/arch/riscv/kernel/perf_callchain.c b/arch/riscv/kernel/perf_callchain.c
> new file mode 100644
> index 0000000..eb3ddbb
> --- /dev/null
> +++ b/arch/riscv/kernel/perf_callchain.c
> @@ -0,0 +1,122 @@
> +// SPDX-License-Identifier: GPL-2.0
> +// Copyright (C) 2019 Hangzhou C-SKY Microsystems co.,ltd.
> +
> +#include <linux/perf_event.h>
> +#include <linux/uaccess.h>
> +
> +/* Kernel callchain */
> +struct stackframe {
> +	unsigned long fp;
> +	unsigned long ra;
> +};
> +
> +static int unwind_frame_kernel(struct stackframe *frame)
> +{
> +	int graph = 0;
> +
> +	/* 0x3 means misalignment */
> +	if (!kstack_end((void *)frame->fp) &&
> +	    !((unsigned long)frame->fp & 0x3) &&
> +	    ((unsigned long)frame->fp >= TASK_SIZE)) {
> +		frame->ra = ((struct stackframe *)frame->fp - 1)->ra;
> +		frame->fp = ((struct stackframe *)frame->fp - 1)->fp;

It looks like this depends on having a frame pointer?  In that case, shouldn't
we add some Kconfig logic to make CONFIG_PERF_EVENTS select
-fno-omit-frame-pointer?  Frame pointers aren't enabled by default on RISC-V
and therefor are unlikely to exist at all.

> +		/* make sure CONFIG_FUNCTION_GRAPH_TRACER is turned on */

Should that also be mandated by a Kconfig?

> +		if (__kernel_text_address(frame->ra))
> +			frame->ra = ftrace_graph_ret_addr(NULL, &graph,
> +							frame->ra, NULL);
> +		return 0;
> +	} else {
> +		return -EPERM;
> +	}
> +}
> +
> +static void notrace walk_stackframe(struct stackframe *fr,
> +			struct perf_callchain_entry_ctx *entry)
> +{
> +	while (1) {
> +		int ret;
> +
> +		perf_callchain_store(entry, fr->ra);
> +
> +		ret = unwind_frame_kernel(fr);
> +		if (ret < 0)
> +			break;
> +	}
> +}
> +
> +/*
> + * Get the return address for a single stackframe and return a pointer to the
> + * next frame tail.
> + */
> +static unsigned long user_backtrace(struct perf_callchain_entry_ctx *entry,
> +			unsigned long fp, unsigned long reg_ra)
> +{
> +	struct stackframe buftail;
> +	unsigned long ra = 0;
> +	unsigned long *user_frame_tail = (unsigned long *)(fp - sizeof(struct stackframe));
> +
> +	/* Check accessibility of one struct frame_tail beyond */
> +	if (!access_ok(user_frame_tail, sizeof(buftail)))
> +		return 0;
> +	if (__copy_from_user_inatomic(&buftail, user_frame_tail,
> +				      sizeof(buftail)))
> +		return 0;
> +
> +	if (reg_ra != 0)
> +		ra = reg_ra;
> +	else
> +		ra = buftail.ra;
> +
> +	fp = buftail.fp;
> +	perf_callchain_store(entry, ra);
> +
> +	return fp;
> +}
> +
> +/*
> + * This will be called when the target is in user mode
> + * This function will only be called when we use
> + * "PERF_SAMPLE_CALLCHAIN" in
> + * kernel/events/core.c:perf_prepare_sample()
> + *
> + * How to trigger perf_callchain_[user/kernel] :
> + * $ perf record -e cpu-clock --call-graph fp ./program
> + * $ perf report --call-graph
> + *
> + * On RISC-V platform, the program being sampled and the C library
> + * need to be compiled with * -mbacktrace, otherwise the user

What is "-mbacktrace"?  I don't remember that ever being a RISC-V GCC option,
and my compiler doesn't undersand it.  It understands "-fbacktrace" but that
doesn't produce a frame pointer.

> + * stack will not contain function frame.
> + */
> +void perf_callchain_user(struct perf_callchain_entry_ctx *entry,
> +			 struct pt_regs *regs)
> +{
> +	unsigned long fp = 0;
> +
> +	/* RISC-V does not support virtualization. */
> +	if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
> +		return;
> +
> +	fp = regs->s0;
> +	perf_callchain_store(entry, regs->sepc);
> +
> +	fp = user_backtrace(entry, fp, regs->ra);
> +	while ((entry->nr < entry->max_stack) &&
> +		fp && !((unsigned long)fp & 0x3))
> +		fp = user_backtrace(entry, fp, 0);
> +}
> +
> +void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
> +			   struct pt_regs *regs)
> +{
> +	struct stackframe fr;
> +
> +	/* RISC-V does not support virtualization. */
> +	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
> +		pr_warn("RISC-V does not support perf in guest mode!");
> +		return;
> +	}
> +
> +	fr.fp = regs->s0;
> +	fr.ra = regs->ra;
> +	walk_stackframe(&fr, entry);
> +}