From: Peter Zijlstra <[email protected]>
Since ftrace has trampolines, don't use thunks for the __fentry__ site
but instead require that every function called from there includes
accounting. This very much includes all the direct-call functions.
Additionally, ftrace uses ROP tricks in two places:
- return_to_handler(), and
- ftrace_regs_caller() when pt_regs->orig_ax is set by a direct-call.
return_to_handler() already uses a retpoline to replace an
indirect-jump to defeat IBT, since this is a jump-type retpoline, make
sure there is no accounting done and ALTERNATIVE the RET into a ret.
ftrace_regs_caller() does much the same but currently causes an RSB
imbalance by effectively doing a PUSH+RET combo, rebalance.
Signed-off-by: Peter Zijlstra (Intel) <[email protected]>
Signed-off-by: Thomas Gleixner <[email protected]>
---
arch/x86/include/asm/nospec-branch.h | 8 +++++++
arch/x86/kernel/ftrace.c | 16 ++++++++++----
arch/x86/kernel/ftrace_64.S | 31 ++++++++++++++++++++++++++--
arch/x86/net/bpf_jit_comp.c | 6 +++++
kernel/trace/trace_selftest.c | 5 +++-
samples/ftrace/ftrace-direct-modify.c | 2 +
samples/ftrace/ftrace-direct-multi-modify.c | 2 +
samples/ftrace/ftrace-direct-multi.c | 1
samples/ftrace/ftrace-direct-too.c | 1
samples/ftrace/ftrace-direct.c | 1
10 files changed, 66 insertions(+), 7 deletions(-)
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -293,6 +293,11 @@ static inline void x86_set_skl_return_th
x86_return_thunk = &__x86_return_skl;
}
+#define CALL_DEPTH_ACCOUNT \
+ ALTERNATIVE("", \
+ __stringify(INCREMENT_CALL_DEPTH), \
+ X86_FEATURE_CALL_DEPTH)
+
DECLARE_PER_CPU(u64, __x86_call_depth);
#ifdef CONFIG_CALL_THUNKS_DEBUG
DECLARE_PER_CPU(u64, __x86_call_count);
@@ -302,6 +307,9 @@ DECLARE_PER_CPU(u64, __x86_ctxsw_count);
#endif
#else
static inline void x86_set_skl_return_thunk(void) {}
+
+#define CALL_DEPTH_ACCOUNT
+
#endif
#ifdef CONFIG_RETPOLINE
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -69,6 +69,10 @@ static const char *ftrace_nop_replace(vo
static const char *ftrace_call_replace(unsigned long ip, unsigned long addr)
{
+ /*
+ * No need to translate into a callthunk. The trampoline does
+ * the depth accounting itself.
+ */
return text_gen_insn(CALL_INSN_OPCODE, (void *)ip, (void *)addr);
}
@@ -316,7 +320,7 @@ create_trampoline(struct ftrace_ops *ops
unsigned long size;
unsigned long *ptr;
void *trampoline;
- void *ip;
+ void *ip, *dest;
/* 48 8b 15 <offset> is movq <offset>(%rip), %rdx */
unsigned const char op_ref[] = { 0x48, 0x8b, 0x15 };
unsigned const char retq[] = { RET_INSN_OPCODE, INT3_INSN_OPCODE };
@@ -403,10 +407,14 @@ create_trampoline(struct ftrace_ops *ops
/* put in the call to the function */
mutex_lock(&text_mutex);
call_offset -= start_offset;
+ /*
+ * No need to translate into a callthunk. The trampoline does
+ * the depth accounting before the call already.
+ */
+ dest = ftrace_ops_get_func(ops);
memcpy(trampoline + call_offset,
- text_gen_insn(CALL_INSN_OPCODE,
- trampoline + call_offset,
- ftrace_ops_get_func(ops)), CALL_INSN_SIZE);
+ text_gen_insn(CALL_INSN_OPCODE, trampoline + call_offset, dest),
+ CALL_INSN_SIZE);
mutex_unlock(&text_mutex);
/* ALLOC_TRAMP flags lets us know we created it */
--- a/arch/x86/kernel/ftrace_64.S
+++ b/arch/x86/kernel/ftrace_64.S
@@ -132,6 +132,7 @@
#ifdef CONFIG_DYNAMIC_FTRACE
SYM_FUNC_START(__fentry__)
+ CALL_DEPTH_ACCOUNT
RET
SYM_FUNC_END(__fentry__)
EXPORT_SYMBOL(__fentry__)
@@ -140,6 +141,8 @@ SYM_FUNC_START(ftrace_caller)
/* save_mcount_regs fills in first two parameters */
save_mcount_regs
+ CALL_DEPTH_ACCOUNT
+
/* Stack - skipping return address of ftrace_caller */
leaq MCOUNT_REG_SIZE+8(%rsp), %rcx
movq %rcx, RSP(%rsp)
@@ -155,6 +158,9 @@ SYM_INNER_LABEL(ftrace_caller_op_ptr, SY
/* Only ops with REGS flag set should have CS register set */
movq $0, CS(%rsp)
+ /* Account for the function call below */
+ CALL_DEPTH_ACCOUNT
+
SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
ANNOTATE_NOENDBR
call ftrace_stub
@@ -195,6 +201,8 @@ SYM_FUNC_START(ftrace_regs_caller)
save_mcount_regs 8
/* save_mcount_regs fills in first two parameters */
+ CALL_DEPTH_ACCOUNT
+
SYM_INNER_LABEL(ftrace_regs_caller_op_ptr, SYM_L_GLOBAL)
ANNOTATE_NOENDBR
/* Load the ftrace_ops into the 3rd parameter */
@@ -225,6 +233,9 @@ SYM_INNER_LABEL(ftrace_regs_caller_op_pt
/* regs go into 4th parameter */
leaq (%rsp), %rcx
+ /* Account for the function call below */
+ CALL_DEPTH_ACCOUNT
+
SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL)
ANNOTATE_NOENDBR
call ftrace_stub
@@ -280,7 +291,19 @@ SYM_INNER_LABEL(ftrace_regs_caller_end,
/* Restore flags */
popfq
UNWIND_HINT_FUNC
- jmp ftrace_epilogue
+
+ /*
+ * Since we're effectively emulating a tail-call with PUSH;RET
+ * make sure we don't unbalance the RSB and mess up accounting.
+ */
+ ANNOTATE_INTRA_FUNCTION_CALL
+ call 2f
+ int3
+2:
+ add $8, %rsp
+ ALTERNATIVE __stringify(RET), \
+ __stringify(ANNOTATE_UNRET_SAFE; ret; int3), \
+ X86_FEATURE_CALL_DEPTH
SYM_FUNC_END(ftrace_regs_caller)
STACK_FRAME_NON_STANDARD_FP(ftrace_regs_caller)
@@ -289,6 +312,8 @@ STACK_FRAME_NON_STANDARD_FP(ftrace_regs_
#else /* ! CONFIG_DYNAMIC_FTRACE */
SYM_FUNC_START(__fentry__)
+ CALL_DEPTH_ACCOUNT
+
cmpq $ftrace_stub, ftrace_trace_function
jnz trace
@@ -345,6 +370,8 @@ SYM_CODE_START(return_to_handler)
int3
.Ldo_rop:
mov %rdi, (%rsp)
- RET
+ ALTERNATIVE __stringify(RET), \
+ __stringify(ANNOTATE_UNRET_SAFE; ret; int3), \
+ X86_FEATURE_CALL_DEPTH
SYM_CODE_END(return_to_handler)
#endif
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -12,6 +12,7 @@
#include <linux/memory.h>
#include <linux/sort.h>
#include <asm/extable.h>
+#include <asm/ftrace.h>
#include <asm/set_memory.h>
#include <asm/nospec-branch.h>
#include <asm/text-patching.h>
@@ -2090,6 +2091,11 @@ int arch_prepare_bpf_trampoline(struct b
prog = image;
EMIT_ENDBR();
+ /*
+ * This is the direct-call trampoline, as such it needs accounting
+ * for the __fentry__ call.
+ */
+ x86_call_depth_emit_accounting(&prog, __fentry__);
EMIT1(0x55); /* push rbp */
EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */
EMIT4(0x48, 0x83, 0xEC, stack_size); /* sub rsp, stack_size */
--- a/kernel/trace/trace_selftest.c
+++ b/kernel/trace/trace_selftest.c
@@ -785,7 +785,10 @@ static struct fgraph_ops fgraph_ops __in
};
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
-noinline __noclone static void trace_direct_tramp(void) { }
+noinline __noclone static void trace_direct_tramp(void)
+{
+ asm(CALL_DEPTH_ACCOUNT);
+}
#endif
/*
--- a/samples/ftrace/ftrace-direct-modify.c
+++ b/samples/ftrace/ftrace-direct-modify.c
@@ -34,6 +34,7 @@ asm (
ASM_ENDBR
" pushq %rbp\n"
" movq %rsp, %rbp\n"
+ CALL_DEPTH_ACCOUNT
" call my_direct_func1\n"
" leave\n"
" .size my_tramp1, .-my_tramp1\n"
@@ -45,6 +46,7 @@ asm (
ASM_ENDBR
" pushq %rbp\n"
" movq %rsp, %rbp\n"
+ CALL_DEPTH_ACCOUNT
" call my_direct_func2\n"
" leave\n"
ASM_RET
--- a/samples/ftrace/ftrace-direct-multi-modify.c
+++ b/samples/ftrace/ftrace-direct-multi-modify.c
@@ -32,6 +32,7 @@ asm (
ASM_ENDBR
" pushq %rbp\n"
" movq %rsp, %rbp\n"
+ CALL_DEPTH_ACCOUNT
" pushq %rdi\n"
" movq 8(%rbp), %rdi\n"
" call my_direct_func1\n"
@@ -46,6 +47,7 @@ asm (
ASM_ENDBR
" pushq %rbp\n"
" movq %rsp, %rbp\n"
+ CALL_DEPTH_ACCOUNT
" pushq %rdi\n"
" movq 8(%rbp), %rdi\n"
" call my_direct_func2\n"
--- a/samples/ftrace/ftrace-direct-multi.c
+++ b/samples/ftrace/ftrace-direct-multi.c
@@ -27,6 +27,7 @@ asm (
ASM_ENDBR
" pushq %rbp\n"
" movq %rsp, %rbp\n"
+ CALL_DEPTH_ACCOUNT
" pushq %rdi\n"
" movq 8(%rbp), %rdi\n"
" call my_direct_func\n"
--- a/samples/ftrace/ftrace-direct-too.c
+++ b/samples/ftrace/ftrace-direct-too.c
@@ -29,6 +29,7 @@ asm (
ASM_ENDBR
" pushq %rbp\n"
" movq %rsp, %rbp\n"
+ CALL_DEPTH_ACCOUNT
" pushq %rdi\n"
" pushq %rsi\n"
" pushq %rdx\n"
--- a/samples/ftrace/ftrace-direct.c
+++ b/samples/ftrace/ftrace-direct.c
@@ -26,6 +26,7 @@ asm (
ASM_ENDBR
" pushq %rbp\n"
" movq %rsp, %rbp\n"
+ CALL_DEPTH_ACCOUNT
" pushq %rdi\n"
" call my_direct_func\n"
" popq %rdi\n"
On Sun, 17 Jul 2022 01:18:08 +0200 (CEST)
Thomas Gleixner <[email protected]> wrote:
> @@ -280,7 +291,19 @@ SYM_INNER_LABEL(ftrace_regs_caller_end,
> /* Restore flags */
> popfq
> UNWIND_HINT_FUNC
> - jmp ftrace_epilogue
> +
> + /*
> + * Since we're effectively emulating a tail-call with PUSH;RET
> + * make sure we don't unbalance the RSB and mess up accounting.
> + */
> + ANNOTATE_INTRA_FUNCTION_CALL
> + call 2f
> + int3
> +2:
> + add $8, %rsp
> + ALTERNATIVE __stringify(RET), \
> + __stringify(ANNOTATE_UNRET_SAFE; ret; int3), \
> + X86_FEATURE_CALL_DEPTH
>
> SYM_FUNC_END(ftrace_regs_caller)
Would this code be simpler if we nuked the ftrace_epilogue altogether?
After commit 0c0593b45c9b ("x86/ftrace: Make function graph use ftrace
directly"), the ftrace_epilogue is no longer needed. That was there to make
sure all the trampolines would call the function graph tracer. But now that
function graph tracing is just another ftrace caller, it's not needed
anymore.
Something like the below. It booted and passed the ftrace kselftests.
Feel free to include this in your series.
-- Steve
From 533f10bd48ffbc4ee5d2a07f0a7fe99aeb1c823a Mon Sep 17 00:00:00 2001
From: "Steven Rostedt (Google)" <[email protected]>
Date: Mon, 18 Jul 2022 16:01:07 -0400
Subject: [PATCH] ftrace/x86: Remove jumps to ftrace_epilogue
The jumps to ftrace_epilogue were done as a way to make sure all the
function tracing trampolines ended at the function graph trampoline, as
the ftrace_epilogue was the location that it would handle that.
With the advent of function graph tracer now being just one of the
callbacks of the function tracer there is no more requirement that all
trampolines go to a single location.
Remove the jumps to the ftrace_epilogue and replace them with return
statements.
Note, the ftrace_epilogue can probably be renamed to ftrace_stub and the
weak logic for that could probably be removed. But lets leave that as a
separate change.
Signed-off-by: Steven Rostedt (Google) <[email protected]>
---
arch/x86/kernel/ftrace_64.S | 16 +++++++---------
1 file changed, 7 insertions(+), 9 deletions(-)
diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S
index dfeb227de561..8f225fafa5fb 100644
--- a/arch/x86/kernel/ftrace_64.S
+++ b/arch/x86/kernel/ftrace_64.S
@@ -173,7 +173,9 @@ SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
SYM_INNER_LABEL(ftrace_caller_end, SYM_L_GLOBAL)
ANNOTATE_NOENDBR
- jmp ftrace_epilogue
+ UNWIND_HINT_FUNC
+ ENDBR
+ RET
SYM_FUNC_END(ftrace_caller);
STACK_FRAME_NON_STANDARD_FP(ftrace_caller)
@@ -261,15 +263,9 @@ SYM_INNER_LABEL(ftrace_regs_caller_jmp, SYM_L_GLOBAL)
/* Restore flags */
popfq
- /*
- * As this jmp to ftrace_epilogue can be a short jump
- * it must not be copied into the trampoline.
- * The trampoline will add the code to jump
- * to the return.
- */
SYM_INNER_LABEL(ftrace_regs_caller_end, SYM_L_GLOBAL)
ANNOTATE_NOENDBR
- jmp ftrace_epilogue
+ jmp 2f
/* Swap the flags with orig_rax */
1: movq MCOUNT_REG_SIZE(%rsp), %rdi
@@ -279,8 +275,10 @@ SYM_INNER_LABEL(ftrace_regs_caller_end, SYM_L_GLOBAL)
restore_mcount_regs 8
/* Restore flags */
popfq
+2:
UNWIND_HINT_FUNC
- jmp ftrace_epilogue
+ ENDBR
+ RET
SYM_FUNC_END(ftrace_regs_caller)
STACK_FRAME_NON_STANDARD_FP(ftrace_regs_caller)
--
2.35.1
On Mon, Jul 18, 2022 at 05:01:23PM -0400, Steven Rostedt wrote:
> From 533f10bd48ffbc4ee5d2a07f0a7fe99aeb1c823a Mon Sep 17 00:00:00 2001
> From: "Steven Rostedt (Google)" <[email protected]>
> Date: Mon, 18 Jul 2022 16:01:07 -0400
> Subject: [PATCH] ftrace/x86: Remove jumps to ftrace_epilogue
>
> The jumps to ftrace_epilogue were done as a way to make sure all the
> function tracing trampolines ended at the function graph trampoline, as
> the ftrace_epilogue was the location that it would handle that.
>
> With the advent of function graph tracer now being just one of the
> callbacks of the function tracer there is no more requirement that all
> trampolines go to a single location.
>
> Remove the jumps to the ftrace_epilogue and replace them with return
> statements.
>
> Note, the ftrace_epilogue can probably be renamed to ftrace_stub and the
> weak logic for that could probably be removed. But lets leave that as a
> separate change.
>
> Signed-off-by: Steven Rostedt (Google) <[email protected]>
> ---
> arch/x86/kernel/ftrace_64.S | 16 +++++++---------
> 1 file changed, 7 insertions(+), 9 deletions(-)
>
> diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S
> index dfeb227de561..8f225fafa5fb 100644
> --- a/arch/x86/kernel/ftrace_64.S
> +++ b/arch/x86/kernel/ftrace_64.S
> @@ -173,7 +173,9 @@ SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
> SYM_INNER_LABEL(ftrace_caller_end, SYM_L_GLOBAL)
> ANNOTATE_NOENDBR
>
> - jmp ftrace_epilogue
> + UNWIND_HINT_FUNC
> + ENDBR
Only the RET should do I think; you definitely don't need an ENDBR here
nor do you need to override the unwind hint. Lemme try..
Yeah, the below is sufficient:
diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S
index dfeb227de561..d6679b65b6f2 100644
--- a/arch/x86/kernel/ftrace_64.S
+++ b/arch/x86/kernel/ftrace_64.S
@@ -172,8 +172,7 @@ SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
*/
SYM_INNER_LABEL(ftrace_caller_end, SYM_L_GLOBAL)
ANNOTATE_NOENDBR
-
- jmp ftrace_epilogue
+ RET
SYM_FUNC_END(ftrace_caller);
STACK_FRAME_NON_STANDARD_FP(ftrace_caller)
@@ -269,7 +268,7 @@ SYM_INNER_LABEL(ftrace_regs_caller_jmp, SYM_L_GLOBAL)
*/
SYM_INNER_LABEL(ftrace_regs_caller_end, SYM_L_GLOBAL)
ANNOTATE_NOENDBR
- jmp ftrace_epilogue
+ RET
/* Swap the flags with orig_rax */
1: movq MCOUNT_REG_SIZE(%rsp), %rdi
@@ -280,7 +279,7 @@ SYM_INNER_LABEL(ftrace_regs_caller_end, SYM_L_GLOBAL)
/* Restore flags */
popfq
UNWIND_HINT_FUNC
- jmp ftrace_epilogue
+ RET
SYM_FUNC_END(ftrace_regs_caller)
STACK_FRAME_NON_STANDARD_FP(ftrace_regs_caller)
On Tue, 19 Jul 2022 10:46:48 +0200
Peter Zijlstra <[email protected]> wrote:
> Only the RET should do I think; you definitely don't need an ENDBR here
> nor do you need to override the unwind hint. Lemme try..
I'll replace with the RET and resend v2.
-- Steve