Add support for kprobes on ftrace call sites to avoid much of the overhead
with regular kprobes. Try it with simple steps:
cd /sys/kernel/debug/tracing/
echo 'p:myprobe sys_clone r0=%r0 r1=%r1 r2=%r2' > kprobe_events
echo 1 > events/kprobes/enable
echo 1 > events/kprobes/myprobe/enable
cat trace
# tracer: nop
#
# entries-in-buffer/entries-written: 2/2 #P:4
#
# _-----=> irqs-off/BH-disabled
# / _----=> need-resched
# | / _---=> hardirq/softirq
# || / _--=> preempt-depth
# ||| / _-=> migrate-disable
# |||| / delay
# TASK-PID CPU# ||||| TIMESTAMP FUNCTION
# | | | ||||| | |
sh-75 [000] ..... 33.793362: myprobe: (sys_clone+0xc/0xa0) r0=0x1200011 r1=0x0 r2=0x0
sh-75 [000] ..... 34.817804: myprobe: (sys_clone+0xc/0xa0) r0=0x1200011 r1=0x0 r2=0x0
cat /sys/kernel/debug/kprobes/list
c03453e8 k sys_clone+0xc [FTRACE]
^^^^^^
Signed-off-by: Jinjie Ruan <[email protected]>
---
.../debug/kprobes-on-ftrace/arch-support.txt | 2 +-
arch/arm/Kconfig | 1 +
arch/arm/include/asm/ftrace.h | 17 ++++++
arch/arm/kernel/ftrace.c | 17 ------
arch/arm/probes/Makefile | 1 +
arch/arm/probes/ftrace.c | 53 +++++++++++++++++++
arch/arm/probes/kprobes/core.c | 32 +++++++++++
7 files changed, 105 insertions(+), 18 deletions(-)
create mode 100644 arch/arm/probes/ftrace.c
diff --git a/Documentation/features/debug/kprobes-on-ftrace/arch-support.txt b/Documentation/features/debug/kprobes-on-ftrace/arch-support.txt
index 02febc883588..4ecd7d53e859 100644
--- a/Documentation/features/debug/kprobes-on-ftrace/arch-support.txt
+++ b/Documentation/features/debug/kprobes-on-ftrace/arch-support.txt
@@ -8,7 +8,7 @@
-----------------------
| alpha: | TODO |
| arc: | TODO |
- | arm: | TODO |
+ | arm: | ok |
| arm64: | TODO |
| csky: | ok |
| hexagon: | TODO |
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index ee5115252aac..ed13b1743f94 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -114,6 +114,7 @@ config ARM
select HAVE_KERNEL_LZO
select HAVE_KERNEL_XZ
select HAVE_KPROBES if !XIP_KERNEL && !CPU_ENDIAN_BE32 && !CPU_V7M
+ select HAVE_KPROBES_ON_FTRACE if !XIP_KERNEL && !CPU_ENDIAN_BE32 && !CPU_V7M
select HAVE_KRETPROBES if HAVE_KPROBES
select HAVE_MOD_ARCH_SPECIFIC
select HAVE_NMI
diff --git a/arch/arm/include/asm/ftrace.h b/arch/arm/include/asm/ftrace.h
index 5be3ddc96a50..c8e3f808b70c 100644
--- a/arch/arm/include/asm/ftrace.h
+++ b/arch/arm/include/asm/ftrace.h
@@ -22,6 +22,23 @@ struct dyn_arch_ftrace {
#endif
};
+/*
+ * The compiler emitted profiling hook consists of
+ *
+ * PUSH {LR}
+ * BL __gnu_mcount_nc
+ *
+ * To turn this combined sequence into a NOP, we need to restore the value of
+ * SP before the PUSH. Let's use an ADD rather than a POP into LR, as LR is not
+ * modified anyway, and reloading LR from memory is highly likely to be less
+ * efficient.
+ */
+#ifdef CONFIG_THUMB2_KERNEL
+#define NOP 0xf10d0d04 /* add.w sp, sp, #4 */
+#else
+#define NOP 0xe28dd004 /* add sp, sp, #4 */
+#endif
+
static inline unsigned long ftrace_call_adjust(unsigned long addr)
{
/* With Thumb-2, the recorded addresses have the lsb set */
diff --git a/arch/arm/kernel/ftrace.c b/arch/arm/kernel/ftrace.c
index a0b6d1e3812f..f0f1bdf27637 100644
--- a/arch/arm/kernel/ftrace.c
+++ b/arch/arm/kernel/ftrace.c
@@ -25,23 +25,6 @@
#include <asm/stacktrace.h>
#include <asm/patch.h>
-/*
- * The compiler emitted profiling hook consists of
- *
- * PUSH {LR}
- * BL __gnu_mcount_nc
- *
- * To turn this combined sequence into a NOP, we need to restore the value of
- * SP before the PUSH. Let's use an ADD rather than a POP into LR, as LR is not
- * modified anyway, and reloading LR from memory is highly likely to be less
- * efficient.
- */
-#ifdef CONFIG_THUMB2_KERNEL
-#define NOP 0xf10d0d04 /* add.w sp, sp, #4 */
-#else
-#define NOP 0xe28dd004 /* add sp, sp, #4 */
-#endif
-
#ifdef CONFIG_DYNAMIC_FTRACE
static int __ftrace_modify_code(void *data)
diff --git a/arch/arm/probes/Makefile b/arch/arm/probes/Makefile
index 8b0ea5ace100..b3c355942a21 100644
--- a/arch/arm/probes/Makefile
+++ b/arch/arm/probes/Makefile
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_UPROBES) += decode.o decode-arm.o uprobes/
obj-$(CONFIG_KPROBES) += decode.o kprobes/
+obj-$(CONFIG_KPROBES_ON_FTRACE) += ftrace.o
ifdef CONFIG_THUMB2_KERNEL
obj-$(CONFIG_KPROBES) += decode-thumb.o
else
diff --git a/arch/arm/probes/ftrace.c b/arch/arm/probes/ftrace.c
new file mode 100644
index 000000000000..0f54b8e5d2a6
--- /dev/null
+++ b/arch/arm/probes/ftrace.c
@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/kprobes.h>
+
+/* Ftrace callback handler for kprobes -- called under preepmt disabled */
+void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
+ struct ftrace_ops *ops, struct ftrace_regs *regs)
+{
+ struct kprobe *p;
+ struct kprobe_ctlblk *kcb;
+
+ p = get_kprobe((kprobe_opcode_t *)ip);
+ if (unlikely(!p) || kprobe_disabled(p))
+ return;
+
+ kcb = get_kprobe_ctlblk();
+ if (kprobe_running()) {
+ kprobes_inc_nmissed_count(p);
+ } else {
+ unsigned long orig_ip = instruction_pointer(&(regs->regs));
+
+ instruction_pointer_set(&(regs->regs), ip);
+
+ __this_cpu_write(current_kprobe, p);
+ kcb->kprobe_status = KPROBE_HIT_ACTIVE;
+ if (!p->pre_handler || !p->pre_handler(p, &(regs->regs))) {
+ /*
+ * Emulate singlestep (and also recover regs->pc)
+ * as if there is a nop
+ */
+ instruction_pointer_set(&(regs->regs),
+ (unsigned long)p->addr + MCOUNT_INSN_SIZE);
+ if (unlikely(p->post_handler)) {
+ kcb->kprobe_status = KPROBE_HIT_SSDONE;
+ p->post_handler(p, &(regs->regs), 0);
+ }
+ instruction_pointer_set(&(regs->regs), orig_ip);
+ }
+
+ /*
+ * If pre_handler returns !0, it changes regs->pc. We have to
+ * skip emulating post_handler.
+ */
+ __this_cpu_write(current_kprobe, NULL);
+ }
+}
+NOKPROBE_SYMBOL(kprobe_ftrace_handler);
+
+int arch_prepare_kprobe_ftrace(struct kprobe *p)
+{
+ p->ainsn.insn = NULL;
+ return 0;
+}
diff --git a/arch/arm/probes/kprobes/core.c b/arch/arm/probes/kprobes/core.c
index d8238da095df..5e2f18cfd766 100644
--- a/arch/arm/probes/kprobes/core.c
+++ b/arch/arm/probes/kprobes/core.c
@@ -45,6 +45,38 @@ DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
+kprobe_opcode_t *arch_adjust_kprobe_addr(unsigned long addr, unsigned long offset,
+ bool *on_func_entry)
+{
+#ifdef CONFIG_KPROBES_ON_FTRACE
+ unsigned long nop_offset = 0;
+ u32 insn = 0;
+
+ /*
+ * Since 'addr' is not guaranteed to be safe to access, use
+ * copy_from_kernel_nofault() to read the instruction:
+ */
+ if (copy_from_kernel_nofault(&insn, (void *)(addr + nop_offset),
+ sizeof(u32)))
+ return NULL;
+
+ while (insn != NOP) {
+ nop_offset += 4;
+ if (copy_from_kernel_nofault(&insn, (void *)(addr + nop_offset),
+ sizeof(u32)))
+ return NULL;
+ }
+
+ *on_func_entry = offset <= nop_offset;
+ if (*on_func_entry)
+ offset = nop_offset;
+#else
+ *on_func_entry = !offset;
+#endif
+
+ return (kprobe_opcode_t *)(addr + offset);
+}
+
int __kprobes arch_prepare_kprobe(struct kprobe *p)
{
kprobe_opcode_t insn;
--
2.34.1
Hi Jinjie,
kernel test robot noticed the following build errors:
[auto build test ERROR on soc/for-next]
[also build test ERROR on linus/master v6.10-rc3 next-20240613]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Jinjie-Ruan/arm-Add-KPROBES_ON_FTRACE-supported/20240614-102440
base: https://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git for-next
patch link: https://lore.kernel.org/r/20240614022318.2600814-1-ruanjinjie%40huawei.com
patch subject: [PATCH] arm: Add KPROBES_ON_FTRACE supported
config: arm-allmodconfig (https://download.01.org/0day-ci/archive/20240616/[email protected]/config)
compiler: arm-linux-gnueabi-gcc (GCC) 13.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20240616/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All error/warnings (new ones prefixed by >>):
>> drivers/gpu/drm/xe/xe_lrc.c:92: warning: "NOP" redefined
92 | #define NOP(x) (BIT(7) | (x))
|
In file included from include/linux/ftrace.h:23,
from include/linux/kprobes.h:28,
from include/linux/kgdb.h:19,
from include/drm/drm_util.h:36,
from include/drm/drm_connector.h:32,
from drivers/gpu/drm/i915/display/intel_display_core.h:16,
from drivers/gpu/drm/xe/xe_device_types.h:27,
from drivers/gpu/drm/xe/xe_vm_types.h:16,
from drivers/gpu/drm/xe/xe_bo.h:13,
from drivers/gpu/drm/xe/xe_lrc.c:14:
arch/arm/include/asm/ftrace.h:39: note: this is the location of the previous definition
39 | #define NOP 0xe28dd004 /* add sp, sp, #4 */
|
--
>> drivers/gpu/drm/xe/xe_pci.c:80: warning: "NOP" redefined
80 | #define NOP(x) x
|
In file included from include/linux/ftrace.h:23,
from include/linux/kprobes.h:28,
from include/linux/kgdb.h:19,
from include/drm/drm_util.h:36,
from drivers/gpu/drm/xe/xe_device.h:12,
from drivers/gpu/drm/xe/display/xe_display.h:9,
from drivers/gpu/drm/xe/xe_pci.c:18:
arch/arm/include/asm/ftrace.h:39: note: this is the location of the previous definition
39 | #define NOP 0xe28dd004 /* add sp, sp, #4 */
|
--
In file included from include/linux/ftrace.h:23,
from include/linux/perf_event.h:52,
from include/linux/trace_events.h:10,
from include/trace/syscall.h:7,
from include/linux/syscalls.h:93,
from drivers/scsi/aacraid/linit.c:32:
>> arch/arm/include/asm/ftrace.h:39:25: error: expected identifier before numeric constant
39 | #define NOP 0xe28dd004 /* add sp, sp, #4 */
| ^~~~~~~~~~
include/scsi/scsi_status.h:19:9: note: in expansion of macro 'NOP'
19 | NOP = 0x08,
| ^~~
In file included from drivers/scsi/aacraid/linit.c:38:
include/scsi/scsi_cmnd.h: In function 'scsi_msg_to_host_byte':
>> include/scsi/scsi_cmnd.h:372:14: error: 'TARGET_RESET' undeclared (first use in this function)
372 | case TARGET_RESET:
| ^~~~~~~~~~~~
include/scsi/scsi_cmnd.h:372:14: note: each undeclared identifier is reported only once for each function it appears in
vim +39 arch/arm/include/asm/ftrace.h
24
25 /*
26 * The compiler emitted profiling hook consists of
27 *
28 * PUSH {LR}
29 * BL __gnu_mcount_nc
30 *
31 * To turn this combined sequence into a NOP, we need to restore the value of
32 * SP before the PUSH. Let's use an ADD rather than a POP into LR, as LR is not
33 * modified anyway, and reloading LR from memory is highly likely to be less
34 * efficient.
35 */
36 #ifdef CONFIG_THUMB2_KERNEL
37 #define NOP 0xf10d0d04 /* add.w sp, sp, #4 */
38 #else
> 39 #define NOP 0xe28dd004 /* add sp, sp, #4 */
40 #endif
41
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki