Hi,
I have found the exact problem with respect to ARM jprobes.
The problem with configure i.e, CONFIG_ARM_UNWIND = y; is enabled.
Due to the stack unwinding, the jprobe is not able to return since
Where jprobe starts getting 'system call" is when you have exceptions
or setjmp/longjmp
which simply unwinding the stack without returning.
After disabling (CONFIG_ARM_UNWIND = n;), all probes are working fine.
The dump_stack() is also not an issue after the above change. Please ignore the
previous mail.
Finally there are no issues with ARM kprobes,jprobes and return probes
But 'CONFIG_ARM_UNWIND' should be disabled for jprobes testing.
Best regards,
Venkappa
On Mon, Aug 31, 2009 at 3:41 PM, venki kaps<[email protected]> wrote:
> Hi,
>
> After dump_stack change, most of the system calls
> (do_fork,do_execve,do_gettimeofday,
> sys_gettimeofday,sys_open,sys_close, sys_read,sys_write) are working
> fine with kprobes
> and kretprobes.
>
> But in jprobes, still 'SEGV,sysetm hangs,OOPS' will be getting for 'do_execve,
> do_gettimeofday,sys_gettimeofday,sys_close, sys_read,sys_write' system calls.
>
> ?I have surprised with jprobes which are working well for
> 'do_fork,sys_open' system calls.
>
> Best Regards,
> Venkappa
>
> On Mon, Aug 31, 2009 at 10:57 AM, venki kaps<[email protected]> wrote:
>> Hi,
>>
>> I have further investigated with respect to this issue and found the
>> problem with
>> 'dump_stack()' which calls in my sample kprobe,kretprobe and jprobe
>> modules to get
>> the stack dump.
>>
>> Here it is giving sample example module which covers all the probes.
>>
>> Sample module test program:
>> ---------------------------
>> #include <linux/module.h>
>> #include <linux/init.h>
>> #include <linux/kprobes.h>
>>
>> static int k_count1 = 0;
>> static int k_count2 = 0;
>> static int k_count3 = 0;
>> static int k_count4 = 0;
>>
>> /* Proxy routine having the same arguments as actual sys_open() routine */
>> long jsys_open(unsigned int fd, char __user * buf, size_t count)
>> {
>> ? ? ? ?printk("%s %d, Proxy sys_open, arguments are %d, %d\n",
>> __FILE__, __LINE__, fd, count);
>> ? ? ? ?printk("%s %d, Stack_dump :\n", __FILE__, __LINE__);
>> ? ? ? ?dump_stack();
>> ? ? ? ?/* Always end with a call to jprobe_return(). */
>> ? ? ? ?k_count1++;
>> ? ? ? ?jprobe_return();
>> ? ? ? ?/* NOTREACHED */
>> ? ? ? ?return 0;
>> }
>>
>> static struct jprobe my_jprobe = {
>> ? ? ? ?.entry = JPROBE_ENTRY(jsys_open)
>> };
>>
>> static const char *probed_func = "sys_open";
>>
>> /* Return-probe handler: Log the return value from the probed function. */
>> static int ret_handler(struct kretprobe_instance *ri, struct pt_regs *regs)
>> {
>> ? ? ? ?int retval = regs_return_value(regs);
>>
>> ? ? ? ?printk("%s %d, %s returns %d\n", __FILE__, __LINE__,
>> probed_func, retval);
>> ? ? ? ?printk("%s %d, Stack_dump :\n", __FILE__, __LINE__);
>> ? ? ? ?dump_stack();
>> ? ? ? ?k_count2++;
>> ? ? ? ?return 0;
>> }
>>
>> static struct kretprobe my_kretprobe = {
>> ? ? ? ?.handler = ret_handler,
>> ? ? ? ?/* Probe up to 20 instances concurrently. */
>> ? ? ? ?.maxactive = 20
>> };
>>
>> static struct kprobe k_001_kpr;
>>
>> static int k_001_before_hook(struct kprobe *k_001_kpr, struct pt_regs *p)
>> {
>> ? ? ? ?printk("%s %d\nStack dump for the kprobe pre handler for
>> instruction at %p\n", __FILE__, __LINE__, k_001_kpr->addr);
>> ? ? ? ?dump_stack();
>> ? ? ? ?k_count3++;
>> ? ? ? ?return 0;
>> }
>>
>> void k_001_after_hook(struct kprobe *k_001_kpr, struct pt_regs *p,
>> unsigned long flags)
>> {
>> ? ? ? ?printk("%s %d\nStack dump for the kprobe post handler at
>> %p\n", __FILE__, __LINE__, k_001_kpr->addr);
>> ? ? ? ?dump_stack();
>> ? ? ? ?printk("%s %d, The Registers are:\n", __FILE__, __LINE__);
>> ? ? ? ?k_count4++;
>> }
>>
>>
>> static int __init k_001_init_probe(void)
>> {
>> ? ? ? ?int ret;
>> ? ? ? ?int retj;
>> ? ? ? ?printk("%s %d\nInserting the kprobe for sys_open\n", __FILE__,
>> __LINE__);
>>
>> ? ? ? ?/* Registering a kprobe */
>> ? ? ? ?k_001_kpr.pre_handler = (kprobe_pre_handler_t) k_001_before_hook;
>> ? ? ? ?k_001_kpr.post_handler = (kprobe_post_handler_t) k_001_after_hook;
>> ? ? ? ?k_001_kpr.symbol_name = "sys_open", __FILE__, __LINE__;
>> ? ? ? ?if (register_kprobe(&k_001_kpr) < 0) {
>> ? ? ? ? ? ? ? ?printk("%s %dk-001.c:register_kprobe is failed\n",
>> __FILE__, __LINE__);
>> ? ? ? ? ? ? ? ?return -1;
>> ? ? ? ?}
>> ? ? ? ?printk("%s %d, register_kprobe is successful\n", __FILE__, __LINE__);
>>
>> ? ? ? ?printk("%s %d, Inserting the kretprobe for sys_open\n",
>> __FILE__, __LINE__);
>> ? ? ? ?my_kretprobe.kp.symbol_name = (char *)probed_func;
>>
>> ? ? ? ?if ((ret = register_kretprobe(&my_kretprobe)) < 0) {
>> ? ? ? ? ? ? ? ?printk("%s %d, register_kretprobe failed, returned
>> %d\n", __FILE__, __LINE__, ret);
>> ? ? ? ? ? ? ? ?return -1;
>> ? ? ? ?}
>> ? ? ? ?printk("%s %d, Planted return probe for sys_open at %p\n",
>> __FILE__, __LINE__, my_kretprobe.kp.addr);
>>
>> ? ? ? ?my_jprobe.kp.symbol_name = "sys_open";
>>
>> ? ? ? ?if ((retj = register_jprobe(&my_jprobe)) < 0) {
>> ? ? ? ? ? ? ? ?printk("%s %d,register_jprobe failed, returned %d\n",
>> __FILE__, __LINE__, ret);
>> ? ? ? ? ? ? ? ?return -1;
>> ? ? ? ?}
>> ? ? ? ?printk("%s %d,Planted jprobe at %p, handler addr %p\n",
>> __FILE__, __LINE__, my_jprobe.kp.addr, my_jprobe.entry);
>>
>> ? ? ? ?return 0;
>> }
>>
>> static void __exit k_001_exit_probe(void)
>> {
>> ? ? ? ?unregister_kprobe(&k_001_kpr);
>> ? ? ? ?printk("%s %d\nkprobe unregistered from sys_open \n",
>> __FILE__, __LINE__);
>>
>> ? ? ? ?unregister_kretprobe(&my_kretprobe);
>> ? ? ? ?printk("%s %dkretprobe unregistered\n", __FILE__, __LINE__);
>> ? ? ? ?/* nmissed > 0 suggests that maxactive was set too low. */
>> ? ? ? ?printk("%s %dMissed probing %d instances of %s\n", __FILE__,
>> __LINE__, my_kretprobe.nmissed, probed_func);
>>
>> ? ? ? ?unregister_jprobe(&my_jprobe);
>> ? ? ? ?printk("%s %d,jprobe unregistered\n", __FILE__, __LINE__);
>>
>> ? ? ? ?if (k_count1 > 0 && k_count2 > 0 && k_count3 > 0 && k_count4 > 0)
>> ? ? ? ? ? ? ? ?printk("TEST PASS");
>> ? ? ? ?else
>> ? ? ? ? ? ? ? ?printk("TEST FAIL");
>> }
>>
>> module_init(k_001_init_probe);
>> module_exit(k_001_exit_probe);
>>
>> MODULE_DESCRIPTION("Kprobes test module");
>> MODULE_LICENSE("GPL");
>>
>> I have tested the above program and got result as system hang.
>>
>> Disable dump_stack:
>> ------------------------------
>> I have disabled dump_stack(ARM specific) in the above program
>> and did not notice any problem.
>>
>> After disabling the dump_stack(), all the probes are working
>> fine for 'do_fork,sys_open and sys_close system calls.
>>
>> ARM dump_stack implementation has been changed in 2.6.29 kernel:
>> ----------------------------------------------------------------------------------------------------
>> I have found in 2.6.29 kernel the current ARM dump_stack implementation
>> has been changed.Due to that change the kprobes,jprobes and kretprobes
>> are failing for 'do_fork,sys_open and sys_close' system calls.
>>
>> Current dump_stack implementation:
>>
>> Location: arch/arm/kernel/traps.c
>>
>> void dump_stack(void)
>> {
>> ? ? ? ?dump_backtrace(NULL, NULL);
>> }
>>
>>
>> I have just reverted back the above source to old kernel implementation.
>>
>> Index: b/arch/arm/kernel/traps.c
>> ===================================================================
>> --- a/arch/arm/kernel/traps.c
>> +++ b/arch/arm/kernel/traps.c
>> @@ -202,7 +202,11 @@ static void dump_backtrace(struct pt_reg
>>
>> ?void dump_stack(void)
>> ?{
>> +#if 0
>> ? ? ? ?dump_backtrace(NULL, NULL);
>> +#else
>> + ? ? ? __backtrace();
>> +#endif
>> ?}
>>
>> ?EXPORT_SYMBOL(dump_stack);
>>
>>
>> After the above change,All probes are working fine with
>> enabling of dump_stack() in my sample modules.
>>
>> I have some queries with repect to the above changes.
>>
>> Queries:
>> ========
>> 1) Revert back the old kernel implementation might not be a good solution.
>> ? ? ? ?I anticipate it needs to be fixed in 2.6.29 kernel implementations.
>> ? ? ? ?Am i right/wrong.
>>
>> 2) Shall i avoid by calling dump_stack() in my sample test modules?
>>
>> 3) Only do_fork, sys_open and sys_close system calls are working fine with
>> ? ? ? ?dump_stack() but still 'SEGV,sysetm hangs' for do_execve,do_gettimeofday,
>> ? ? ? ?sys_gettimeofday,sys_read,sys_write, etc system calls.
>>
>> I have been further investigating with respect to the above issues
>> Meanwhile could you please provide the inputs with respect to the above queries?
>>
>> Thanks in advance.
>>
>> Best regards,
>> Venkappa
>>
>>
>> On Tue, Aug 25, 2009 at 11:07 PM, Nicolas Pitre<[email protected]> wrote:
>>> On Tue, 25 Aug 2009, venki kaps wrote:
>>>
>>>> Hi,
>>>>
>>>> I have been tracing kernel system call information using
>>>> the Kprobes/jprobes/kretprobes implementation in the 2.6.29 kernel on
>>>> ARM architecture.
>>>> Although the mainline kprobe/jprobe/kretprobe examples are working
>>>> fine (do_fork),
>>>> I have been facing some issues while running my own jprobe/kretprobe tests.
>>>>
>>> [...]
>>>>
>>>> Query:
>>>> =====
>>>> ? ? ? - Are there any limitations for jptobes/kretporbes in mainline kernel for ARM?
>>>
>>> No limitation in particular that I know of.
>>>
>>>> ? ? ? - Why it works for only do_fork and why not for others
>>>> (do_execve/sys_open/sys_close/sys_read/sys_write)?
>>>
>>> I don't know. ?Will try to have a look.
>>>
>>>> ? ? ? - Is it required any additional setup to achieve this?
>>>
>>> Not supposed to need anything special.
>>>
>>>
>>> Nicolas
>>>
>>
>
venki kaps <[email protected]> wrote:
> I have found the exact problem with respect to ARM jprobes.
>
> The problem with configure i.e, CONFIG_ARM_UNWIND = y; is enabled.
I haven't followed the kprobes implementation for ARM but does it make
any assumptions about the existence of a frame pointer on the stack?
Enabling stack unwinding automatically disables the framepointer.
--
Catalin
On Tue, Sep 01, 2009 at 02:54:54PM +0100, Catalin Marinas wrote:
> venki kaps <[email protected]> wrote:
> > I have found the exact problem with respect to ARM jprobes.
> >
> > The problem with configure i.e, CONFIG_ARM_UNWIND = y; is enabled.
>
> I haven't followed the kprobes implementation for ARM but does it make
> any assumptions about the existence of a frame pointer on the stack?
> Enabling stack unwinding automatically disables the framepointer.
If it uses CALLER_ADDRESSx() then it won't work with unwinding enabled.
See 5613/1 (which is pending in the devel branch.)
--
Russell King
Linux kernel 2.6 ARM Linux - http://www.arm.linux.org.uk/
maintainer of:
On Tue, 2009-09-01 at 15:25 +0100, Russell King wrote:
> On Tue, Sep 01, 2009 at 02:54:54PM +0100, Catalin Marinas wrote:
> > venki kaps <[email protected]> wrote:
> > > I have found the exact problem with respect to ARM jprobes.
> > >
> > > The problem with configure i.e, CONFIG_ARM_UNWIND = y; is enabled.
> >
> > I haven't followed the kprobes implementation for ARM but does it make
> > any assumptions about the existence of a frame pointer on the stack?
> > Enabling stack unwinding automatically disables the framepointer.
>
> If it uses CALLER_ADDRESSx() then it won't work with unwinding enabled.
> See 5613/1 (which is pending in the devel branch.)
In addition to that, when CONFIG_FRAME_POINTER is disabled, the lr
register isn't always saved on the stack by the called function. I'm not
sure whether kretprobe_trampoline is aware of this.
--
Catalin
On Tue, 1 Sep 2009, Catalin Marinas wrote:
> On Tue, 2009-09-01 at 15:25 +0100, Russell King wrote:
> > On Tue, Sep 01, 2009 at 02:54:54PM +0100, Catalin Marinas wrote:
> > > venki kaps <[email protected]> wrote:
> > > > I have found the exact problem with respect to ARM jprobes.
> > > >
> > > > The problem with configure i.e, CONFIG_ARM_UNWIND = y; is enabled.
> > >
> > > I haven't followed the kprobes implementation for ARM but does it make
> > > any assumptions about the existence of a frame pointer on the stack?
> > > Enabling stack unwinding automatically disables the framepointer.
> >
> > If it uses CALLER_ADDRESSx() then it won't work with unwinding enabled.
> > See 5613/1 (which is pending in the devel branch.)
>
> In addition to that, when CONFIG_FRAME_POINTER is disabled, the lr
> register isn't always saved on the stack by the called function. I'm not
> sure whether kretprobe_trampoline is aware of this.
The way a kretprobe works is to put a trap at the very first instruction
of the targetted function, preserve the value of LR when the trap is
hit, and substitute it with the address of kretprobe_trampoline. Then
the original first instruction is emulated to pass over the trap point
and normal execution is resumed. So whether or not LR is then saved on
the stack doesn't matter to kretprobe_trampoline as it will restore the
LR value saved during the initial trap.
Of course if you end up generating a backtrace within a kretprobed
function then the result might look funny.
Nicolas
I have done some more analysis with respect to ARM kretprobes
and describing as below,
Before disabling unwinding support,
i have tested mainlined sample kretprobes_example.c with some changes.
the changes are,
Index: b/samples/kprobes/kretprobe_example.c
===================================================================
--- a/samples/kprobes/kretprobe_example.c
+++ b/samples/kprobes/kretprobe_example.c
@@ -59,6 +59,9 @@ static int ret_handler(struct kretprobe_
s64 delta;
ktime_t now;
+ printk("%s %d, Stack_dump :\n", __FILE__, __LINE__);
+ dump_stack();
+
now = ktime_get();
delta = ktime_to_ns(ktime_sub(now, data->entry_stamp));
printk(KERN_INFO "%s returned %d and took %lld ns to execute\n",
$ insmod kretprobe_example.ko
$ ls
<---- system hang
As i mentioned jprobes and kretprobes are having some issues with backtrace
with enabling unwinding support and as we know well about ARM, unwinding tables
are not stabilized which was pending issue in present kernels.
After disabling unwinding support,
Kernel hacking --->
-*- frame pointer support
[] Enable stack unwinding support
Based on the above settings, I have verified the above test with dump_stack and
noticed test result as PASS.
I wanted to rely on FRAME_POINTER So have just added
'''select FRAME_POINTER''' to kprobe related configs as below as,
Index: b/samples/Kconfig
===================================================================
--- a/samples/Kconfig
+++ b/samples/Kconfig
@@ -31,6 +31,7 @@ config SAMPLE_KOBJECT
config SAMPLE_KPROBES
tristate "Build kprobes examples -- loadable modules only"
depends on KPROBES && m
+ select FRAME_POINTER
help
This build several kprobes example modules.
@@ -38,6 +39,7 @@ config SAMPLE_KRETPROBES
tristate "Build kretprobes example -- loadable modules only"
default m
depends on SAMPLE_KPROBES && KRETPROBES
+ select FRAME_POINTER
config SAMPLE_PSRWLOCK
tristate "Build psrwlock example -- loadable modules only"
Index: b/arch/Kconfig
===================================================================
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -36,6 +36,7 @@ config KPROBES
bool "Kprobes"
depends on KALLSYMS && MODULES
depends on HAVE_KPROBES
+ select FRAME_POINTER
help
Kprobes allows you to trap at almost any kernel address and
execute a callback function. register_kprobe() establishes
@@ -68,6 +69,7 @@ config HAVE_SYSCALL_WRAPPERS
config KRETPROBES
def_bool y
depends on KPROBES && HAVE_KRETPROBES
+ select FRAME_POINTER
config HAVE_IOREMAP_PROT
bool
I have some queries,
- shall i rely on FRAME_POINTER in kprobe related configs?
- shall i proceed with above changes?
Best Regards,
Venkappa
On Tue, Sep 1, 2009 at 11:26 PM, Nicolas Pitre <[email protected]> wrote:
> On Tue, 1 Sep 2009, Catalin Marinas wrote:
>
>> On Tue, 2009-09-01 at 15:25 +0100, Russell King wrote:
>> > On Tue, Sep 01, 2009 at 02:54:54PM +0100, Catalin Marinas wrote:
>> > > venki kaps <[email protected]> wrote:
>> > > > I have found the exact problem with respect to ARM jprobes.
>> > > >
>> > > > The problem with configure i.e, CONFIG_ARM_UNWIND = y; is enabled.
>> > >
>> > > I haven't followed the kprobes implementation for ARM but does it make
>> > > any assumptions about the existence of a frame pointer on the stack?
>> > > Enabling stack unwinding automatically disables the framepointer.
>> >
>> > If it uses CALLER_ADDRESSx() then it won't work with unwinding enabled.
>> > See 5613/1 (which is pending in the devel branch.)
>>
>> In addition to that, when CONFIG_FRAME_POINTER is disabled, the lr
>> register isn't always saved on the stack by the called function. I'm not
>> sure whether kretprobe_trampoline is aware of this.
>
> The way a kretprobe works is to put a trap at the very first instruction
> of the targetted function, preserve the value of LR when the trap is
> hit, and substitute it with the address of kretprobe_trampoline. ?Then
> the original first instruction is emulated to pass over the trap point
> and normal execution is resumed. ?So whether or not LR is then saved on
> the stack doesn't matter to kretprobe_trampoline as it will restore the
> LR value saved during the initial trap.
>
> Of course if you end up generating a backtrace within a kretprobed
> function then the result might look funny.
>
>
> Nicolas
>