This patch provides stacktrace filtering feature.
The stacktrace filter allows failing only for the caller you are
interested in.
stacktrace filter is enabled by setting the value of
/debugfs/*/stacktrace-depth more than 0.
and specify the range of the virtual address
by the /debugfs/*/address-start and /debugfs/*/address-end
Please see the example that demostrates how to inject slab allocation
failures only for a specific module
in Documentation/fault-injection/fault-injection.txt
Cc: [email protected]
Signed-off-by: Akinobu Mita <[email protected]>
include/linux/fault-inject.h | 15 +++++++++
lib/Kconfig.debug | 2 +
lib/fault-inject-debugfs.c | 32 +++++++++++++++++++
lib/fault-inject.c | 71 +++++++++++++++++++++++++++++++++++++++++++
4 files changed, 120 insertions(+)
Index: work-shouldfail/lib/fault-inject.c
===================================================================
--- work-shouldfail.orig/lib/fault-inject.c
+++ work-shouldfail/lib/fault-inject.c
@@ -6,6 +6,9 @@
#include <linux/fs.h>
#include <linux/module.h>
#include <linux/interrupt.h>
+#include <linux/unwind.h>
+#include <linux/stacktrace.h>
+#include <linux/kallsyms.h>
#include <linux/fault-inject.h>
int setup_fault_attr(struct fault_attr *attr, char *str)
@@ -59,6 +62,72 @@ static int fail_process(struct fault_att
return !in_interrupt() && task->make_it_fail;
}
+#ifdef CONFIG_STACK_UNWIND
+
+static asmlinkage int fail_stacktrace_callback(struct unwind_frame_info *info,
+ void *arg)
+{
+ int depth;
+ struct fault_attr *attr = arg;
+
+ for (depth = 0; depth < attr->stacktrace_depth
+ && unwind(info) == 0 && UNW_PC(info); depth++) {
+ if (arch_unw_user_mode(info))
+ break;
+ if (attr->address_start <= UNW_PC(info) &&
+ UNW_PC(info) < attr->address_end)
+ return 1;
+ }
+ return 0;
+}
+
+static int fail_stacktrace(struct fault_attr *attr)
+{
+ struct unwind_frame_info info;
+
+ /* stacktrace filter is disabled */
+ if (attr->stacktrace_depth == 0)
+ return 1;
+
+ return unwind_init_running(&info, fail_stacktrace_callback, attr);
+}
+
+#elif defined(CONFIG_STACKTRACE)
+
+#include <linux/stacktrace.h>
+
+#define MAX_STACK_TRACE_DEPTH 10
+
+static int fail_stacktrace(struct fault_attr *attr)
+{
+ struct stack_trace trace;
+ int depth = attr->stacktrace_depth;
+ unsigned long entries[MAX_STACK_TRACE_DEPTH];
+ int n;
+
+ /* stacktrace filter is disabled */
+ if (depth == 0)
+ return 1;
+
+ trace.nr_entries = 0;
+ trace.entries = entries;
+ trace.max_entries = (depth < MAX_STACK_TRACE_DEPTH) ?
+ depth : MAX_STACK_TRACE_DEPTH;
+
+ save_stack_trace(&trace, NULL, 0, 1);
+ for (n = 0; n < trace.nr_entries; n++)
+ if (attr->address_start <= entries[n] &&
+ entries[n] < attr->address_end)
+ return 1;
+ return 0;
+}
+
+#else
+
+#define fail_stacktrace(attr) (0)
+
+#endif
+
/*
* This code is stolen from failmalloc-1.0
* http://www.nongnu.org/failmalloc/
@@ -68,6 +137,8 @@ int should_fail(struct fault_attr *attr,
{
if (!fail_process(attr, current))
return 0;
+ if (!fail_stacktrace(attr))
+ return 0;
if (atomic_read(&max_failures(attr)) == 0)
return 0;
Index: work-shouldfail/include/linux/fault-inject.h
===================================================================
--- work-shouldfail.orig/include/linux/fault-inject.h
+++ work-shouldfail/include/linux/fault-inject.h
@@ -30,6 +30,21 @@ struct fault_attr {
/* A value of '0' means process filter is disabled. */
u32 process_filter;
+
+ /*
+ * maximam number of stacktrace depth walking allowed
+ * A value of '0' means stacktrace filter is disabled.
+ */
+ unsigned long stacktrace_depth;
+
+ /*
+ * If stacktrace_depth is enabled, it allows failing only when it
+ * has been called from the virtual addresses in the range
+ * 'address_start' to 'address_end-1'
+ */
+ unsigned long address_start;
+ unsigned long address_end;
+
};
#define DEFINE_FAULT_ATTR(name) \
Index: work-shouldfail/lib/fault-inject-debugfs.c
===================================================================
--- work-shouldfail.orig/lib/fault-inject-debugfs.c
+++ work-shouldfail/lib/fault-inject-debugfs.c
@@ -9,6 +9,9 @@ struct fault_attr_entries {
struct dentry *times_file;
struct dentry *space_file;
struct dentry *process_filter_file;
+ struct dentry *stacktrace_depth_file;
+ struct dentry *address_start_file;
+ struct dentry *address_end_file;
};
static void debugfs_ul_set(void *data, u64 val)
@@ -71,6 +74,18 @@ static void cleanup_fault_attr_entries(s
debugfs_remove(entries->process_filter_file);
entries->process_filter_file = NULL;
}
+ if (entries->stacktrace_depth_file) {
+ debugfs_remove(entries->stacktrace_depth_file);
+ entries->stacktrace_depth_file = NULL;
+ }
+ if (entries->address_start_file) {
+ debugfs_remove(entries->address_start_file);
+ entries->address_start_file = NULL;
+ }
+ if (entries->address_end_file) {
+ debugfs_remove(entries->address_end_file);
+ entries->address_end_file = NULL;
+ }
debugfs_remove(entries->dir);
entries->dir = NULL;
}
@@ -116,6 +131,23 @@ static int init_fault_attr_entries(struc
goto fail;
entries->process_filter_file = file;
+ file = debugfs_create_ul("stacktrace-depth", mode, dir,
+ &attr->stacktrace_depth);
+ if (!file)
+ goto fail;
+ entries->stacktrace_depth_file = file;
+
+ file = debugfs_create_ul("address-start", mode, dir,
+ &attr->address_start);
+ if (!file)
+ goto fail;
+ entries->address_start_file = file;
+
+ file = debugfs_create_ul("address-end", mode, dir, &attr->address_end);
+ if (!file)
+ goto fail;
+ entries->address_end_file = file;
+
return 0;
fail:
cleanup_fault_attr_entries(entries);
Index: work-shouldfail/lib/Kconfig.debug
===================================================================
--- work-shouldfail.orig/lib/Kconfig.debug
+++ work-shouldfail/lib/Kconfig.debug
@@ -371,6 +371,8 @@ config RCU_TORTURE_TEST
config FAULT_INJECTION
bool
+ select STACKTRACE
+ select FRAME_POINTER
config FAILSLAB
bool "fault-injection capabilitiy for kmalloc"
--
Undo temporary fix-up for clean application of patch
"[patch 8/8] stacktrace filtering for fault-injection capabilities".
Fix bug in !(CONFIG_STACK_UNWIND || CONFIG_STACKTRACE) case, based on
code inspection only. Anyone with a non-i386, -x86_64, -s390 willing
to test this?
Reintroduce stacktrace-specific documentation, factored out earlier.
Signed-off-by: Don Mullis <[email protected]>
---
Documentation/fault-injection/fault-injection.txt | 13 +++++++++++++
include/linux/fault-inject.h | 2 +-
lib/fault-inject.c | 2 +-
3 files changed, 15 insertions(+), 2 deletions(-)
Index: linux-2.6.17/lib/fault-inject.c
===================================================================
--- linux-2.6.17.orig/lib/fault-inject.c
+++ linux-2.6.17/lib/fault-inject.c
@@ -136,7 +136,7 @@ static int fail_stacktrace(struct fault_
#else
-#define fail_stacktrace(attr) (0)
+#define fail_stacktrace(attr) (1)
#endif
Index: linux-2.6.17/Documentation/fault-injection/fault-injection.txt
===================================================================
--- linux-2.6.17.orig/Documentation/fault-injection/fault-injection.txt
+++ linux-2.6.17/Documentation/fault-injection/fault-injection.txt
@@ -61,6 +61,19 @@ configuration of fault-injection capabil
A negative value means that failures are enabled for
pid==-process_filter irrespective of /proc/<pid>/make-it-fail.
+- /debug/*/stacktrace-depth:
+
+ specifies the maximum stacktrace depth walked during search
+ for a caller within [address-start,address-end). A value of 0
+ disables the stacktrace filter.
+
+- /debug/*/address-start:
+- /debug/*/address-end:
+
+ specifies the range of virtual addresses tested during
+ stacktrace walking. Failure is injected only if some caller
+ in the walked stacktrace lies within this range.
+
o Boot option
In order to inject faults while debugfs is not available (early boot time),
Index: linux-2.6.17/include/linux/fault-inject.h
===================================================================
--- linux-2.6.17.orig/include/linux/fault-inject.h
+++ linux-2.6.17/include/linux/fault-inject.h
@@ -16,7 +16,7 @@ struct fault_attr {
atomic_t times;
atomic_t space;
unsigned long count;
- u32 process_filter;
+ atomic_t process_filter;
unsigned long stacktrace_depth;
unsigned long address_start;
unsigned long address_end;
Push fault-injection entries earlier in the list, so that they appear
nested under DEBUG_KERNEL in menuconfig/xconfig.
Signed-off-by: Don Mullis <[email protected]>
---
lib/Kconfig.debug | 70 +++++++++++++++++++++++++++---------------------------
1 file changed, 35 insertions(+), 35 deletions(-)
Index: linux-2.6.17/lib/Kconfig.debug
===================================================================
--- linux-2.6.17.orig/lib/Kconfig.debug
+++ linux-2.6.17/lib/Kconfig.debug
@@ -294,6 +294,41 @@ config DEBUG_INFO
If unsure, say N.
+config FAULT_INJECTION
+ bool
+ depends on DEBUG_KERNEL
+ select STACKTRACE
+ select FRAME_POINTER
+
+config FAILSLAB
+ bool "fault-injection capability for kmalloc"
+ depends on DEBUG_KERNEL
+ select FAULT_INJECTION
+ help
+ This option provides fault-injection capability for kmalloc.
+
+config FAIL_PAGE_ALLOC
+ bool "fault-injection capability for alloc_pages()"
+ depends on DEBUG_KERNEL
+ select FAULT_INJECTION
+ help
+ This option provides fault-injection capability for alloc_pages().
+
+config FAIL_MAKE_REQUEST
+ bool "fault-injection capability for disk IO"
+ depends on DEBUG_KERNEL
+ select FAULT_INJECTION
+ help
+ This option provides fault-injection capability to disk IO.
+
+config FAULT_INJECTION_DEBUGFS
+ tristate "runtime configuration for fault-injection capabilities"
+ depends on DEBUG_KERNEL && SYSFS && FAULT_INJECTION
+ select DEBUG_FS
+ help
+ This option provides kernel module that provides runtime
+ configuration interface by debugfs.
+
config DEBUG_FS
bool "Debug Filesystem"
depends on SYSFS
@@ -368,38 +403,3 @@ config RCU_TORTURE_TEST
at boot time (you probably don't).
Say M if you want the RCU torture tests to build as a module.
Say N if you are unsure.
-
-config FAULT_INJECTION
- bool
- select STACKTRACE
- select FRAME_POINTER
-
-config FAILSLAB
- bool "fault-injection capabilitiy for kmalloc"
- depends on DEBUG_KERNEL
- select FAULT_INJECTION
- help
- This option provides fault-injection capabilitiy for kmalloc.
-
-config FAIL_PAGE_ALLOC
- bool "fault-injection capabilitiy for alloc_pages()"
- depends on DEBUG_KERNEL
- select FAULT_INJECTION
- help
- This option provides fault-injection capabilitiy for alloc_pages().
-
-config FAIL_MAKE_REQUEST
- bool "fault-injection capabilitiy for disk IO"
- depends on DEBUG_KERNEL
- select FAULT_INJECTION
- help
- This option provides fault-injection capabilitiy to disk IO.
-
-config FAULT_INJECTION_DEBUGFS
- tristate "runtime configuration for fault-injection capabilities"
- depends on DEBUG_KERNEL && SYSFS && FAULT_INJECTION
- select DEBUG_FS
- help
- This option provides kernel module that provides runtime
- configuration interface by debugfs.
-
On Mon, Sep 18, 2006 at 10:56:28PM -0700, Don Mullis wrote:
> Fix bug in !(CONFIG_STACK_UNWIND || CONFIG_STACKTRACE) case, based on
> code inspection only. Anyone with a non-i386, -x86_64, -s390 willing
> to test this?
Yes, it didn't work completely without this patch on other architectures.
Thank you.
On Mon, Sep 18, 2006 at 10:57:51PM -0700, Don Mullis wrote:
> Push fault-injection entries earlier in the list, so that they appear
> nested under DEBUG_KERNEL in menuconfig/xconfig.
Disabling the option Kernel debugging can hide all config options
realated to fault-injection without this patch.
Do I misunderstand something?
On Tue, 2006-09-19 at 17:09 +0800, Akinobu Mita wrote:
> On Mon, Sep 18, 2006 at 10:57:51PM -0700, Don Mullis wrote:
> > Push fault-injection entries earlier in the list, so that they appear
> > nested under DEBUG_KERNEL in menuconfig/xconfig.
>
> Disabling the option Kernel debugging can hide all config options
> realated to fault-injection without this patch.
>
> Do I misunderstand something?
>
There's no problem with hiding per se, but rather with the indentation
level. It's most natural for the user to have dependent options
indented under their "parent". For an example, in "menuconfig" try
setting "Compile the kernel with frame unwind information";
notice that "Stack unwind support" appears immediately underneath
it, indented. The indentation reminds the user why "Stack unwind
support" has appeared.
Note that several of the pre-existing, non-fault-injection options
under "Kernel debugging", are also broken in this way.
On Tue, Sep 19, 2006 at 10:35:27AM -0700, Don Mullis wrote:
> There's no problem with hiding per se, but rather with the indentation
> level. It's most natural for the user to have dependent options
> indented under their "parent". For an example, in "menuconfig" try
> setting "Compile the kernel with frame unwind information";
> notice that "Stack unwind support" appears immediately underneath
> it, indented. The indentation reminds the user why "Stack unwind
> support" has appeared.
>
I see.
> Note that several of the pre-existing, non-fault-injection options
> under "Kernel debugging", are also broken in this way.
Perhaps you can move UNWIND_INFO, STACK_UNWIND, and DEBUG_FS
entries ealier in the list. It improves improve appearance for
other DEBUG_KERNEL dependent config options like (DEBUG_VM,
FRAME_POINTER, ...).
On Wed, 2006-09-20 at 21:39 +0800, Akinobu Mita wrote:
> Perhaps you can move UNWIND_INFO, STACK_UNWIND, and DEBUG_FS
> entries ealier in the list. It improves improve appearance for
> other DEBUG_KERNEL dependent config options like (DEBUG_VM,
> FRAME_POINTER, ...).
Okay, I will prepare such a janitorial patch.
> Index: work-fault-inject/lib/Kconfig.debug
> ===================================================================
> --- work-fault-inject.orig/lib/Kconfig.debug
> +++ work-fault-inject/lib/Kconfig.debug
> @@ -401,6 +401,8 @@ config LKDTM
>
> config FAULT_INJECTION
> bool
> + select STACKTRACE
> + select FRAME_POINTER
I'm afraid this won't work fully reliably on i386/x86-64 at least. The problem
is that if even with frame pointers the new unwinder will try its unwinding
first and if it gets stuck it will log the fallback entries into the stack
trace buffer too. And those fallback entries can be randomly everything left over
from previous stack traces. Then the fault injection will be more
like Russian roulette and could randomly hit other code paths too.
To make this work there would need to be some way to turn off fallback
for these particular stack traces.
Or maybe just always use frame pointer
there, but even that will likely not help because the few places
where the dwarf2 unwinder still gets stuck (usually assembly code)
are usually broken with frame pointers too.
-Andi