2006-09-14 10:21:58

by Akinobu Mita

[permalink] [raw]
Subject: [patch 7/8] process filtering for fault-injection capabilities

This patch provides process filtering feature.
The process filter allows failing only permitted processes
by /proc/<pid>/make-it-fail

Please see the example that demostrates how to inject slab allocation
failures into module init/cleanup code
in Documentation/fault-injection/fault-injection.txt

Signed-off-by: Akinobu Mita <[email protected]>

fs/proc/base.c | 77 +++++++++++++++++++++++++++++++++++++++++++
include/linux/fault-inject.h | 3 +
include/linux/sched.h | 3 +
lib/fault-inject.c | 13 +++++++
4 files changed, 96 insertions(+)

Index: work-shouldfail/fs/proc/base.c
===================================================================
--- work-shouldfail.orig/fs/proc/base.c
+++ work-shouldfail/fs/proc/base.c
@@ -138,6 +138,9 @@ enum pid_directory_inos {
#endif
PROC_TGID_OOM_SCORE,
PROC_TGID_OOM_ADJUST,
+#ifdef CONFIG_FAULT_INJECTION
+ PROC_TGID_FAULT_INJECTION,
+#endif
PROC_TID_INO,
PROC_TID_STATUS,
PROC_TID_MEM,
@@ -181,6 +184,9 @@ enum pid_directory_inos {
#endif
PROC_TID_OOM_SCORE,
PROC_TID_OOM_ADJUST,
+#ifdef CONFIG_FAULT_INJECTION
+ PROC_TID_FAULT_INJECTION,
+#endif

/* Add new entries before this */
PROC_TID_FD_DIR = 0x8000, /* 0x8000-0xffff */
@@ -240,6 +246,9 @@ static struct pid_entry tgid_base_stuff[
#ifdef CONFIG_AUDITSYSCALL
E(PROC_TGID_LOGINUID, "loginuid", S_IFREG|S_IWUSR|S_IRUGO),
#endif
+#ifdef CONFIG_FAULT_INJECTION
+ E(PROC_TGID_FAULT_INJECTION, "make-it-fail", S_IFREG|S_IWUSR|S_IRUGO),
+#endif
{0,0,NULL,0}
};
static struct pid_entry tid_base_stuff[] = {
@@ -282,6 +291,9 @@ static struct pid_entry tid_base_stuff[]
#ifdef CONFIG_AUDITSYSCALL
E(PROC_TID_LOGINUID, "loginuid", S_IFREG|S_IWUSR|S_IRUGO),
#endif
+#ifdef CONFIG_FAULT_INJECTION
+ E(PROC_TID_FAULT_INJECTION, "make-it-fail", S_IFREG|S_IWUSR|S_IRUGO),
+#endif
{0,0,NULL,0}
};

@@ -992,6 +1004,65 @@ static struct file_operations proc_login
};
#endif

+#ifdef CONFIG_FAULT_INJECTION
+static ssize_t proc_fault_inject_read(struct file * file, char __user * buf,
+ size_t count, loff_t *ppos)
+{
+ struct task_struct *task = get_proc_task(file->f_dentry->d_inode);
+ char buffer[PROC_NUMBUF];
+ size_t len;
+ int make_it_fail;
+ loff_t __ppos = *ppos;
+
+ if (!task)
+ return -ESRCH;
+ make_it_fail = task->make_it_fail;
+ put_task_struct(task);
+
+ len = snprintf(buffer, sizeof(buffer), "%i\n", make_it_fail);
+ if (__ppos >= len)
+ return 0;
+ if (count > len-__ppos)
+ count = len-__ppos;
+ if (copy_to_user(buf, buffer + __ppos, count))
+ return -EFAULT;
+ *ppos = __ppos + count;
+ return count;
+}
+
+static ssize_t proc_fault_inject_write(struct file * file,
+ const char __user * buf, size_t count, loff_t *ppos)
+{
+ struct task_struct *task;
+ char buffer[PROC_NUMBUF], *end;
+ int make_it_fail;
+
+ if (!capable(CAP_SYS_RESOURCE))
+ return -EPERM;
+ memset(buffer, 0, sizeof(buffer));
+ if (count > sizeof(buffer) - 1)
+ count = sizeof(buffer) - 1;
+ if (copy_from_user(buffer, buf, count))
+ return -EFAULT;
+ make_it_fail = simple_strtol(buffer, &end, 0);
+ if (*end == '\n')
+ end++;
+ task = get_proc_task(file->f_dentry->d_inode);
+ if (!task)
+ return -ESRCH;
+ task->make_it_fail = make_it_fail;
+ put_task_struct(task);
+ if (end - buffer == 0)
+ return -EIO;
+ return end - buffer;
+}
+
+static struct file_operations proc_fault_inject_operations = {
+ .read = proc_fault_inject_read,
+ .write = proc_fault_inject_write,
+};
+#endif
+
#ifdef CONFIG_SECCOMP
static ssize_t seccomp_read(struct file *file, char __user *buf,
size_t count, loff_t *ppos)
@@ -1834,6 +1905,12 @@ static struct dentry *proc_pident_lookup
inode->i_fop = &proc_loginuid_operations;
break;
#endif
+#ifdef CONFIG_FAULT_INJECTION
+ case PROC_TID_FAULT_INJECTION:
+ case PROC_TGID_FAULT_INJECTION:
+ inode->i_fop = &proc_fault_inject_operations;
+ break;
+#endif
default:
printk("procfs: impossible type (%d)",p->type);
iput(inode);
Index: work-shouldfail/include/linux/sched.h
===================================================================
--- work-shouldfail.orig/include/linux/sched.h
+++ work-shouldfail/include/linux/sched.h
@@ -996,6 +996,9 @@ struct task_struct {
#ifdef CONFIG_TASK_DELAY_ACCT
struct task_delay_info *delays;
#endif
+#ifdef CONFIG_FAULT_INJECTION
+ int make_it_fail;
+#endif
};

static inline pid_t process_group(struct task_struct *tsk)
Index: work-shouldfail/include/linux/fault-inject.h
===================================================================
--- work-shouldfail.orig/include/linux/fault-inject.h
+++ work-shouldfail/include/linux/fault-inject.h
@@ -27,6 +27,9 @@ struct fault_attr {
atomic_t space;

unsigned long count;
+
+ /* A value of '0' means process filter is disabled. */
+ u32 process_filter;
};

#define DEFINE_FAULT_ATTR(name) \
Index: work-shouldfail/lib/fault-inject.c
===================================================================
--- work-shouldfail.orig/lib/fault-inject.c
+++ work-shouldfail/lib/fault-inject.c
@@ -5,6 +5,7 @@
#include <linux/types.h>
#include <linux/fs.h>
#include <linux/module.h>
+#include <linux/interrupt.h>
#include <linux/fault-inject.h>

int setup_fault_attr(struct fault_attr *attr, char *str)
@@ -49,6 +50,15 @@ void should_fail_srandom(unsigned long e
should_fail_random();
}

+static int fail_process(struct fault_attr *attr, struct task_struct *task)
+{
+ /* process filter is disabled */
+ if (!attr->process_filter)
+ return 1;
+
+ return !in_interrupt() && task->make_it_fail;
+}
+
/*
* This code is stolen from failmalloc-1.0
* http://www.nongnu.org/failmalloc/
@@ -56,6 +66,9 @@ void should_fail_srandom(unsigned long e

int should_fail(struct fault_attr *attr, ssize_t size)
{
+ if (!fail_process(attr, current))
+ return 0;
+
if (atomic_read(&max_failures(attr)) == 0)
return 0;


--


2006-09-19 06:00:42

by Don Mullis

[permalink] [raw]
Subject: Re: [patch 7/8] process filtering for fault-injection capabilities

Temporary fix-up so that next patch:
"[patch 8/8] stacktrace filtering for fault-injection capabilities"
applies cleanly.


Signed-off-by: Don Mullis <[email protected]>

---
include/linux/fault-inject.h | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)

Index: linux-2.6.17/include/linux/fault-inject.h
===================================================================
--- linux-2.6.17.orig/include/linux/fault-inject.h
+++ linux-2.6.17/include/linux/fault-inject.h
@@ -16,7 +16,9 @@ struct fault_attr {
atomic_t times;
atomic_t space;
unsigned long count;
- atomic_t process_filter;
+
+ /* A value of '0' means process filter is disabled. */
+ u32 process_filter;
};

#define DEFINE_FAULT_ATTR(name) \


2006-09-19 06:00:27

by Don Mullis

[permalink] [raw]
Subject: Re: [patch 7/8] process filtering for fault-injection capabilities

Add functionality to the process_filter variable: A negative argument
injects failures for only for pid==-process_filter, thereby permitting
per-process failures from boot time.

Restore process-filter arg to kernel command line.

Reintroduce process_filter_file addition to debugfs,
factored out earlier in series.

Add printk, called upon each failure injection.


Signed-off-by: Don Mullis <[email protected]>

---
Documentation/fault-injection/fault-injection.txt | 9 +++++++
include/linux/fault-inject.h | 1
lib/fault-inject-debugfs.c | 11 ++++++++
lib/fault-inject.c | 27 ++++++++++++++++------
4 files changed, 40 insertions(+), 8 deletions(-)

Index: linux-2.6.17/include/linux/fault-inject.h
===================================================================
--- linux-2.6.17.orig/include/linux/fault-inject.h
+++ linux-2.6.17/include/linux/fault-inject.h
@@ -15,11 +15,8 @@ struct fault_attr {
unsigned long interval;
atomic_t times;
atomic_t space;
-
unsigned long count;
-
- /* A value of '0' means process filter is disabled. */
- u32 process_filter;
+ atomic_t process_filter;
};

#define DEFINE_FAULT_ATTR(name) \
Index: linux-2.6.17/lib/fault-inject.c
===================================================================
--- linux-2.6.17.orig/lib/fault-inject.c
+++ linux-2.6.17/lib/fault-inject.c
@@ -14,11 +14,12 @@ int setup_fault_attr(struct fault_attr *
unsigned long interval;
int times;
int space;
+ int process_filter;

- /* "<interval>,<probability>,<space>,<times>" */
- if (sscanf(str, "%lu,%lu,%d,%d",
- &interval, &probability, &space, &times) < 4) {
- printk(KERN_WARNING "SHOULD_FAIL: failed to parse arguments\n");
+ /* "<interval>,<probability>,<process-filter>,<space>,<times>" */
+ if (sscanf(str, "%lu,%lu,%d,%d,%d",
+ &interval, &probability, &process_filter, &space, &times) < 4) {
+ printk(KERN_WARNING "FAULT_INJECTION: failed to parse arguments\n");
return 0;
}

@@ -26,6 +27,7 @@ int setup_fault_attr(struct fault_attr *
attr->interval = interval;
atomic_set(&attr->times, times);
atomic_set(&attr->space, space);
+ atomic_set(&attr->process_filter, process_filter);

return 1;
}
@@ -53,10 +55,20 @@ void should_fail_srandom(unsigned long e
static int fail_process(struct fault_attr *attr, struct task_struct *task)
{
/* process filter is disabled */
- if (!attr->process_filter)
+ if (atomic_read(&attr->process_filter)==0)
return 1;

- return !in_interrupt() && task->make_it_fail;
+ /* controlled by /proc/<pid>/make-it-fail, but boolean not set */
+ if (atomic_read(&attr->process_filter)>0 && !task->make_it_fail)
+ return 0;
+
+ /* enabled for single pid, but no match with pid */
+ if (atomic_read(&attr->process_filter)<0 &&
+ -atomic_read(&attr->process_filter)!=task->pid)
+ return 0;
+
+ /* XX Are we sure we never want calls from interrupt context to fail? */
+ return !in_interrupt();
}

/*
@@ -89,7 +101,8 @@ int should_fail(struct fault_attr *attr,
return 0;

fail:
-
+ printk(KERN_NOTICE "FAULT_INJECTION: forcing a failure, pid==%d\n",
+ current->pid);
if (atomic_read(&max_failures(attr)) != -1)
atomic_dec_not_zero(&max_failures(attr));

Index: linux-2.6.17/Documentation/fault-injection/fault-injection.txt
===================================================================
--- linux-2.6.17.orig/Documentation/fault-injection/fault-injection.txt
+++ linux-2.6.17/Documentation/fault-injection/fault-injection.txt
@@ -52,6 +52,15 @@ configuration of fault-injection capabil
on each call to should_fail(,size). Failure injection is
suppressed until "space" reaches zero.

+- /debug/*/process-filter:
+
+ filter failures by pid.
+ A value of '0' disables filtering by process.
+ Any positive value limits failures to only processes indicated by
+ /proc/<pid>/make-it-fail==1.
+ A negative value means that failures are enabled for
+ pid==-process_filter irrespective of /proc/<pid>/make-it-fail.
+
o Boot option

In order to inject faults while debugfs is not available (early boot time),
Index: linux-2.6.17/lib/fault-inject-debugfs.c
===================================================================
--- linux-2.6.17.orig/lib/fault-inject-debugfs.c
+++ linux-2.6.17/lib/fault-inject-debugfs.c
@@ -8,6 +8,7 @@ struct fault_attr_entries {
struct dentry *interval_file;
struct dentry *times_file;
struct dentry *space_file;
+ struct dentry *process_filter_file;
};

static void debugfs_ul_set(void *data, u64 val)
@@ -66,6 +67,10 @@ static void cleanup_fault_attr_entries(s
debugfs_remove(entries->space_file);
entries->space_file = NULL;
}
+ if (entries->process_filter_file) {
+ debugfs_remove(entries->process_filter_file);
+ entries->process_filter_file = NULL;
+ }
debugfs_remove(entries->dir);
entries->dir = NULL;
}
@@ -105,6 +110,12 @@ static int init_fault_attr_entries(struc
goto fail;
entries->space_file = file;

+ file = debugfs_create_atomic_t("process-filter", mode, dir,
+ &attr->process_filter);
+ if (!file)
+ goto fail;
+ entries->process_filter_file = file;
+
return 0;
fail:
cleanup_fault_attr_entries(entries);


2006-09-19 09:57:14

by Akinobu Mita

[permalink] [raw]
Subject: Re: [patch 7/8] process filtering for fault-injection capabilities

On Mon, Sep 18, 2006 at 10:54:51PM -0700, Don Mullis wrote:
> Add functionality to the process_filter variable: A negative argument
> injects failures for only for pid==-process_filter, thereby permitting
> per-process failures from boot time.
>

Is it better to add new filter for this purpose?
Because someone may want to filter by tgid instead of pid.

- positive value is for task->pid
- nevative value is for task->tgid

> Add printk, called upon each failure injection.
>

This printk() will be very useful. But it is better to make
configurable, and the pid is not reliable in interrupt context.

2006-09-19 17:43:50

by Don Mullis

[permalink] [raw]
Subject: Re: [patch 7/8] process filtering for fault-injection capabilities

On Tue, 2006-09-19 at 17:05 +0800, Akinobu Mita wrote:
> On Mon, Sep 18, 2006 at 10:54:51PM -0700, Don Mullis wrote:
> > Add functionality to the process_filter variable: A negative argument
> > injects failures for only for pid==-process_filter, thereby permitting
> > per-process failures from boot time.
> >
>
> Is it better to add new filter for this purpose?
> Because someone may want to filter by tgid instead of pid.
>
> - positive value is for task->pid
> - nevative value is for task->tgid

Your idea sounds good to me.

> > Add printk, called upon each failure injection.
> >
>
> This printk() will be very useful. But it is better to make
> configurable, and the pid is not reliable in interrupt context.

Okay. We do need some output now, for testing.