Date: Wed, 24 Jun 2015 10:17:45 -0400
From: Steven Rostedt <rostedt@goodmis.org>
To: Josef Bacik <jbacik@fb.com>
Cc: <linux-kernel@vger.kernel.org>, <kernel-team@fb.com>
Subject: Re: [PATCH] trace-cmd: add a kernel memory leak detector
Message-ID: <20150624101745.20669cec@gandalf.local.home>
In-Reply-To: <1435100799-32723-1-git-send-email-jbacik@fb.com>
References: <1435100799-32723-1-git-send-email-jbacik@fb.com>
MIME-Version: 1.0
Content-Type: text/plain; charset=US-ASCII
Content-Transfer-Encoding: 7bit
Sender: linux-kernel-owner@vger.kernel.org
Content-Length: 35326
Lines: 1157

On Tue, 23 Jun 2015 16:06:39 -0700
Josef Bacik <jbacik@fb.com> wrote:

> I needed to track down a very slow memory leak so I adapted the same approach
> trace-cmd profile uses to track kernel memory allocations.  You run this with
> 
> trace-cmd kmemleak
> 
> and then you can kill -SIGUSR2 <trace-cmd pid> to get current status updates, or
> just stop the process when you are ready.  It will tell you how much was lost
> and the size of the objects that were allocated, along with the tracebacks and
> the counts of the allocators.  Thanks,

Thanks! I'll take a look at this today. I'm hoping to release 2.6 soon.

Can you write up another patch that adds a man page for this?

 Documentation/trace-cmd-kmemleak.1.txt

-- Steve

> 
> Signed-off-by: Josef Bacik <jbacik@fb.com>
> ---
>  Makefile         |   2 +-
>  trace-cmd.c      |   3 +-
>  trace-cmd.h      |   6 +
>  trace-hash.h     |   5 +
>  trace-input.c    |  23 +++
>  trace-kmemleak.c | 552 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
>  trace-local.h    |  20 +-
>  trace-profile.c  |  10 +-
>  trace-read.c     |  13 +-
>  trace-record.c   |  71 +++++--
>  trace-stream.c   |  12 +-
>  trace-usage.c    |   6 +
>  12 files changed, 680 insertions(+), 43 deletions(-)
>  create mode 100644 trace-kmemleak.c
> 
> diff --git a/Makefile b/Makefile
> index 402f711..1e3626e 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -308,7 +308,7 @@ TRACE_GUI_OBJS = trace-filter.o trace-compat.o trace-filter-hash.o trace-dialog.
>  		trace-xml.o
>  TRACE_CMD_OBJS = trace-cmd.o trace-record.o trace-read.o trace-split.o trace-listen.o \
>  	 trace-stack.o trace-hist.o trace-mem.o trace-snapshot.o trace-stat.o \
> -	 trace-hash.o trace-profile.o trace-stream.o
> +	 trace-hash.o trace-profile.o trace-stream.o trace-kmemleak.o
>  TRACE_VIEW_OBJS = trace-view.o trace-view-store.o
>  TRACE_GRAPH_OBJS = trace-graph.o trace-plot.o trace-plot-cpu.o trace-plot-task.o
>  TRACE_VIEW_MAIN_OBJS = trace-view-main.o $(TRACE_VIEW_OBJS) $(TRACE_GUI_OBJS)
> diff --git a/trace-cmd.c b/trace-cmd.c
> index 4c5b564..fc30c3c 100644
> --- a/trace-cmd.c
> +++ b/trace-cmd.c
> @@ -483,7 +483,8 @@ int main (int argc, char **argv)
>  		   strcmp(argv[1], "stream") == 0 ||
>  		   strcmp(argv[1], "profile") == 0 ||
>  		   strcmp(argv[1], "restart") == 0 ||
> -		   strcmp(argv[1], "reset") == 0) {
> +		   strcmp(argv[1], "reset") == 0 ||
> +		   strcmp(argv[1], "kmemleak") == 0) {
>  		trace_record(argc, argv);
>  		exit(0);
>  
> diff --git a/trace-cmd.h b/trace-cmd.h
> index 7bce2a5..50e2b79 100644
> --- a/trace-cmd.h
> +++ b/trace-cmd.h
> @@ -103,6 +103,9 @@ struct tracecmd_ftrace {
>  	int long_size;
>  };
>  
> +typedef void (*trace_show_data_func)(struct tracecmd_input *handle,
> +				     struct pevent_record *record);
> +
>  struct tracecmd_input *tracecmd_alloc(const char *file);
>  struct tracecmd_input *tracecmd_alloc_fd(int fd);
>  struct tracecmd_input *tracecmd_open(const char *file);
> @@ -184,6 +187,9 @@ tracecmd_get_cursor(struct tracecmd_input *handle, int cpu);
>  int tracecmd_ftrace_overrides(struct tracecmd_input *handle, struct tracecmd_ftrace *finfo);
>  struct pevent *tracecmd_get_pevent(struct tracecmd_input *handle);
>  bool tracecmd_get_use_trace_clock(struct tracecmd_input *handle);
> +trace_show_data_func tracecmd_get_show_data_func(struct tracecmd_input *handle);
> +void tracecmd_set_show_data_func(struct tracecmd_input *handle,
> +				 trace_show_data_func func);
>  
>  char *tracecmd_get_tracing_file(const char *name);
>  void tracecmd_put_tracing_file(char *name);
> diff --git a/trace-hash.h b/trace-hash.h
> index 2529f4d..2636a9c 100644
> --- a/trace-hash.h
> +++ b/trace-hash.h
> @@ -44,9 +44,14 @@ static inline void trace_hash_del(struct trace_hash_item *item)
>  {
>  	struct trace_hash_item *prev = item->prev;
>  
> +	if (!prev)
> +		return;
> +
>  	prev->next = item->next;
>  	if (item->next)
>  		item->next->prev = prev;
> +	item->next = NULL;
> +	item->prev = NULL;
>  }
>  
>  #define trace_hash_for_each_bucket(bucket, hash)			\
> diff --git a/trace-input.c b/trace-input.c
> index 4120189..bb8076d 100644
> --- a/trace-input.c
> +++ b/trace-input.c
> @@ -37,6 +37,7 @@
>  #include <errno.h>
>  
>  #include "trace-cmd-local.h"
> +#include "trace-local.h"
>  #include "kbuffer.h"
>  #include "list.h"
>  
> @@ -108,6 +109,9 @@ struct tracecmd_input {
>  	size_t			ftrace_files_start;
>  	size_t			event_files_start;
>  	size_t			total_file_size;
> +
> +	/* For custom profilers. */
> +	trace_show_data_func	show_data_func;
>  };
>  
>  __thread struct tracecmd_input *tracecmd_curr_thread_handle;
> @@ -2938,3 +2942,22 @@ bool tracecmd_get_use_trace_clock(struct tracecmd_input *handle)
>  {
>  	return handle->use_trace_clock;
>  }
> +
> +/**
> + * tracecmd_get_show_data_func - return the show data func
> + * @handle: input handle for the trace.dat file
> + */
> +trace_show_data_func tracecmd_get_show_data_func(struct tracecmd_input *handle)
> +{
> +	return handle->show_data_func;
> +}
> +
> +/**
> + * tracecmd_set_show_data_func - set the show data func
> + * @handle: input handle for the trace.dat file
> + */
> +void tracecmd_set_show_data_func(struct tracecmd_input *handle,
> +				 trace_show_data_func func)
> +{
> +	handle->show_data_func = func;
> +}
> diff --git a/trace-kmemleak.c b/trace-kmemleak.c
> new file mode 100644
> index 0000000..2e288fe
> --- /dev/null
> +++ b/trace-kmemleak.c
> @@ -0,0 +1,552 @@
> +#define _LARGEFILE64_SOURCE
> +#include <sys/types.h>
> +#include <unistd.h>
> +#include <stdio.h>
> +#include <getopt.h>
> +#include <signal.h>
> +
> +#include "trace-local.h"
> +#include "trace-hash.h"
> +#include "list.h"
> +
> +#define memory_from_item(item)	container_of(item, struct memory, hash)
> +#define memory_from_phash(item)	container_of(item, struct memory, phash)
> +#define leak_from_item(item)	container_of(item, struct memory_leak, hash)
> +#define edata_from_item(item)	container_of(item, struct event_data, hash)
> +#define stack_from_item(item)	container_of(item, struct stack_trace, hash)
> +
> +struct kmemleak_handle {
> +	struct trace_hash	event_hash;
> +	struct trace_hash	alloc_hash;
> +	struct trace_hash	pid_hash;
> +	struct tracecmd_input	*handle;
> +	struct pevent		*pevent;
> +	struct format_field	*common_pid;
> +	struct kmemleak_handle	*next;
> +};
> +
> +struct memory {
> +	struct trace_hash_item	hash;
> +	struct trace_hash_item	phash;
> +	unsigned long		pid;
> +	unsigned long		ptr;
> +	size_t			alloc_size;
> +	unsigned long		stack_size;
> +	char			*caller;
> +};
> +
> +struct stack_trace {
> +	struct trace_hash_item	hash;
> +	char			*caller;
> +	unsigned long		size;
> +	int			count;
> +};
> +
> +struct memory_leak {
> +	struct trace_hash_item	hash;
> +	size_t			total_lost;
> +	struct trace_hash	stack_hash;
> +};
> +
> +struct event_data;
> +typedef void (*event_handler)(struct kmemleak_handle *h,
> +			      struct pevent_record *record,
> +			      struct event_data *edata);
> +
> +struct event_data {
> +	struct trace_hash_item	hash;
> +	int			id;
> +	struct format_field	*ptr_field;
> +	struct format_field	*data_field;
> +	event_handler		handler;
> +};
> +
> +static struct kmemleak_handle *handles = NULL;
> +static struct kmemleak_handle *last_handle = NULL;
> +
> +static int match_pid(struct trace_hash_item *item, void *data)
> +{
> +	struct memory *mem = memory_from_phash(item);
> +	unsigned long pid = (unsigned long)data;
> +
> +	return mem->pid == pid;
> +}
> +
> +static void handle_kmalloc(struct kmemleak_handle *h,
> +			   struct pevent_record *record,
> +			   struct event_data *edata)
> +{
> +	struct memory *mem;
> +	unsigned long long ptr, size, pid;
> +	int ret;
> +
> +	mem = malloc_or_die(sizeof(*mem));
> +	memset(mem, 0, sizeof(*mem));
> +	ret = pevent_read_number_field(edata->ptr_field, record->data, &ptr);
> +	ret |= pevent_read_number_field(edata->data_field, record->data,
> +					&size);
> +	ret |= pevent_read_number_field(h->common_pid, record->data, &pid);
> +	if (ret)
> +		die("missing important filed in kmalloc");
> +
> +	mem->ptr = ptr;
> +	mem->pid = pid;
> +	mem->alloc_size = size;
> +	mem->hash.key = trace_hash(mem->ptr);
> +	trace_hash_add(&h->alloc_hash, &mem->hash);
> +
> +	mem->phash.key = trace_hash(pid);
> +	trace_hash_add(&h->pid_hash, &mem->phash);
> +}
> +
> +static void handle_stacktrace(struct kmemleak_handle *h,
> +			      struct pevent_record *record,
> +			      struct event_data *edata)
> +{
> +	struct memory *mem;
> +	struct trace_hash_item *item;
> +	void *caller;
> +	unsigned long long size, pid;
> +	int ret;
> +
> +	ret = pevent_read_number_field(h->common_pid, record->data, &pid);
> +	if (ret)
> +		die("missing pid field");
> +	item = trace_hash_find(&h->pid_hash, trace_hash(pid), match_pid,
> +			       (unsigned long *)pid);
> +	if (!item)
> +		return;
> +	mem = memory_from_phash(item);
> +	trace_hash_del(item);
> +
> +	size = record->size - edata->data_field->offset;
> +	caller = record->data + edata->data_field->offset;
> +
> +	mem->stack_size = size;
> +	mem->caller = malloc_or_die(size);
> +	memset(mem->caller, 0, size);
> +	memcpy(mem->caller, caller, size);
> +}
> +
> +static int match_memory(struct trace_hash_item *item, void *data)
> +{
> +	struct memory *mem = memory_from_item(item);
> +	unsigned long ptr = (unsigned long)data;
> +
> +	return mem->ptr == ptr;
> +}
> +
> +static void handle_kfree(struct kmemleak_handle *h,
> +			 struct pevent_record *record,
> +			 struct event_data *edata)
> +{
> +	struct memory *mem;
> +	struct trace_hash_item *item;
> +	unsigned long long ptr;
> +	unsigned long long key;
> +	int ret;
> +
> +	ret = pevent_read_number_field(edata->ptr_field,
> +				       record->data, &ptr);
> +	if (ret)
> +		die("missing important field in kfree/kmemcache_free");
> +
> +	key = trace_hash(ptr);
> +	item = trace_hash_find(&h->alloc_hash, key, match_memory,
> +			       (unsigned long *)ptr);
> +	if (!item)
> +		return;
> +	mem = memory_from_item(item);
> +	trace_hash_del(item);
> +	trace_hash_del(&mem->phash);
> +	if (mem->caller)
> +		free(mem->caller);
> +	free(mem);
> +}
> +
> +static void handle_missed_events(struct kmemleak_handle *h)
> +{
> +	struct trace_hash_item **bucket;
> +	struct trace_hash_item *item;
> +	struct memory *mem;
> +
> +	trace_hash_for_each_bucket(bucket, &h->alloc_hash) {
> +		trace_hash_while_item(item, bucket) {
> +			mem = memory_from_item(item);
> +			trace_hash_del(item);
> +			trace_hash_del(&mem->phash);
> +			if (mem->caller)
> +				free(mem->caller);
> +			free(mem);
> +		}
> +	}
> +
> +	trace_hash_for_each_bucket(bucket, &h->pid_hash) {
> +		trace_hash_while_item(item, bucket) {
> +			mem = memory_from_phash(item);
> +			printf("found something on the pid hash\n");
> +			trace_hash_del(item);
> +			if (mem->caller)
> +				free(mem->caller);
> +			free(mem);
> +		}
> +	}
> +	fprintf(stderr, "Missed events, results won't be accurate\n");
> +}
> +
> +static void usr1_handler(int signum)
> +{
> +	trace_kmemleak_output();
> +}
> +
> +void trace_kmemleak_global_init(void)
> +{
> +	struct sigaction new_action;
> +
> +	new_action.sa_handler = usr1_handler;
> +	sigemptyset(&new_action.sa_mask);
> +	new_action.sa_flags = 0;
> +
> +	sigaction(SIGUSR2, &new_action, NULL);
> +}
> +
> +static int compare_stacks(const void *a, const void *b)
> +{
> +	struct stack_trace * const *A = a;
> +	struct stack_trace * const *B = b;
> +
> +	if ((*A)->count < (*B)->count)
> +		return 1;
> +	else if ((*A)->count > (*B)->count)
> +		return -1;
> +	return 0;
> +}
> +
> +static unsigned long long
> +stack_value(struct stack_trace *stack, int longsize, int level)
> +{
> +	void *ptr;
> +
> +	ptr = &stack->caller[longsize * level];
> +	return longsize == 8 ? *(u64 *)ptr : *(unsigned *)ptr;
> +}
> +
> +static void output_stack(struct kmemleak_handle *h, struct stack_trace *stack)
> +{
> +	const char *func;
> +	unsigned long long val;
> +	int longsize = pevent_get_long_size(h->pevent);
> +	int cnt = stack->size / longsize, i;
> +
> +	printf("\tStack count: %d\n", stack->count);
> +	for (i = 0; i < cnt; i++) {
> +		val = stack_value(stack, longsize, i);
> +		func = pevent_find_function(h->pevent, val);
> +		if (func)
> +			printf("\t\t%s (0x%llx)\n", func, val);
> +		else
> +			printf("\t\t0x%llx\n", val);
> +	}
> +}
> +
> +static void output_stacks(struct kmemleak_handle *h, struct memory_leak *leak)
> +{
> +	struct trace_hash_item **bucket;
> +	struct trace_hash_item *item;
> +	struct stack_trace **stacks;
> +	int nr_stacks = 0, i;
> +
> +	trace_hash_for_each_bucket(bucket, &leak->stack_hash) {
> +		trace_hash_for_each_item(item, bucket)
> +			nr_stacks++;
> +	}
> +
> +	stacks = malloc_or_die(sizeof(*stacks) * nr_stacks);
> +
> +	nr_stacks = 0;
> +	trace_hash_for_each_bucket(bucket, &leak->stack_hash) {
> +		trace_hash_while_item(item, bucket) {
> +			stacks[nr_stacks++] = stack_from_item(item);
> +			trace_hash_del(item);
> +		}
> +	}
> +
> +	qsort(stacks, nr_stacks, sizeof(*stacks), compare_stacks);
> +
> +	for (i = 0; i < nr_stacks; i++) {
> +		output_stack(h, stacks[i]);
> +		free(stacks[i]->caller);
> +		free(stacks[i]);
> +	}
> +	free(stacks);
> +}
> +
> +static void output_leak(struct kmemleak_handle *h, struct memory_leak *leak)
> +{
> +	printf("Leaked %llu bytes of size %llu\n",
> +	       (unsigned long long)leak->total_lost,
> +	       (unsigned long long)leak->hash.key);
> +	output_stacks(h, leak);
> +}
> +
> +struct stack_match {
> +	void		*caller;
> +	unsigned long	size;
> +};
> +
> +static int match_stack(struct trace_hash_item *item, void *data)
> +{
> +	struct stack_trace *stack = stack_from_item(item);
> +	struct stack_match *match = data;
> +
> +	if (match->size != stack->size)
> +		return 0;
> +
> +	return memcmp(stack->caller, match->caller, stack->size) == 0;
> +}
> +
> +static void add_stack(struct memory_leak *leak, void *caller,
> +		      unsigned long size)
> +{
> +	struct trace_hash_item *item;
> +	struct stack_match match;
> +	struct stack_trace *stack;
> +	unsigned long long key;
> +	int i;
> +
> +	match.caller = caller;
> +	match.size = size;
> +
> +	if (size < sizeof(int))
> +		return;
> +
> +	for (key = 0, i = 0; i <= size - sizeof(int); i += sizeof(int))
> +		key += trace_hash(*(int *)(caller + i));
> +
> +	item = trace_hash_find(&leak->stack_hash, key, match_stack, &match);
> +	if (!item) {
> +		stack = malloc_or_die(sizeof(*stack));
> +		memset(stack, 0, sizeof(*stack));
> +		stack->hash.key = key;
> +		stack->caller = malloc_or_die(size);
> +		memcpy(stack->caller, caller, size);
> +		stack->size = size;
> +		stack->count = 1;
> +		trace_hash_add(&leak->stack_hash, &stack->hash);
> +	} else {
> +		stack = stack_from_item(item);
> +		stack->count++;
> +	}
> +}
> +
> +static int compare_leaks(const void *a, const void *b)
> +{
> +	struct memory_leak * const *A = a;
> +	struct memory_leak * const *B = b;
> +
> +	if ((*A)->total_lost < (*B)->total_lost)
> +		return 1;
> +	else if ((*A)->total_lost > (*B)->total_lost)
> +		return -1;
> +	return 0;
> +}
> +
> +static void output_handle(struct kmemleak_handle *h, int *total_leaks)
> +{
> +	struct memory_leak **leaks;
> +	struct trace_hash_item **bucket;
> +	struct trace_hash_item *item;
> +	struct trace_hash leak_hash;
> +	int nr_leaks = 0, i;
> +
> +	/* God I hope we don't have more than 64 leak buckets */
> +	if (trace_hash_init(&leak_hash, 64))
> +		die("Couldn't allocate leak hash table");
> +
> +	trace_hash_for_each_bucket(bucket, &h->alloc_hash) {
> +		trace_hash_for_each_item(item, bucket) {
> +			struct trace_hash_item *tmp;
> +			struct memory *mem;
> +			struct memory_leak *leak;
> +
> +			mem = memory_from_item(item);
> +			tmp = trace_hash_find(&leak_hash, mem->alloc_size,
> +					      NULL, NULL);
> +			if (tmp) {
> +				leak = leak_from_item(tmp);
> +				leak->total_lost += mem->alloc_size;
> +			} else {
> +				leak = malloc_or_die(sizeof(*leak));
> +				memset(leak, 0, sizeof(*leak));
> +				leak->hash.key = mem->alloc_size;
> +				leak->total_lost = mem->alloc_size;
> +				trace_hash_init(&leak->stack_hash, 1024);
> +				trace_hash_add(&leak_hash, &leak->hash);
> +				nr_leaks++;
> +			}
> +			add_stack(leak, mem->caller, mem->stack_size);
> +		}
> +	}
> +
> +	if (!nr_leaks)
> +		return;
> +
> +	leaks = malloc_or_die(sizeof(*leaks) * nr_leaks);
> +	*total_leaks += nr_leaks;
> +	nr_leaks = 0;
> +
> +	trace_hash_for_each_bucket(bucket, &leak_hash) {
> +		trace_hash_while_item(item, bucket) {
> +			leaks[nr_leaks++] = leak_from_item(item);
> +			trace_hash_del(item);
> +		}
> +	}
> +
> +	qsort(leaks, nr_leaks, sizeof(*leaks), compare_leaks);
> +
> +	for (i = 0; i < nr_leaks; i++) {
> +		output_leak(h, leaks[i]);
> +		trace_hash_free(&leaks[i]->stack_hash);
> +		free(leaks[i]);
> +	}
> +
> +	free(leaks);
> +	trace_hash_free(&leak_hash);
> +}
> +
> +static int match_event(struct trace_hash_item *item, void *data)
> +{
> +	struct event_data *edata = edata_from_item(item);
> +	int id = (int)(unsigned long)data;
> +
> +	return edata->id == id;
> +}
> +
> +void trace_kmemleak_record(struct tracecmd_input *handle,
> +			   struct pevent_record *record)
> +{
> +	struct kmemleak_handle *h;
> +	struct pevent *pevent;
> +	struct event_data *edata;
> +	struct trace_hash_item *item;
> +	unsigned long id;
> +
> +	if (last_handle && last_handle->handle == handle)
> +		h = last_handle;
> +	else {
> +		for (h = handles; h; h = h->next) {
> +			if (h->handle == handle)
> +				break;
> +		}
> +		if (!h)
> +			die("Handle not found?");
> +		last_handle = h;
> +	}
> +
> +	pevent = h->pevent;
> +
> +	if (record->missed_events)
> +		handle_missed_events(h);
> +
> +	id = pevent_data_type(pevent, record);
> +	item = trace_hash_find(&h->event_hash, trace_hash(id), match_event,
> +			       (void *)id);
> +	if (!item)
> +		return;
> +	edata = edata_from_item(item);
> +	edata->handler(h, record, edata);
> +}
> +
> +static void setup_fields(struct kmemleak_handle *h)
> +{
> +	struct event_format *event;
> +	struct event_data *edata;
> +	struct pevent *pevent = h->pevent;
> +
> +	edata = malloc_or_die(sizeof(*edata));
> +	memset(edata, 0, sizeof(*edata));
> +	event = pevent_find_event_by_name(pevent, "kmem", "kmalloc");
> +	if (!event)
> +		die("Can't find kmem:kmalloc event");
> +	h->common_pid = pevent_find_common_field(event, "common_pid");
> +	edata->id = event->id;
> +	edata->hash.key = trace_hash(edata->id);
> +	edata->ptr_field = pevent_find_field(event, "ptr");
> +	edata->data_field = pevent_find_field(event, "bytes_alloc");
> +	edata->handler = handle_kmalloc;
> +	if (!edata->ptr_field || !edata->data_field)
> +		die("Missing key fields");
> +	trace_hash_add(&h->event_hash, &edata->hash);
> +
> +	edata = malloc_or_die(sizeof(*edata));
> +	memset(edata, 0, sizeof(*edata));
> +	event = pevent_find_event_by_name(pevent, "kmem", "kfree");
> +	if (!event)
> +		die("Can't find kmem:kfree event");
> +	edata->id = event->id;
> +	edata->hash.key = trace_hash(edata->id);
> +	edata->ptr_field = pevent_find_field(event, "ptr");
> +	edata->handler = handle_kfree;
> +	if (!edata->ptr_field)
> +		die("Missing key kfree fields");
> +	trace_hash_add(&h->event_hash, &edata->hash);
> +
> +	edata = malloc_or_die(sizeof(*edata));
> +	memset(edata, 0, sizeof(*edata));
> +	event = pevent_find_event_by_name(pevent, "kmem", "kmem_cache_free");
> +	if (!event)
> +		die("Can't find kmem:kmem_cache_free event");
> +	edata->id = event->id;
> +	edata->hash.key = trace_hash(edata->id);
> +	edata->ptr_field = pevent_find_field(event, "ptr");
> +	edata->handler = handle_kfree;
> +	if (!edata->ptr_field)
> +		die("Missing key kmem_cache_free field");
> +	trace_hash_add(&h->event_hash, &edata->hash);
> +
> +	edata = malloc_or_die(sizeof(*edata));
> +	memset(edata, 0, sizeof(*edata));
> +	event = pevent_find_event_by_name(pevent, "ftrace", "kernel_stack");
> +	if (!event)
> +		die("Can't find ftrace:kernel_stack event");
> +	edata->id = event->id;
> +	edata->hash.key = trace_hash(edata->id);
> +	edata->data_field = pevent_find_field(event, "caller");
> +	edata->handler = handle_stacktrace;
> +	if (!edata->data_field)
> +		die("Missing caller in stack trace");
> +	trace_hash_add(&h->event_hash, &edata->hash);
> +}
> +
> +void trace_init_kmemleak(struct tracecmd_input *handle, struct hook_list *hook,
> +			 int global)
> +{
> +	struct pevent *pevent = tracecmd_get_pevent(handle);
> +	struct kmemleak_handle *h;
> +
> +	tracecmd_set_show_data_func(handle, trace_kmemleak_record);
> +	h = malloc_or_die(sizeof(*h));
> +	memset(h, 0, sizeof(*h));
> +	h->next = handles;
> +	handles = h;
> +
> +	trace_hash_init(&h->alloc_hash, 1024);
> +	trace_hash_init(&h->pid_hash, 1024);
> +	trace_hash_init(&h->event_hash, 16);
> +	h->handle = handle;
> +	h->pevent = pevent;
> +
> +	setup_fields(h);
> +}
> +
> +void trace_kmemleak_output(void)
> +{
> +	struct kmemleak_handle *h;
> +	int leaks = 0;
> +
> +	printf("Printing kmemleak summary\n");
> +	for (h = handles; h; h = h->next)
> +		output_handle(h, &leaks);
> +	if (!leaks)
> +		printf("Hooray no leakage!\n");
> +}
> diff --git a/trace-local.h b/trace-local.h
> index d9a4fac..89ad06f 100644
> --- a/trace-local.h
> +++ b/trace-local.h
> @@ -47,6 +47,9 @@ struct pid_record_data {
>  	struct pevent_record	*record;
>  };
>  
> +typedef void (*handle_init_func)(struct tracecmd_input *handle,
> +				 struct hook_list *hook, int global);
> +
>  void show_file(const char *name);
>  
>  struct tracecmd_input *read_trace_header(const char *file);
> @@ -76,21 +79,23 @@ void trace_stat(int argc, char **argv);
>  
>  struct hook_list;
>  
> -int trace_profile_record(struct tracecmd_input *handle,
> -			 struct pevent_record *record, int cpu);
>  void trace_init_profile(struct tracecmd_input *handle, struct hook_list *hooks,
>  			int global);
>  int trace_profile(void);
>  void trace_profile_set_merge_like_comms(void);
>  
> +void trace_init_kmemleak(struct tracecmd_input *handle, struct hook_list *hook,
> +			 int global);
> +void trace_kmemleak_global_init(void);
> +void trace_kmemleak_output(void);
> +
>  struct tracecmd_input *
>  trace_stream_init(struct buffer_instance *instance, int cpu, int fd, int cpus,
> -		  int profile, struct hook_list *hooks, int global);
> -int trace_stream_read(struct pid_record_data *pids, int nr_pids, struct timeval *tv,
> -		      int profile);
> +		  struct hook_list *hooks, handle_init_func handle_init,
> +		  int global);
> +int trace_stream_read(struct pid_record_data *pids, int nr_pids, struct timeval *tv);
>  
> -void trace_show_data(struct tracecmd_input *handle, struct pevent_record *record,
> -		     int profile);
> +void trace_show_data(struct tracecmd_input *handle, struct pevent_record *record);
>  
>  /* --- event interation --- */
>  
> @@ -167,6 +172,7 @@ struct buffer_instance {
>  	int			keep;
>  	int			buffer_size;
>  	int			profile;
> +	int			kmemleak;
>  };
>  
>  extern struct buffer_instance top_instance;
> diff --git a/trace-profile.c b/trace-profile.c
> index 2356701..aa54f0d 100644
> --- a/trace-profile.c
> +++ b/trace-profile.c
> @@ -698,8 +698,8 @@ find_event_data(struct handle_data *h, int id)
>  	return NULL;
>  }
>  
> -int trace_profile_record(struct tracecmd_input *handle,
> -			 struct pevent_record *record, int cpu)
> +static void trace_profile_record(struct tracecmd_input *handle,
> +				struct pevent_record *record)
>  {
>  	static struct handle_data *last_handle;
>  	struct pevent_record *stack_record;
> @@ -708,6 +708,7 @@ int trace_profile_record(struct tracecmd_input *handle,
>  	struct handle_data *h;
>  	struct pevent *pevent;
>  	unsigned long long pid;
> +	int cpu = record->cpu;
>  	int id;
>  
>  	if (last_handle && last_handle->handle == handle)
> @@ -732,7 +733,7 @@ int trace_profile_record(struct tracecmd_input *handle,
>  	event_data = find_event_data(h, id);
>  
>  	if (!event_data)
> -		return -1;
> +		return;
>  
>  
>  	/* Get this current PID */
> @@ -751,8 +752,6 @@ int trace_profile_record(struct tracecmd_input *handle,
>  		free_record(stack_record);
>  		task->last_stack = NULL;
>  	}
> -
> -	return 0;
>  }
>  
>  static struct event_data *
> @@ -1225,6 +1224,7 @@ void trace_init_profile(struct tracecmd_input *handle, struct hook_list *hook,
>  	int ret;
>  	int i;
>  
> +	tracecmd_set_show_data_func(handle, trace_profile_record);
>  	h = malloc_or_die(sizeof(*h));
>  	memset(h, 0, sizeof(*h));
>  	h->next = handles;
> diff --git a/trace-read.c b/trace-read.c
> index f4dffd6..103bc0c 100644
> --- a/trace-read.c
> +++ b/trace-read.c
> @@ -737,22 +737,21 @@ static void finish_wakeup(void)
>  	trace_hash_free(&wakeup_hash);
>  }
>  
> -void trace_show_data(struct tracecmd_input *handle, struct pevent_record *record,
> -		     int profile)
> +void trace_show_data(struct tracecmd_input *handle, struct pevent_record *record)
>  {
> +	trace_show_data_func func = tracecmd_get_show_data_func(handle);
>  	struct pevent *pevent;
>  	struct trace_seq s;
>  	int cpu = record->cpu;
>  	bool use_trace_clock;
>  
> -	pevent = tracecmd_get_pevent(handle);
> -
>  	test_save(record, cpu);
>  
> -	if (profile) {
> -		trace_profile_record(handle, record, cpu);
> +	if (func) {
> +		func(handle, record);
>  		return;
>  	}
> +	pevent = tracecmd_get_pevent(handle);
>  
>  	trace_seq_init(&s);
>  	if (record->missed_events > 0)
> @@ -1109,7 +1108,7 @@ static void read_data_info(struct list_head *handle_list, enum output_type otype
>  		}
>  		if (last_record) {
>  			print_handle_file(last_handle);
> -			trace_show_data(last_handle->handle, last_record, profile);
> +			trace_show_data(last_handle->handle, last_record);
>  			free_handle_record(last_handle);
>  		}
>  	} while (last_record);
> diff --git a/trace-record.c b/trace-record.c
> index 102bfe1..cd36585 100644
> --- a/trace-record.c
> +++ b/trace-record.c
> @@ -67,9 +67,10 @@ enum trace_type {
>  	TRACE_TYPE_START	= (1 << 1),
>  	TRACE_TYPE_STREAM	= (1 << 2),
>  	TRACE_TYPE_EXTRACT	= (1 << 3),
> -	TRACE_TYPE_PROFILE	= (1 << 4) | TRACE_TYPE_STREAM,
>  };
>  
> +static handle_init_func handle_init = NULL;
> +
>  static int rt_prio;
>  
>  static int use_tcp;
> @@ -489,7 +490,6 @@ static void delete_thread_data(void)
>  static void stop_threads(enum trace_type type)
>  {
>  	struct timeval tv = { 0, 0 };
> -	int profile = (type & TRACE_TYPE_PROFILE) == TRACE_TYPE_PROFILE;
>  	int ret;
>  	int i;
>  
> @@ -506,7 +506,7 @@ static void stop_threads(enum trace_type type)
>  	/* Flush out the pipes */
>  	if (type & TRACE_TYPE_STREAM) {
>  		do {
> -			ret = trace_stream_read(pids, recorder_threads, &tv, profile);
> +			ret = trace_stream_read(pids, recorder_threads, &tv);
>  		} while (ret > 0);
>  	}
>  
> @@ -839,7 +839,6 @@ static pid_t trace_waitpid(enum trace_type type, pid_t pid, int *status, int opt
>  {
>  	struct timeval tv = { 1, 0 };
>  	int ret;
> -	int profile = (type & TRACE_TYPE_PROFILE) == TRACE_TYPE_PROFILE;
>  
>  	if (type & TRACE_TYPE_STREAM)
>  		options |= WNOHANG;
> @@ -850,7 +849,7 @@ static pid_t trace_waitpid(enum trace_type type, pid_t pid, int *status, int opt
>  			return ret;
>  
>  		if (type & TRACE_TYPE_STREAM)
> -			trace_stream_read(pids, recorder_threads, &tv, profile);
> +			trace_stream_read(pids, recorder_threads, &tv);
>  	} while (1);
>  }
>  #ifndef NO_PTRACE
> @@ -1008,12 +1007,11 @@ static inline void ptrace_attach(int pid) { }
>  static void trace_or_sleep(enum trace_type type)
>  {
>  	struct timeval tv = { 1 , 0 };
> -	int profile = (type & TRACE_TYPE_PROFILE) == TRACE_TYPE_PROFILE;
>  
>  	if (do_ptrace && filter_pid >= 0)
>  		ptrace_wait(type, filter_pid);
>  	else if (type & TRACE_TYPE_STREAM)
> -		trace_stream_read(pids, recorder_threads, &tv, profile);
> +		trace_stream_read(pids, recorder_threads, &tv);
>  	else
>  		sleep(10);
>  }
> @@ -2494,7 +2492,6 @@ static void finish_network(void)
>  
>  static void start_threads(enum trace_type type, int global)
>  {
> -	int profile = (type & TRACE_TYPE_PROFILE) == TRACE_TYPE_PROFILE;
>  	struct buffer_instance *instance;
>  	int *brass = NULL;
>  	int i = 0;
> @@ -2518,7 +2515,7 @@ static void start_threads(enum trace_type type, int global)
>  					die("pipe");
>  				pids[i].stream = trace_stream_init(instance, x,
>  								   brass[0], cpu_count,
> -								   profile, hooks,
> +								   hooks, handle_init,
>  								   global);
>  				if (!pids[i].stream)
>  					die("Creating stream for %d", i);
> @@ -3334,7 +3331,8 @@ static void check_function_plugin(void)
>  
>  static int __check_doing_something(struct buffer_instance *instance)
>  {
> -	return instance->profile || instance->plugin || instance->events;
> +	return instance->kmemleak || instance->profile || instance->plugin ||
> +		instance->events;
>  }
>  
>  static void check_doing_something(void)
> @@ -3654,6 +3652,26 @@ static void enable_profile(struct buffer_instance *instance)
>  		profile_add_event(instance, events[i], 0);
>  }
>  
> +static void enable_kmemleak(struct buffer_instance *instance)
> +{
> +	int stacktrace = 0, i;
> +	char *events[] = {
> +		"kmem:kfree",
> +		"kmem:kmem_cache_free",
> +		NULL,
> +	};
> +
> +	if (test_stacktrace_trigger(instance))
> +		stacktrace = 1;
> +	else
> +		save_option("stacktrace");
> +
> +	profile_add_event(instance, "kmem:kmalloc", stacktrace);
> +
> +	for (i = 0; events[i]; i++)
> +		profile_add_event(instance, events[i], 0);
> +}
> +
>  static struct event_list *
>  create_hook_event(struct buffer_instance *instance,
>  		  const char *system, const char *event)
> @@ -3703,6 +3721,7 @@ static void add_hook(struct buffer_instance *instance, const char *arg)
>  }
>  
>  enum {
> +	OPT_kmemleak	= 249,
>  	OPT_bycomm	= 250,
>  	OPT_stderr	= 251,
>  	OPT_profile	= 252,
> @@ -3738,7 +3757,7 @@ void trace_record (int argc, char **argv)
>  	int neg_event = 0;
>  	int date = 0;
>  	int manual = 0;
> -
> +	int kmemleak = 0;
>  	int c;
>  
>  	init_instance(instance);
> @@ -3754,8 +3773,11 @@ void trace_record (int argc, char **argv)
>  	else if ((stream = strcmp(argv[1], "stream") == 0))
>  		; /* do nothing */
>  	else if ((profile = strcmp(argv[1], "profile") == 0)) {
> +		handle_init = trace_init_profile;
> +		events = 1;
> +	} else if ((kmemleak = strcmp(argv[1], "kmemleak") == 0)) {
> +		handle_init = trace_init_kmemleak;
>  		events = 1;
> -
>  	} else if (strcmp(argv[1], "stop") == 0) {
>  		int topt = 0;
>  		for (;;) {
> @@ -3873,6 +3895,7 @@ void trace_record (int argc, char **argv)
>  			{"profile", no_argument, NULL, OPT_profile},
>  			{"stderr", no_argument, NULL, OPT_stderr},
>  			{"by-comm", no_argument, NULL, OPT_bycomm},
> +			{"kmemleak", no_argument, NULL, OPT_kmemleak},
>  			{"help", no_argument, NULL, '?'},
>  			{NULL, 0, NULL, 0}
>  		};
> @@ -4002,7 +4025,7 @@ void trace_record (int argc, char **argv)
>  				die("only one output file allowed");
>  			output = optarg;
>  
> -			if (profile) {
> +			if (profile || kmemleak) {
>  				int fd;
>  
>  				/* pipe the output to this file instead of stdout */
> @@ -4076,6 +4099,8 @@ void trace_record (int argc, char **argv)
>  			add_instance(instance);
>  			if (profile)
>  				instance->profile = 1;
> +			if (kmemleak)
> +				instance->kmemleak = 1;
>  			break;
>  		case 'k':
>  			keep = 1;
> @@ -4093,6 +4118,7 @@ void trace_record (int argc, char **argv)
>  			recorder_flags |= TRACECMD_RECORD_NOSPLICE;
>  			break;
>  		case OPT_profile:
> +			handle_init = trace_init_profile;
>  			instance->profile = 1;
>  			events = 1;
>  			break;
> @@ -4107,6 +4133,11 @@ void trace_record (int argc, char **argv)
>  		case OPT_bycomm:
>  			trace_profile_set_merge_like_comms();
>  			break;
> +		case OPT_kmemleak:
> +			handle_init = trace_init_kmemleak;
> +			events = 1;
> +			instance->kmemleak = 1;
> +			break;
>  		default:
>  			usage(argv);
>  		}
> @@ -4131,6 +4162,8 @@ void trace_record (int argc, char **argv)
>  	 */
>  	if (profile && !buffer_instances)
>  		top_instance.profile = 1;
> +	if (kmemleak && !buffer_instances)
> +		top_instance.kmemleak = 1;
>  
>  	/*
>  	 * If top_instance doesn't have any plugins or events, then
> @@ -4154,6 +4187,10 @@ void trace_record (int argc, char **argv)
>  
>  		if (!manual && instance->profile)
>  			enable_profile(instance);
> +		if (!manual && instance->kmemleak) {
> +			trace_kmemleak_global_init();
> +			enable_kmemleak(instance);
> +		}
>  
>  		instance->tracing_on_init_val = read_tracing_on(instance);
>  		/* Some instances may not be created yet */
> @@ -4203,8 +4240,9 @@ void trace_record (int argc, char **argv)
>  	else if (extract)
>  		type = TRACE_TYPE_EXTRACT;
>  	else if (profile)
> -		/* PROFILE includes the STREAM bit */
> -		type = TRACE_TYPE_PROFILE;
> +		type = TRACE_TYPE_STREAM;
> +	else if (kmemleak)
> +		type = TRACE_TYPE_STREAM;
>  	else
>  		type = TRACE_TYPE_START;
>  
> @@ -4295,6 +4333,7 @@ void trace_record (int argc, char **argv)
>  
>  	if (profile)
>  		trace_profile();
> -
> +	if (kmemleak)
> +		trace_kmemleak_output();
>  	exit(0);
>  }
> diff --git a/trace-stream.c b/trace-stream.c
> index 9ebe65b..b103fda 100644
> --- a/trace-stream.c
> +++ b/trace-stream.c
> @@ -35,7 +35,8 @@
>   */
>  struct tracecmd_input *
>  trace_stream_init(struct buffer_instance *instance, int cpu, int fd, int cpus,
> -		  int profile, struct hook_list *hooks, int global)
> +		  struct hook_list *hooks, handle_init_func handle_init,
> +		  int global)
>  {
>  	struct tracecmd_input *trace_input;
>  	struct tracecmd_output *trace_output;
> @@ -75,8 +76,8 @@ trace_stream_init(struct buffer_instance *instance, int cpu, int fd, int cpus,
>  	if (tracecmd_read_headers(trace_input) < 0)
>  		goto fail_free_input;
>  
> -	if (profile)
> -		trace_init_profile(trace_input, hooks, global);
> +	if (handle_init)
> +		handle_init(trace_input, hooks, global);
>  
>   make_pipe:
>  	/* Do not block on this pipe */
> @@ -98,8 +99,7 @@ trace_stream_init(struct buffer_instance *instance, int cpu, int fd, int cpus,
>  	return NULL;
>  }
>  
> -int trace_stream_read(struct pid_record_data *pids, int nr_pids, struct timeval *tv,
> -		      int profile)
> +int trace_stream_read(struct pid_record_data *pids, int nr_pids, struct timeval *tv)
>  {
>  	struct pevent_record *record;
>  	struct pid_record_data *pid;
> @@ -127,7 +127,7 @@ int trace_stream_read(struct pid_record_data *pids, int nr_pids, struct timeval
>  			last_pid = pid;
>  	}
>  	if (last_pid) {
> -		trace_show_data(last_pid->instance->handle, last_pid->record, profile);
> +		trace_show_data(last_pid->instance->handle, last_pid->record);
>  		free_record(last_pid->record);
>  		last_pid->record = NULL;
>  		return 1;
> diff --git a/trace-usage.c b/trace-usage.c
> index bdd5727..27235e5 100644
> --- a/trace-usage.c
> +++ b/trace-usage.c
> @@ -169,6 +169,12 @@ static struct usage_help usage_help[] = {
>  		"          -H Allows users to hook two events together for timings\n"
>  	},
>  	{
> +		"kmemleak",
> +		"Streaming kmemleak detector",
> +		" %s kmemleak\n"
> +		"          Uses same options as record\n"
> +	},
> +	{
>  		"hist",
>  		"show a historgram of the trace.dat information",
>  		" %s hist [-i file][-P] [file]"

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/