Events Tracing infrastructure contains lot of files, directories
(internally in terms of inodes, dentries). And ends up by consuming
memory in MBs. We can have multiple events of Events Tracing, which
further requires more memory.
Instead of creating inodes/dentries, eventfs could keep meta-data and
skip the creation of inodes/dentries. As and when require, eventfs will
create the inodes/dentries only for required files/directories.
Also eventfs would delete the inodes/dentries once no more requires
but preserve the meta data.
Tracing events took ~9MB, with this approach it took ~4.5MB
for ~10K files/dir.
v2:
Patch 01: new patch:'Require all trace events to have a TRACE_SYSTEM'
Patch 02: moved from v1 1/9
Patch 03: moved from v1 2/9
As suggested by Zheng Yejian, introduced eventfs_prepare_ef()
helper function to add files or directories to eventfs
fix WARNING reported by kernel test robot in v1 8/9
Patch 04: moved from v1 3/9
used eventfs_prepare_ef() to add files
fix WARNING reported by kernel test robot in v1 8/9
Patch 05: moved from v1 4/9
fix compiling warning reported by kernel test robot in v1 4/9
Patch 06: moved from v1 5/9
Patch 07: moved from v1 6/9
Patch 08: moved from v1 7/9
Patch 09: moved from v1 8/9
rebased because of v3 01/10
Patch 10: moved from v1 9/9
v1:
Patch 1: add header file
Patch 2: resolved kernel test robot issues
protecting eventfs lists using nested eventfs_rwsem
Patch 3: protecting eventfs lists using nested eventfs_rwsem
Patch 4: improve events cleanup code to fix crashes
Patch 5: resolved kernel test robot issues
removed d_instantiate_anon() calls
Patch 6: resolved kernel test robot issues
fix kprobe test in eventfs_root_lookup()
protecting eventfs lists using nested eventfs_rwsem
Patch 7: remove header file
Patch 8: pass eventfs_rwsem as argument to eventfs functions
called eventfs_remove_events_dir() instead of tracefs_remove()
from event_trace_del_tracer()
Patch 9: new patch to fix kprobe test case
fs/tracefs/Makefile | 1 +
fs/tracefs/event_inode.c | 757 ++++++++++++++++++
fs/tracefs/inode.c | 124 ++-
fs/tracefs/internal.h | 25 +
include/linux/trace_events.h | 1 +
include/linux/tracefs.h | 49 ++
kernel/trace/trace.h | 3 +-
kernel/trace/trace_events.c | 78 +-
.../ftrace/test.d/kprobe/kprobe_args_char.tc | 4 +-
.../test.d/kprobe/kprobe_args_string.tc | 4 +-
10 files changed, 994 insertions(+), 52 deletions(-)
create mode 100644 fs/tracefs/event_inode.c
create mode 100644 fs/tracefs/internal.h
--
2.40.0
Till now /sys/kernel/debug/tracing/events is a part of tracefs,
with-in this patch creating 'events' and it's sub-dir as eventfs.
Basically replacing tracefs calls with eventfs calls for 'events'.
Signed-off-by: Ajay Kaher <[email protected]>
Co-developed-by: Steven Rostedt (VMware) <[email protected]>
Signed-off-by: Steven Rostedt (VMware) <[email protected]>
Tested-by: Ching-lin Yu <[email protected]>
---
fs/tracefs/inode.c | 18 ++++++++++
include/linux/trace_events.h | 1 +
kernel/trace/trace.h | 2 +-
kernel/trace/trace_events.c | 67 +++++++++++++++++++-----------------
4 files changed, 55 insertions(+), 33 deletions(-)
diff --git a/fs/tracefs/inode.c b/fs/tracefs/inode.c
index 76820d3e9..a098d7153 100644
--- a/fs/tracefs/inode.c
+++ b/fs/tracefs/inode.c
@@ -374,6 +374,23 @@ static const struct super_operations tracefs_super_operations = {
.show_options = tracefs_show_options,
};
+static void tracefs_dentry_iput(struct dentry *dentry, struct inode *inode)
+{
+ struct tracefs_inode *ti;
+
+ if (!dentry || !inode)
+ return;
+
+ ti = get_tracefs(inode);
+ if (ti && ti->flags & TRACEFS_EVENT_INODE)
+ eventfs_set_ef_status_free(dentry);
+ iput(inode);
+}
+
+static const struct dentry_operations tracefs_dentry_operations = {
+ .d_iput = tracefs_dentry_iput,
+};
+
static int trace_fill_super(struct super_block *sb, void *data, int silent)
{
static const struct tree_descr trace_files[] = {{""}};
@@ -396,6 +413,7 @@ static int trace_fill_super(struct super_block *sb, void *data, int silent)
goto fail;
sb->s_op = &tracefs_super_operations;
+ sb->s_d_op = &tracefs_dentry_operations;
tracefs_apply_options(sb, false);
diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h
index 0e373222a..696843d46 100644
--- a/include/linux/trace_events.h
+++ b/include/linux/trace_events.h
@@ -635,6 +635,7 @@ struct trace_event_file {
struct list_head list;
struct trace_event_call *event_call;
struct event_filter __rcu *filter;
+ struct eventfs_file *ef;
struct dentry *dir;
struct trace_array *tr;
struct trace_subsystem_dir *system;
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index b895c3346..b265ae2df 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -1310,7 +1310,7 @@ struct trace_subsystem_dir {
struct list_head list;
struct event_subsystem *subsystem;
struct trace_array *tr;
- struct dentry *entry;
+ struct eventfs_file *ef;
int ref_count;
int nr_events;
};
diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c
index 16bc5ba45..94aa6f9c9 100644
--- a/kernel/trace/trace_events.c
+++ b/kernel/trace/trace_events.c
@@ -988,7 +988,8 @@ static void remove_subsystem(struct trace_subsystem_dir *dir)
return;
if (!--dir->nr_events) {
- tracefs_remove(dir->entry);
+ if (dir->ef)
+ eventfs_remove(dir->ef);
list_del(&dir->list);
__put_system_dir(dir);
}
@@ -1009,7 +1010,8 @@ static void remove_event_file_dir(struct trace_event_file *file)
tracefs_remove(dir);
}
-
+ if (file->ef)
+ eventfs_remove(file->ef);
list_del(&file->list);
remove_subsystem(file->system);
free_event_filter(file->filter);
@@ -2295,13 +2297,13 @@ create_new_subsystem(const char *name)
return NULL;
}
-static struct dentry *
+static struct eventfs_file *
event_subsystem_dir(struct trace_array *tr, const char *name,
struct trace_event_file *file, struct dentry *parent)
{
struct event_subsystem *system, *iter;
struct trace_subsystem_dir *dir;
- struct dentry *entry;
+ int res;
/* First see if we did not already create this dir */
list_for_each_entry(dir, &tr->systems, list) {
@@ -2309,7 +2311,7 @@ event_subsystem_dir(struct trace_array *tr, const char *name,
if (strcmp(system->name, name) == 0) {
dir->nr_events++;
file->system = dir;
- return dir->entry;
+ return dir->ef;
}
}
@@ -2333,8 +2335,8 @@ event_subsystem_dir(struct trace_array *tr, const char *name,
} else
__get_system(system);
- dir->entry = tracefs_create_dir(name, parent);
- if (!dir->entry) {
+ dir->ef = eventfs_add_subsystem_dir(name, parent, &tr->eventfs_rwsem);
+ if (IS_ERR(dir->ef)) {
pr_warn("Failed to create system directory %s\n", name);
__put_system(system);
goto out_free;
@@ -2349,22 +2351,22 @@ event_subsystem_dir(struct trace_array *tr, const char *name,
/* the ftrace system is special, do not create enable or filter files */
if (strcmp(name, "ftrace") != 0) {
- entry = tracefs_create_file("filter", TRACE_MODE_WRITE,
- dir->entry, dir,
+ res = eventfs_add_file("filter", TRACE_MODE_WRITE,
+ dir->ef, dir,
&ftrace_subsystem_filter_fops);
- if (!entry) {
+ if (res) {
kfree(system->filter);
system->filter = NULL;
pr_warn("Could not create tracefs '%s/filter' entry\n", name);
}
- trace_create_file("enable", TRACE_MODE_WRITE, dir->entry, dir,
+ eventfs_add_file("enable", TRACE_MODE_WRITE, dir->ef, dir,
&ftrace_system_enable_fops);
}
list_add(&dir->list, &tr->systems);
- return dir->entry;
+ return dir->ef;
out_free:
kfree(dir);
@@ -2418,7 +2420,7 @@ event_create_dir(struct dentry *parent, struct trace_event_file *file)
{
struct trace_event_call *call = file->event_call;
struct trace_array *tr = file->tr;
- struct dentry *d_events;
+ struct eventfs_file *ef_subsystem = NULL;
const char *name;
int ret;
@@ -2430,24 +2432,24 @@ event_create_dir(struct dentry *parent, struct trace_event_file *file)
if (WARN_ON_ONCE(strcmp(call->class->system, TRACE_SYSTEM) == 0))
return -ENODEV;
- d_events = event_subsystem_dir(tr, call->class->system, file, parent);
- if (!d_events)
+ ef_subsystem = event_subsystem_dir(tr, call->class->system, file, parent);
+ if (!ef_subsystem)
return -ENOMEM;
name = trace_event_name(call);
- file->dir = tracefs_create_dir(name, d_events);
- if (!file->dir) {
+ file->ef = eventfs_add_dir(name, ef_subsystem, &tr->eventfs_rwsem);
+ if (IS_ERR(file->ef)) {
pr_warn("Could not create tracefs '%s' directory\n", name);
return -1;
}
if (call->class->reg && !(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE))
- trace_create_file("enable", TRACE_MODE_WRITE, file->dir, file,
+ eventfs_add_file("enable", TRACE_MODE_WRITE, file->ef, file,
&ftrace_enable_fops);
#ifdef CONFIG_PERF_EVENTS
if (call->event.type && call->class->reg)
- trace_create_file("id", TRACE_MODE_READ, file->dir,
+ eventfs_add_file("id", TRACE_MODE_READ, file->ef,
(void *)(long)call->event.type,
&ftrace_event_id_fops);
#endif
@@ -2463,27 +2465,27 @@ event_create_dir(struct dentry *parent, struct trace_event_file *file)
* triggers or filters.
*/
if (!(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)) {
- trace_create_file("filter", TRACE_MODE_WRITE, file->dir,
+ eventfs_add_file("filter", TRACE_MODE_WRITE, file->ef,
file, &ftrace_event_filter_fops);
- trace_create_file("trigger", TRACE_MODE_WRITE, file->dir,
+ eventfs_add_file("trigger", TRACE_MODE_WRITE, file->ef,
file, &event_trigger_fops);
}
#ifdef CONFIG_HIST_TRIGGERS
- trace_create_file("hist", TRACE_MODE_READ, file->dir, file,
+ eventfs_add_file("hist", TRACE_MODE_READ, file->ef, file,
&event_hist_fops);
#endif
#ifdef CONFIG_HIST_TRIGGERS_DEBUG
- trace_create_file("hist_debug", TRACE_MODE_READ, file->dir, file,
+ eventfs_add_file("hist_debug", TRACE_MODE_READ, file->ef, file,
&event_hist_debug_fops);
#endif
- trace_create_file("format", TRACE_MODE_READ, file->dir, call,
+ eventfs_add_file("format", TRACE_MODE_READ, file->ef, call,
&ftrace_event_format_fops);
#ifdef CONFIG_TRACE_EVENT_INJECT
if (call->event.type && call->class->reg)
- trace_create_file("inject", 0200, file->dir, file,
+ eventfs_add_file("inject", 0200, file->ef, file,
&event_inject_fops);
#endif
@@ -3636,21 +3638,22 @@ create_event_toplevel_files(struct dentry *parent, struct trace_array *tr)
{
struct dentry *d_events;
struct dentry *entry;
+ int error = 0;
entry = trace_create_file("set_event", TRACE_MODE_WRITE, parent,
tr, &ftrace_set_event_fops);
if (!entry)
return -ENOMEM;
- d_events = tracefs_create_dir("events", parent);
- if (!d_events) {
+ d_events = eventfs_create_events_dir("events", parent, &tr->eventfs_rwsem);
+ if (IS_ERR(d_events)) {
pr_warn("Could not create tracefs 'events' directory\n");
return -ENOMEM;
}
- entry = trace_create_file("enable", TRACE_MODE_WRITE, d_events,
+ error = eventfs_add_top_file("enable", TRACE_MODE_WRITE, d_events,
tr, &ftrace_tr_enable_fops);
- if (!entry)
+ if (error)
return -ENOMEM;
/* There are not as crucial, just warn if they are not created */
@@ -3663,11 +3666,11 @@ create_event_toplevel_files(struct dentry *parent, struct trace_array *tr)
&ftrace_set_event_notrace_pid_fops);
/* ring buffer internal formats */
- trace_create_file("header_page", TRACE_MODE_READ, d_events,
+ eventfs_add_top_file("header_page", TRACE_MODE_READ, d_events,
ring_buffer_print_page_header,
&ftrace_show_header_fops);
- trace_create_file("header_event", TRACE_MODE_READ, d_events,
+ eventfs_add_top_file("header_event", TRACE_MODE_READ, d_events,
ring_buffer_print_entry_header,
&ftrace_show_header_fops);
@@ -3755,7 +3758,7 @@ int event_trace_del_tracer(struct trace_array *tr)
down_write(&trace_event_sem);
__trace_remove_event_dirs(tr);
- tracefs_remove(tr->event_dir);
+ eventfs_remove_events_dir(tr->event_dir);
up_write(&trace_event_sem);
tr->event_dir = NULL;
--
2.40.0
> On 01-Jun-2023, at 2:30 PM, Ajay Kaher <[email protected]> wrote:
>
> Events Tracing infrastructure contains lot of files, directories
> (internally in terms of inodes, dentries). And ends up by consuming
> memory in MBs. We can have multiple events of Events Tracing, which
> further requires more memory.
>
> Instead of creating inodes/dentries, eventfs could keep meta-data and
> skip the creation of inodes/dentries. As and when require, eventfs will
> create the inodes/dentries only for required files/directories.
> Also eventfs would delete the inodes/dentries once no more requires
> but preserve the meta data.
>
> Tracing events took ~9MB, with this approach it took ~4.5MB
> for ~10K files/dir.
>
Steve, I have used nested rw-semaphore for eventfs locking (same as in cifs).
As per Amit Nadav, this has to be revisited/reviewed. Please have a look and
share your thoughts.
-Ajay
Hello,
kernel test robot noticed "WARNING:possible_circular_locking_dependency_detected" on:
commit: a3bb763435d444023d3bca5479da632c57724619 ("[PATCH v3 09/10] eventfs: moving tracing/events to eventfs")
url: https://github.com/intel-lab-lkp/linux/commits/Ajay-Kaher/tracing-Require-all-trace-events-to-have-a-TRACE_SYSTEM/20230601-230657
base: https://git.kernel.org/cgit/linux/kernel/git/shuah/linux-kselftest.git next
patch link: https://lore.kernel.org/all/[email protected]/
patch subject: [PATCH v3 09/10] eventfs: moving tracing/events to eventfs
in testcase: kernel-selftests
version: kernel-selftests-x86_64-60acb023-1_20230329
with following parameters:
group: ftrace
test-description: The kernel contains a set of "self tests" under the tools/testing/selftests/ directory. These are intended to be small unit tests to exercise individual code paths in the kernel.
test-url: https://www.kernel.org/doc/Documentation/kselftest.txt
compiler: gcc-12
test machine: 36 threads 1 sockets Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz (Cascade Lake) with 32G memory
(please refer to attached dmesg/kmsg for entire log/backtrace)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-lkp/[email protected]
kern :warn : [ 173.884312] WARNING: possible circular locking dependency detected
kern :warn : [ 173.884947] 6.4.0-rc1-00014-ga3bb763435d4 #1 Not tainted
kern :warn : [ 173.885501] ------------------------------------------------------
kern :warn : [ 173.886125] ftracetest/2186 is trying to acquire lock:
kern :warn : [ 173.886665] ffff88810045d368 (&sb->s_type->i_mutex_key#5){++++}-{3:3}, at: dcache_dir_open_wrapper (fs/tracefs/event_inode.c:373)
kern :warn : [ 173.887638]
but task is already holding lock:
kern :warn : [ 173.888299] ffffffff84e6d640 (eventfs_rwsem/1){.+.+}-{3:3}, at: dcache_dir_open_wrapper (fs/tracefs/event_inode.c:364)
kern :warn : [ 173.889183]
which lock already depends on the new lock.
kern :warn : [ 173.890101]
the existing dependency chain (in reverse order) is:
kern :warn : [ 173.890898]
-> #1 (eventfs_rwsem/1){.+.+}-{3:3}:
kern :warn : [ 173.891600] lock_acquire (kernel/locking/lockdep.c:467 kernel/locking/lockdep.c:5693 kernel/locking/lockdep.c:5656)
kern :warn : [ 173.892066] down_read_nested (kernel/locking/rwsem.c:1263 kernel/locking/rwsem.c:1646)
kern :warn : [ 173.892553] eventfs_root_lookup (fs/tracefs/event_inode.c:283)
kern :warn : [ 173.893058] __lookup_slow (include/linux/dcache.h:359 include/linux/dcache.h:364 fs/namei.c:1691)
kern :warn : [ 173.893529] walk_component (include/linux/fs.h:790 fs/namei.c:1708 fs/namei.c:1998)
kern :warn : [ 173.894006] path_lookupat (fs/namei.c:2455 fs/namei.c:2479)
kern :warn : [ 173.894476] filename_lookup (fs/namei.c:2508)
kern :warn : [ 173.894974] vfs_statx (fs/stat.c:239)
kern :warn : [ 173.895410] vfs_fstatat (fs/stat.c:277)
kern :warn : [ 173.895851] __do_sys_newfstatat (fs/stat.c:447)
kern :warn : [ 173.896350] do_syscall_64 (arch/x86/entry/common.c:50 arch/x86/entry/common.c:80)
kern :warn : [ 173.896815] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:120)
kern :warn : [ 173.897392]
-> #0 (&sb->s_type->i_mutex_key#5){++++}-{3:3}:
kern :warn : [ 173.898158] check_prev_add (kernel/locking/lockdep.c:3109)
kern :warn : [ 173.898643] __lock_acquire (kernel/locking/lockdep.c:3228 kernel/locking/lockdep.c:3842 kernel/locking/lockdep.c:5074)
kern :warn : [ 173.899133] lock_acquire (kernel/locking/lockdep.c:467 kernel/locking/lockdep.c:5693 kernel/locking/lockdep.c:5656)
kern :warn : [ 173.899610] down_write (arch/x86/include/asm/preempt.h:80 kernel/locking/rwsem.c:1304 kernel/locking/rwsem.c:1315 kernel/locking/rwsem.c:1574)
kern :warn : [ 173.900054] dcache_dir_open_wrapper (fs/tracefs/event_inode.c:373)
kern :warn : [ 173.900603] do_dentry_open (fs/open.c:920)
kern :warn : [ 173.901081] do_open (fs/namei.c:3636)
kern :warn : [ 173.901508] path_openat (fs/namei.c:3792)
kern :warn : [ 173.901963] do_filp_open (fs/namei.c:3818)
kern :warn : [ 173.902425] do_sys_openat2 (fs/open.c:1356)
kern :warn : [ 173.902902] __x64_sys_openat (fs/open.c:1383)
kern :warn : [ 173.903408] do_syscall_64 (arch/x86/entry/common.c:50 arch/x86/entry/common.c:80)
kern :warn : [ 173.903864] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:120)
kern :warn : [ 173.904451]
other info that might help us debug this:
kern :warn : [ 173.905372] Possible unsafe locking scenario:
kern :warn : [ 173.906049] CPU0 CPU1
kern :warn : [ 173.906538] ---- ----
kern :warn : [ 173.907027] rlock(eventfs_rwsem/1);
kern :warn : [ 173.907464] lock(&sb->s_type->i_mutex_key#5);
kern :warn : [ 173.908171] lock(eventfs_rwsem/1);
kern :warn : [ 173.908800] lock(&sb->s_type->i_mutex_key#5);
kern :warn : [ 173.909291]
*** DEADLOCK ***
kern :warn : [ 173.910026] 1 lock held by ftracetest/2186:
kern :warn : [ 173.910488] #0: ffffffff84e6d640 (eventfs_rwsem/1){.+.+}-{3:3}, at: dcache_dir_open_wrapper (fs/tracefs/event_inode.c:364)
kern :warn : [ 173.911413]
stack backtrace:
kern :warn : [ 173.911955] CPU: 13 PID: 2186 Comm: ftracetest Not tainted 6.4.0-rc1-00014-ga3bb763435d4 #1
kern :warn : [ 173.912777] Hardware name: Gigabyte Technology Co., Ltd. X299 UD4 Pro/X299 UD4 Pro-CF, BIOS F8a 04/27/2021
kern :warn : [ 173.913694] Call Trace:
kern :warn : [ 173.914014] <TASK>
kern :warn : [ 173.914305] dump_stack_lvl (lib/dump_stack.c:108)
kern :warn : [ 173.914727] check_noncircular (kernel/locking/lockdep.c:2191)
kern :warn : [ 173.915199] ? print_circular_bug (kernel/locking/lockdep.c:2167)
kern :warn : [ 173.915676] ? mark_usage (kernel/locking/lockdep.c:4930)
kern :warn : [ 173.916100] check_prev_add (kernel/locking/lockdep.c:3109)
kern :warn : [ 173.916553] __lock_acquire (kernel/locking/lockdep.c:3228 kernel/locking/lockdep.c:3842 kernel/locking/lockdep.c:5074)
kern :warn : [ 173.917003] ? mark_usage (kernel/locking/lockdep.c:4930)
kern :warn : [ 173.917425] ? __lock_acquire (arch/x86/include/asm/bitops.h:228 arch/x86/include/asm/bitops.h:240 include/asm-generic/bitops/instrumented-non-atomic.h:142 kernel/locking/lockdep.c:228 kernel/locking/lockdep.c:3754 kernel/locking/lockdep.c:3810 kernel/locking/lockdep.c:5074)
kern :warn : [ 173.917889] lock_acquire (kernel/locking/lockdep.c:467 kernel/locking/lockdep.c:5693 kernel/locking/lockdep.c:5656)
kern :warn : [ 173.918309] ? dcache_dir_open_wrapper (fs/tracefs/event_inode.c:373)
kern :warn : [ 173.918822] ? lock_sync (kernel/locking/lockdep.c:5659)
kern :warn : [ 173.919253] ? preempt_notifier_dec (kernel/sched/core.c:10108)
kern :warn : [ 173.919729] ? lock_sync (kernel/locking/lockdep.c:5659)
kern :warn : [ 173.920144] down_write (arch/x86/include/asm/preempt.h:80 kernel/locking/rwsem.c:1304 kernel/locking/rwsem.c:1315 kernel/locking/rwsem.c:1574)
kern :warn : [ 173.920552] ? dcache_dir_open_wrapper (fs/tracefs/event_inode.c:373)
kern :warn : [ 173.921064] ? down_write_killable_nested (kernel/locking/rwsem.c:1571)
kern :warn : [ 173.921599] ? __lock_release+0x111/0x440
kern :warn : [ 173.922097] ? do_dentry_open (fs/open.c:872)
kern :warn : [ 173.922540] dcache_dir_open_wrapper (fs/tracefs/event_inode.c:373)
kern :warn : [ 173.923039] ? fsnotify_perm+0x13b/0x4a0
kern :warn : [ 173.923548] do_dentry_open (fs/open.c:920)
kern :warn : [ 173.923982] ? eventfs_create_dir (fs/tracefs/event_inode.c:349)
kern :warn : [ 173.924470] ? may_open (fs/namei.c:3265)
kern :warn : [ 173.924869] do_open (fs/namei.c:3636)
kern :warn : [ 173.925255] path_openat (fs/namei.c:3792)
kern :warn : [ 173.926598] ? path_lookupat (fs/namei.c:3773)
kern :warn : [ 173.927039] ? .slowpath (kernel/locking/qspinlock.c:317)
kern :warn : [ 173.927454] ? lock_acquire (kernel/locking/lockdep.c:467 kernel/locking/lockdep.c:5693 kernel/locking/lockdep.c:5656)
kern :warn : [ 173.927889] ? __create_object (mm/kmemleak.c:678)
kern :warn : [ 173.928347] do_filp_open (fs/namei.c:3818)
kern :warn : [ 173.928768] ? may_open_dev (fs/namei.c:3812)
kern :warn : [ 173.929202] ? _raw_spin_unlock (arch/x86/include/asm/preempt.h:95 include/linux/spinlock_api_smp.h:143 kernel/locking/spinlock.c:186)
kern :warn : [ 173.929651] ? alloc_fd (fs/file.c:555 (discriminator 10))
kern :warn : [ 173.930060] ? getname_flags (fs/namei.c:206)
kern :warn : [ 173.930544] do_sys_openat2 (fs/open.c:1356)
kern :warn : [ 173.930979] ? build_open_flags (fs/open.c:1342)
kern :warn : [ 173.931460] __x64_sys_openat (fs/open.c:1383)
kern :warn : [ 173.931909] ? __ia32_compat_sys_open (fs/open.c:1383)
kern :warn : [ 173.932416] ? lockdep_hardirqs_on_prepare (kernel/locking/lockdep.c:4539)
kern :warn : [ 173.933016] ? syscall_enter_from_user_mode (arch/x86/include/asm/irqflags.h:42 arch/x86/include/asm/irqflags.h:77 kernel/entry/common.c:111)
kern :warn : [ 173.933552] do_syscall_64 (arch/x86/entry/common.c:50 arch/x86/entry/common.c:80)
kern :warn : [ 173.933966] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:120)
kern :warn : [ 173.934500] RIP: 0033:0x7fc11f551e41
kern :warn : [ 173.934913] Code: 44 24 18 31 c0 41 83 e2 40 75 3e 89 f0 25 00 00 41 00 3d 00 00 41 00 74 30 89 f2 b8 01 01 00 00 48 89 fe bf 9c ff ff ff 0f 05 <48> 3d 00 f0 ff ff 77 3f 48 8b 54 24 18 64 48 2b 14 25 28 00 00 00
All code
========
0: 44 24 18 rex.R and $0x18,%al
3: 31 c0 xor %eax,%eax
5: 41 83 e2 40 and $0x40,%r10d
9: 75 3e jne 0x49
b: 89 f0 mov %esi,%eax
d: 25 00 00 41 00 and $0x410000,%eax
12: 3d 00 00 41 00 cmp $0x410000,%eax
17: 74 30 je 0x49
19: 89 f2 mov %esi,%edx
1b: b8 01 01 00 00 mov $0x101,%eax
20: 48 89 fe mov %rdi,%rsi
23: bf 9c ff ff ff mov $0xffffff9c,%edi
28: 0f 05 syscall
2a:* 48 3d 00 f0 ff ff cmp $0xfffffffffffff000,%rax <-- trapping instruction
30: 77 3f ja 0x71
32: 48 8b 54 24 18 mov 0x18(%rsp),%rdx
37: 64 48 2b 14 25 28 00 sub %fs:0x28,%rdx
3e: 00 00
Code starting with the faulting instruction
===========================================
0: 48 3d 00 f0 ff ff cmp $0xfffffffffffff000,%rax
6: 77 3f ja 0x47
8: 48 8b 54 24 18 mov 0x18(%rsp),%rdx
d: 64 48 2b 14 25 28 00 sub %fs:0x28,%rdx
14: 00 00
To reproduce:
git clone https://github.com/intel/lkp-tests.git
cd lkp-tests
sudo bin/lkp install job.yaml # job file is attached in this email
bin/lkp split-job --compatible job.yaml # generate the yaml file for lkp run
sudo bin/lkp run generated-yaml-file
# if come across any failure that blocks the test,
# please remove ~/.lkp and /lkp dir to run from a clean state.
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
> Hello,
>
> kernel test robot noticed "WARNING:possible_circular_locking_dependency_detected" on:
>
> commit: a3bb763435d444023d3bca5479da632c57724619 ("[PATCH v3 09/10] eventfs: moving tracing/events to eventfs")
> url: https://github.com/intel-lab-lkp/linux/commits/Ajay-Kaher/tracing-Require-all-trace-events-to-have-a-TRACE_SYSTEM/20230601-230657
> base: https://git.kernel.org/cgit/linux/kernel/git/shuah/linux-kselftest.git next
> patch link: [email protected]/">https://lore.kernel.org/all/[email protected]/
> patch subject: [PATCH v3 09/10] eventfs: moving tracing/events to eventfs
>
> in testcase: kernel-selftests
> version: kernel-selftests-x86_64-60acb023-1_20230329
> with following parameters:
>
> group: ftrace
>
> test-description: The kernel contains a set of "self tests" under the tools/testing/selftests/ directory. These are intended to be small unit tests to exercise individual code paths in the kernel.
> test-url: https://www.kernel.org/doc/Documentation/kselftest.txt
>
>
> compiler: gcc-12
> test machine: 36 threads 1 sockets Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz (Cascade Lake) with 32G memory
>
> (please refer to attached dmesg/kmsg for entire log/backtrace)
>
>
>
> If you fix the issue in a separate patch/commit (i.e. not just a new version of
> the same patch/commit), kindly add following tags
> | Reported-by: kernel test robot <[email protected]>
> | Closes: [email protected]">https://lore.kernel.org/oe-lkp/[email protected]
>
>
> kern :warn : [ 173.884312] WARNING: possible circular locking dependency detected
> kern :warn : [ 173.884947] 6.4.0-rc1-00014-ga3bb763435d4 #1 Not tainted
> kern :warn : [ 173.885501] ------------------------------------------------------
> kern :warn : [ 173.886125] ftracetest/2186 is trying to acquire lock:
> kern :warn : [ 173.886665] ffff88810045d368 (&sb->s_type->i_mutex_key#5){++++}-{3:3}, at: dcache_dir_open_wrapper (fs/tracefs/event_inode.c:373)
> kern :warn : [ 173.887638]
> but task is already holding lock:
> kern :warn : [ 173.888299] ffffffff84e6d640 (eventfs_rwsem/1){.+.+}-{3:3}, at: dcache_dir_open_wrapper (fs/tracefs/event_inode.c:364)
> kern :warn : [ 173.889183]
> which lock already depends on the new lock.
>
> kern :warn : [ 173.890101]
> the existing dependency chain (in reverse order) is:
> kern :warn : [ 173.890898]
> -> #1 (eventfs_rwsem/1){.+.+}-{3:3}:
> kern :warn : [ 173.891600] lock_acquire (kernel/locking/lockdep.c:467 kernel/locking/lockdep.c:5693 kernel/locking/lockdep.c:5656)
> kern :warn : [ 173.892066] down_read_nested (kernel/locking/rwsem.c:1263 kernel/locking/rwsem.c:1646)
> kern :warn : [ 173.892553] eventfs_root_lookup (fs/tracefs/event_inode.c:283)
> kern :warn : [ 173.893058] __lookup_slow (include/linux/dcache.h:359 include/linux/dcache.h:364 fs/namei.c:1691)
> kern :warn : [ 173.893529] walk_component (include/linux/fs.h:790 fs/namei.c:1708 fs/namei.c:1998)
> kern :warn : [ 173.894006] path_lookupat (fs/namei.c:2455 fs/namei.c:2479)
> kern :warn : [ 173.894476] filename_lookup (fs/namei.c:2508)
> kern :warn : [ 173.894974] vfs_statx (fs/stat.c:239)
> kern :warn : [ 173.895410] vfs_fstatat (fs/stat.c:277)
> kern :warn : [ 173.895851] __do_sys_newfstatat (fs/stat.c:447)
> kern :warn : [ 173.896350] do_syscall_64 (arch/x86/entry/common.c:50 arch/x86/entry/common.c:80)
> kern :warn : [ 173.896815] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:120)
> kern :warn : [ 173.897392]
> -> #0 (&sb->s_type->i_mutex_key#5){++++}-{3:3}:
> kern :warn : [ 173.898158] check_prev_add (kernel/locking/lockdep.c:3109)
> kern :warn : [ 173.898643] __lock_acquire (kernel/locking/lockdep.c:3228 kernel/locking/lockdep.c:3842 kernel/locking/lockdep.c:5074)
> kern :warn : [ 173.899133] lock_acquire (kernel/locking/lockdep.c:467 kernel/locking/lockdep.c:5693 kernel/locking/lockdep.c:5656)
> kern :warn : [ 173.899610] down_write (arch/x86/include/asm/preempt.h:80 kernel/locking/rwsem.c:1304 kernel/locking/rwsem.c:1315 kernel/locking/rwsem.c:1574)
> kern :warn : [ 173.900054] dcache_dir_open_wrapper (fs/tracefs/event_inode.c:373)
> kern :warn : [ 173.900603] do_dentry_open (fs/open.c:920)
> kern :warn : [ 173.901081] do_open (fs/namei.c:3636)
> kern :warn : [ 173.901508] path_openat (fs/namei.c:3792)
> kern :warn : [ 173.901963] do_filp_open (fs/namei.c:3818)
> kern :warn : [ 173.902425] do_sys_openat2 (fs/open.c:1356)
> kern :warn : [ 173.902902] __x64_sys_openat (fs/open.c:1383)
> kern :warn : [ 173.903408] do_syscall_64 (arch/x86/entry/common.c:50 arch/x86/entry/common.c:80)
> kern :warn : [ 173.903864] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:120)
> kern :warn : [ 173.904451]
> other info that might help us debug this:
>
> kern :warn : [ 173.905372] Possible unsafe locking scenario:
>
> kern :warn : [ 173.906049] CPU0 CPU1
> kern :warn : [ 173.906538] ---- ----
> kern :warn : [ 173.907027] rlock(eventfs_rwsem/1);
> kern :warn : [ 173.907464] lock(&sb->s_type->i_mutex_key#5);
> kern :warn : [ 173.908171] lock(eventfs_rwsem/1);
> kern :warn : [ 173.908800] lock(&sb->s_type->i_mutex_key#5);
> kern :warn : [ 173.909291]
> *** DEADLOCK ***
Steve, this seems not to be a problem here as dcache_dir_open_wrapper()
and eventfs_root_lookup() both lock eventfs_rwsem as read lock, however
it’s known problem in Lockdep for rwlock:
https://lpc.events/event/2/contributions/74/
And available patchset on Lockdep not upstreamed:
https://lore.kernel.org/all/[email protected]/
-Ajay
> On 01-Jun-2023, at 2:30 PM, Ajay Kaher <[email protected]> wrote:
>
> Events Tracing infrastructure contains lot of files, directories
> (internally in terms of inodes, dentries). And ends up by consuming
> memory in MBs. We can have multiple events of Events Tracing, which
> further requires more memory.
>
> Instead of creating inodes/dentries, eventfs could keep meta-data and
> skip the creation of inodes/dentries. As and when require, eventfs will
> create the inodes/dentries only for required files/directories.
> Also eventfs would delete the inodes/dentries once no more requires
> but preserve the meta data.
>
> Tracing events took ~9MB, with this approach it took ~4.5MB
> for ~10K files/dir.
>
> v2:
> Patch 01: new patch:'Require all trace events to have a TRACE_SYSTEM'
> Patch 02: moved from v1 1/9
> Patch 03: moved from v1 2/9
> As suggested by Zheng Yejian, introduced eventfs_prepare_ef()
> helper function to add files or directories to eventfs
> fix WARNING reported by kernel test robot in v1 8/9
> Patch 04: moved from v1 3/9
> used eventfs_prepare_ef() to add files
> fix WARNING reported by kernel test robot in v1 8/9
> Patch 05: moved from v1 4/9
> fix compiling warning reported by kernel test robot in v1 4/9
> Patch 06: moved from v1 5/9
> Patch 07: moved from v1 6/9
> Patch 08: moved from v1 7/9
> Patch 09: moved from v1 8/9
> rebased because of v3 01/10
> Patch 10: moved from v1 9/9
>
> v1:
> Patch 1: add header file
> Patch 2: resolved kernel test robot issues
> protecting eventfs lists using nested eventfs_rwsem
> Patch 3: protecting eventfs lists using nested eventfs_rwsem
> Patch 4: improve events cleanup code to fix crashes
> Patch 5: resolved kernel test robot issues
> removed d_instantiate_anon() calls
> Patch 6: resolved kernel test robot issues
> fix kprobe test in eventfs_root_lookup()
> protecting eventfs lists using nested eventfs_rwsem
> Patch 7: remove header file
> Patch 8: pass eventfs_rwsem as argument to eventfs functions
> called eventfs_remove_events_dir() instead of tracefs_remove()
> from event_trace_del_tracer()
> Patch 9: new patch to fix kprobe test case
>
> fs/tracefs/Makefile | 1 +
> fs/tracefs/event_inode.c | 757 ++++++++++++++++++
> fs/tracefs/inode.c | 124 ++-
> fs/tracefs/internal.h | 25 +
> include/linux/trace_events.h | 1 +
> include/linux/tracefs.h | 49 ++
> kernel/trace/trace.h | 3 +-
> kernel/trace/trace_events.c | 78 +-
> .../ftrace/test.d/kprobe/kprobe_args_char.tc | 4 +-
> .../test.d/kprobe/kprobe_args_string.tc | 4 +-
> 10 files changed, 994 insertions(+), 52 deletions(-)
> create mode 100644 fs/tracefs/event_inode.c
> create mode 100644 fs/tracefs/internal.h
>
Hi Steve, below is the ftracetest results with v3 of eventfs:
root@photon-6 [ ~/linux-6.3-rc5/tools/testing/selftests/ftrace ]# ./ftracetest
=== Ftrace unit tests ===
[1] Basic trace file check [PASS]
[2] Basic test for tracers [PASS]
[3] Basic trace clock test [PASS]
[4] Basic event tracing check [PASS]
[5] Change the ringbuffer size [PASS]
[6] Snapshot and tracing setting [UNSUPPORTED]
[7] trace_pipe and trace_marker [PASS]
[8] Test ftrace direct functions against tracers [UNRESOLVED]
[9] Test ftrace direct functions against kprobes [UNRESOLVED]
[10] Generic dynamic event - add/remove eprobe events [PASS]
[11] Generic dynamic event - add/remove kprobe events [PASS]
[12] Generic dynamic event - add/remove synthetic events [UNSUPPORTED]
[13] Generic dynamic event - selective clear (compatibility) [UNSUPPORTED]
[14] Event probe event parser error log check [PASS]
[15] Generic dynamic event - generic clear event [UNSUPPORTED]
[16] Generic dynamic event - check if duplicate events are caught [PASS]
[17] event tracing - enable/disable with event level files [PASS]
[18] event tracing - restricts events based on pid notrace filtering [PASS]
[19] event tracing - restricts events based on pid [PASS]
[20] event tracing - enable/disable with subsystem level files [PASS]
[21] event tracing - enable/disable with top level files [PASS]
[22] Test trace_printk from module [UNRESOLVED]
[23] event filter function - test event filtering on functions [FAIL]
[24] ftrace - function graph filters with stack tracer [UNSUPPORTED]
[25] ftrace - function graph filters [PASS]
[26] ftrace - function trace with cpumask [PASS]
[27] ftrace - test for function event triggers [PASS]
[28] ftrace - function glob filters [PASS]
[29] ftrace - function pid notrace filters [PASS]
[30] ftrace - function pid filters [PASS]
[31] ftrace - stacktrace filter command [PASS]
[32] ftrace - function trace on module [UNRESOLVED]
[33] ftrace - function profiler with function tracing [UNSUPPORTED]
[34] ftrace - function profiling [UNSUPPORTED]
[35] ftrace - test reading of set_ftrace_filter [PASS]
[36] ftrace - Max stack tracer [UNSUPPORTED]
[37] ftrace - test for function traceon/off triggers [PASS]
[38] ftrace - test tracing error log support [PASS]
[39] Test creation and deletion of trace instances while setting an event [PASS]
[40] Test creation and deletion of trace instances [PASS]
[41] Kprobe dynamic event - adding and removing [PASS]
[42] Kprobe dynamic event - busy event check [PASS]
[43] Kprobe event char type argument [PASS]
[44] Kprobe event with comm arguments [PASS]
[45] Kprobe event string type argument [PASS]
[46] Kprobe event symbol argument [PASS]
[47] Kprobe event argument syntax [PASS]
[48] Kprobe dynamic event with arguments [PASS]
[49] Kprobes event arguments with types [PASS]
[50] Kprobe event user-memory access [PASS]
[51] Kprobe event auto/manual naming [PASS]
[52] Kprobe dynamic event with function tracer [UNSUPPORTED]
[53] Kprobe dynamic event - probing module [UNRESOLVED]
[54] Create/delete multiprobe on kprobe event [PASS]
[55] Kprobe event parser error log check [PASS]
[56] Kretprobe dynamic event with arguments [PASS]
[57] Kretprobe dynamic event with maxactive [PASS]
[58] Kretprobe %return suffix test [PASS]
[59] Register/unregister many kprobe events [PASS]
[60] Kprobe events - probe points [FAIL]
[61] Kprobe profile [PASS]
[62] Uprobe event parser error log check [PASS]
[63] test for the preemptirqsoff tracer [UNSUPPORTED]
[64] Meta-selftest: Checkbashisms [UNRESOLVED]
[65] Test wakeup RT tracer [UNSUPPORTED]
[66] Test wakeup tracer [UNSUPPORTED]
[67] event trigger - test inter-event histogram trigger expected fail actions [UNSUPPORTED]
[68] event trigger - test field variable support [UNSUPPORTED]
[69] event trigger - test inter-event combined histogram trigger [UNSUPPORTED]
[70] event trigger - test multiple actions on hist trigger [UNSUPPORTED]
[71] event trigger - test inter-event histogram trigger onchange action [UNSUPPORTED]
[72] event trigger - test inter-event histogram trigger onmatch action [UNSUPPORTED]
[73] event trigger - test inter-event histogram trigger onmatch-onmax action [UNSUPPORTED]
[74] event trigger - test inter-event histogram trigger onmax action [UNSUPPORTED]
[75] event trigger - test inter-event histogram trigger snapshot action [UNSUPPORTED]
[76] event trigger - test inter-event histogram trigger eprobe on synthetic event [UNSUPPORTED]
[77] event trigger - test synthetic event create remove [UNSUPPORTED]
[78] event trigger - test inter-event histogram trigger trace action with dynamic string param [UNSUPPORTED]
[79] event trigger - test inter-event histogram trigger trace action with dynamic string param [UNSUPPORTED]
[80] event trigger - test synthetic_events syntax parser errors [UNSUPPORTED]
[81] event trigger - test synthetic_events syntax parser [UNSUPPORTED]
[82] event trigger - test inter-event histogram trigger trace action [UNSUPPORTED]
[83] event trigger - test event enable/disable trigger [PASS]
[84] event trigger - test trigger filter [PASS]
[85] event trigger - test histogram expression parsing [UNSUPPORTED]
[86] event trigger - test histogram modifiers [UNSUPPORTED]
[87] event trigger - test histogram parser errors [UNSUPPORTED]
[88] event trigger - test histogram trigger [UNSUPPORTED]
[89] event trigger - test multiple histogram triggers [UNSUPPORTED]
[90] event trigger - test snapshot-trigger [UNSUPPORTED]
[91] event trigger - test stacktrace-trigger [PASS]
[92] trace_marker trigger - test histogram trigger [UNSUPPORTED]
[93] trace_marker trigger - test snapshot trigger [UNSUPPORTED]
[94] trace_marker trigger - test histogram with synthetic event against kernel event [UNSUPPORTED]
[95] trace_marker trigger - test histogram with synthetic event [UNSUPPORTED]
[96] event trigger - test traceon/off trigger [PASS]
[97] (instance) Basic test for tracers [PASS]
[98] (instance) Basic trace clock test [PASS]
[99] (instance) Change the ringbuffer size [PASS]
[100] (instance) Snapshot and tracing setting [UNSUPPORTED]
[101] (instance) trace_pipe and trace_marker [PASS]
[102] (instance) event tracing - enable/disable with event level files [PASS]
[103] (instance) event tracing - restricts events based on pid notrace filtering [PASS]
[104] (instance) event tracing - restricts events based on pid [PASS]
[105] (instance) event tracing - enable/disable with subsystem level files [PASS]
[106] (instance) event filter function - test event filtering on functions [FAIL]
[107] (instance) ftrace - test for function event triggers [PASS]
[108] (instance) ftrace - function pid notrace filters [PASS]
[109] (instance) ftrace - function pid filters [PASS]
[110] (instance) ftrace - stacktrace filter command [PASS]
[111] (instance) ftrace - test for function traceon/off triggers [PASS]
[112] (instance) event trigger - test event enable/disable trigger [PASS]
[113] (instance) event trigger - test trigger filter [PASS]
[114] (instance) event trigger - test histogram modifiers [UNSUPPORTED]
[115] (instance) event trigger - test histogram trigger [UNSUPPORTED]
[116] (instance) event trigger - test multiple histogram triggers [UNSUPPORTED]
[117] (instance) trace_marker trigger - test histogram trigger [UNSUPPORTED]
[118] (instance) trace_marker trigger - test snapshot trigger [UNSUPPORTED]
# of passed: 65
# of failed: 3
# of unresolved: 6
# of untested: 0
# of unsupported: 44
# of xfailed: 0
# of undefined(test bug): 0
These results are same with/without eventfs.
-Ajay
On Mon, 19 Jun 2023 05:38:25 +0000
Ajay Kaher <[email protected]> wrote:
> # of passed: 65
> # of failed: 3
Unrelated to your patches, but have you checked why these fail? Do you have
the latest tests running on the latest kernel?
> # of unresolved: 6
> # of untested: 0
> # of unsupported: 44
> # of xfailed: 0
> # of undefined(test bug): 0
>
> These results are same with/without eventfs.
I'm hoping to look at these patches this week.
Thanks!
-- Steve
> On 20-Jun-2023, at 8:32 PM, Steven Rostedt <[email protected]> wrote:
>
> !! External Email
>
> On Mon, 19 Jun 2023 05:38:25 +0000
> Ajay Kaher <[email protected]> wrote:
>
>> # of passed: 65
>> # of failed: 3
>
> Unrelated to your patches, but have you checked why these fail? Do you have
> the latest tests running on the latest kernel?
>
Failed test got passed after enabling /proc/kallsyms, using:
echo 0 > /proc/sys/kernel/kptr_restrict
Following is the report of ftracetest on Linux v6.4.0-rc7 (with/without eventfs):
# of passed: 68
# of failed: 0
# of unresolved: 6
# of untested: 0
# of unsupported: 45
# of xfailed: 0
# of undefined(test bug): 0
If lockdep is enabled getting same warning as reported by 'kernel test robot' for v3 09/10:
https://lore.kernel.org/all/[email protected]/
>> # of unresolved: 6
>> # of untested: 0
>> # of unsupported: 44
>> # of xfailed: 0
>> # of undefined(test bug): 0
>>
>> These results are same with/without eventfs.
>
> I'm hoping to look at these patches this week.
Yes, please. Thanks.
-Ajay
On Wed, 21 Jun 2023 11:42:24 +0000
Ajay Kaher <[email protected]> wrote:
>
>
> > On 20-Jun-2023, at 8:32 PM, Steven Rostedt <[email protected]> wrote:
> >
> > !! External Email
> >
> > On Mon, 19 Jun 2023 05:38:25 +0000
> > Ajay Kaher <[email protected]> wrote:
> >
> >> # of passed: 65
> >> # of failed: 3
> >
> > Unrelated to your patches, but have you checked why these fail? Do you have
> > the latest tests running on the latest kernel?
> >
>
> Failed test got passed after enabling /proc/kallsyms, using:
> echo 0 > /proc/sys/kernel/kptr_restrict
Oh, interesting. It should be 'unresolved' (skipped) if that happens.
[23] event filter function - test event filtering on functions [FAIL]
[60] Kprobe events - probe points [FAIL]
[106] (instance) event filter function - test event filtering on functions [FAIL]
OK, let me see.
Thanks for reporting!
> Following is the report of ftracetest on Linux v6.4.0-rc7 (with/without eventfs):
> # of passed: 68
> # of failed: 0
> # of unresolved: 6
> # of untested: 0
> # of unsupported: 45
> # of xfailed: 0
> # of undefined(test bug): 0
>
> If lockdep is enabled getting same warning as reported by 'kernel test robot' for v3 09/10:
> https://lore.kernel.org/all/[email protected]/
>
> >> # of unresolved: 6
> >> # of untested: 0
> >> # of unsupported: 44
> >> # of xfailed: 0
> >> # of undefined(test bug): 0
> >>
> >> These results are same with/without eventfs.
> >
> > I'm hoping to look at these patches this week.
>
> Yes, please. Thanks.
>
> -Ajay
>
--
Masami Hiramatsu (Google) <[email protected]>