1. this thread is stuck in f2fs_evict_inode by #2:
[210757.653718] __schedule+0x2e3/0x740
[210757.654811] schedule+0x42/0xb0
[210757.655829] rwsem_down_read_slowpath+0x16c/0x4a0
[210757.657161] __down_read+0x6b/0x80
[210757.658254] __percpu_down_read+0x54/0x80
[210757.659413] __sb_start_write+0x79/0x80
[210757.660546] f2fs_evict_inode+0x354/0x5c0 [f2fs]
[210757.661808] ? var_wake_function+0x30/0x30
[210757.663016] evict+0xd2/0x1b0
[210757.664007] dispose_list+0x39/0x50
[210757.665083] prune_icache_sb+0x5c/0x80
[210757.666226] super_cache_scan+0x132/0x1b0
[210757.667373] do_shrink_slab+0x150/0x2a0
[210757.668510] shrink_slab+0x20c/0x2a0
[210757.669574] drop_slab_node+0x33/0x60
[210757.670646] drop_slab+0x3e/0x70
[210757.671633] drop_caches_sysctl_handler+0x75/0x85
[210757.672904] proc_sys_call_handler+0x1a2/0x1c0
[210757.674104] proc_sys_write+0x14/0x20
[210757.675189] __vfs_write+0x1b/0x40
[210757.676208] vfs_write+0xb9/0x1a0
[210757.677219] ksys_write+0x67/0xe0
[210757.678251] __x64_sys_write+0x1a/0x20
2. another thread is waiting for #1:
[210757.754646] schedule+0x42/0xb0
[210757.755680] rwsem_down_write_slowpath+0x244/0x4d0
[210757.757016] ? _cond_resched+0x19/0x30
[210757.758140] down_write+0x41/0x50
[210757.759223] prealloc_shrinker+0x6a/0x120
[210757.760397] alloc_super+0x275/0x2d0
[210757.761500] sget_fc+0x74/0x220
[210757.762547] ? set_anon_super+0x20/0x20
[210757.763711] ? shmem_create+0x20/0x20
[210757.764842] vfs_get_super+0x3d/0x100
[210757.765925] get_tree_nodev+0x16/0x20
[210757.767046] shmem_get_tree+0x15/0x20
[210757.768165] vfs_get_tree+0x2a/0xc0
[210757.769248] ? ns_capable+0x10/0x20
[210757.770361] do_mount+0x7b6/0x9c0
[210757.771423] ksys_mount+0x82/0xd0
[210757.772469] __x64_sys_mount+0x25/0x30
[210757.773592] do_syscall_64+0x57/0x190
3. thaw_super is waiting for #2, resulting in xfstests/generic/068 being stuck.
[210757.695823] __schedule+0x2e3/0x740
[210757.696897] ? sched_clock+0x9/0x10
[210757.697959] schedule+0x42/0xb0
[210757.698963] rwsem_down_write_slowpath+0x244/0x4d0
[210757.700247] down_write+0x41/0x50
[210757.701259] thaw_super+0x17/0x30
[210757.702235] do_vfs_ioctl+0x56f/0x670
[210757.703247] ? do_user_addr_fault+0x216/0x450
[210757.704395] ? _copy_to_user+0x2c/0x30
[210757.705443] ksys_ioctl+0x67/0x90
[210757.706419] __x64_sys_ioctl+0x1a/0x20
[210757.707437] do_syscall_64+0x57/0x190
Signed-off-by: Jaegeuk Kim <[email protected]>
---
fs/f2fs/inode.c | 2 --
1 file changed, 2 deletions(-)
diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c
index ab8e0c06c78c..882db4bd917b 100644
--- a/fs/f2fs/inode.c
+++ b/fs/f2fs/inode.c
@@ -778,7 +778,6 @@ void f2fs_evict_inode(struct inode *inode)
f2fs_remove_ino_entry(sbi, inode->i_ino, UPDATE_INO);
f2fs_remove_ino_entry(sbi, inode->i_ino, FLUSH_INO);
- sb_start_intwrite(inode->i_sb);
set_inode_flag(inode, FI_NO_ALLOC);
i_size_write(inode, 0);
retry:
@@ -809,7 +808,6 @@ void f2fs_evict_inode(struct inode *inode)
if (dquot_initialize_needed(inode))
set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR);
}
- sb_end_intwrite(inode->i_sb);
no_delete:
dquot_drop(inode);
--
2.35.1.265.g69c8d7142f-goog
On 2022/2/16 6:00, Jaegeuk Kim wrote:
> 1. this thread is stuck in f2fs_evict_inode by #2:
> [210757.653718] __schedule+0x2e3/0x740
> [210757.654811] schedule+0x42/0xb0
> [210757.655829] rwsem_down_read_slowpath+0x16c/0x4a0
Who has held sb->s_writers.rw_sem?
Thanks,
> [210757.657161] __down_read+0x6b/0x80
> [210757.658254] __percpu_down_read+0x54/0x80
> [210757.659413] __sb_start_write+0x79/0x80
> [210757.660546] f2fs_evict_inode+0x354/0x5c0 [f2fs]
> [210757.661808] ? var_wake_function+0x30/0x30
> [210757.663016] evict+0xd2/0x1b0
> [210757.664007] dispose_list+0x39/0x50
> [210757.665083] prune_icache_sb+0x5c/0x80
> [210757.666226] super_cache_scan+0x132/0x1b0
> [210757.667373] do_shrink_slab+0x150/0x2a0
> [210757.668510] shrink_slab+0x20c/0x2a0
> [210757.669574] drop_slab_node+0x33/0x60
> [210757.670646] drop_slab+0x3e/0x70
> [210757.671633] drop_caches_sysctl_handler+0x75/0x85
> [210757.672904] proc_sys_call_handler+0x1a2/0x1c0
> [210757.674104] proc_sys_write+0x14/0x20
> [210757.675189] __vfs_write+0x1b/0x40
> [210757.676208] vfs_write+0xb9/0x1a0
> [210757.677219] ksys_write+0x67/0xe0
> [210757.678251] __x64_sys_write+0x1a/0x20
>
> 2. another thread is waiting for #1:
> [210757.754646] schedule+0x42/0xb0
> [210757.755680] rwsem_down_write_slowpath+0x244/0x4d0
> [210757.757016] ? _cond_resched+0x19/0x30
> [210757.758140] down_write+0x41/0x50
> [210757.759223] prealloc_shrinker+0x6a/0x120
> [210757.760397] alloc_super+0x275/0x2d0
> [210757.761500] sget_fc+0x74/0x220
> [210757.762547] ? set_anon_super+0x20/0x20
> [210757.763711] ? shmem_create+0x20/0x20
> [210757.764842] vfs_get_super+0x3d/0x100
> [210757.765925] get_tree_nodev+0x16/0x20
> [210757.767046] shmem_get_tree+0x15/0x20
> [210757.768165] vfs_get_tree+0x2a/0xc0
> [210757.769248] ? ns_capable+0x10/0x20
> [210757.770361] do_mount+0x7b6/0x9c0
> [210757.771423] ksys_mount+0x82/0xd0
> [210757.772469] __x64_sys_mount+0x25/0x30
> [210757.773592] do_syscall_64+0x57/0x190
>
> 3. thaw_super is waiting for #2, resulting in xfstests/generic/068 being stuck.
> [210757.695823] __schedule+0x2e3/0x740
> [210757.696897] ? sched_clock+0x9/0x10
> [210757.697959] schedule+0x42/0xb0
> [210757.698963] rwsem_down_write_slowpath+0x244/0x4d0
> [210757.700247] down_write+0x41/0x50
> [210757.701259] thaw_super+0x17/0x30
> [210757.702235] do_vfs_ioctl+0x56f/0x670
> [210757.703247] ? do_user_addr_fault+0x216/0x450
> [210757.704395] ? _copy_to_user+0x2c/0x30
> [210757.705443] ksys_ioctl+0x67/0x90
> [210757.706419] __x64_sys_ioctl+0x1a/0x20
> [210757.707437] do_syscall_64+0x57/0x190
>
> Signed-off-by: Jaegeuk Kim <[email protected]>
> ---
> fs/f2fs/inode.c | 2 --
> 1 file changed, 2 deletions(-)
>
> diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c
> index ab8e0c06c78c..882db4bd917b 100644
> --- a/fs/f2fs/inode.c
> +++ b/fs/f2fs/inode.c
> @@ -778,7 +778,6 @@ void f2fs_evict_inode(struct inode *inode)
> f2fs_remove_ino_entry(sbi, inode->i_ino, UPDATE_INO);
> f2fs_remove_ino_entry(sbi, inode->i_ino, FLUSH_INO);
>
> - sb_start_intwrite(inode->i_sb);
> set_inode_flag(inode, FI_NO_ALLOC);
> i_size_write(inode, 0);
> retry:
> @@ -809,7 +808,6 @@ void f2fs_evict_inode(struct inode *inode)
> if (dquot_initialize_needed(inode))
> set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR);
> }
> - sb_end_intwrite(inode->i_sb);
> no_delete:
> dquot_drop(inode);
>
On 02/25, Chao Yu wrote:
> On 2022/2/16 6:00, Jaegeuk Kim wrote:
> > 1. this thread is stuck in f2fs_evict_inode by #2:
> > [210757.653718] __schedule+0x2e3/0x740
> > [210757.654811] schedule+0x42/0xb0
> > [210757.655829] rwsem_down_read_slowpath+0x16c/0x4a0
>
> Who has held sb->s_writers.rw_sem?
I lost the history. Let me drop this and investigate once I hit this again.
>
> Thanks,
>
> > [210757.657161] __down_read+0x6b/0x80
> > [210757.658254] __percpu_down_read+0x54/0x80
> > [210757.659413] __sb_start_write+0x79/0x80
> > [210757.660546] f2fs_evict_inode+0x354/0x5c0 [f2fs]
> > [210757.661808] ? var_wake_function+0x30/0x30
> > [210757.663016] evict+0xd2/0x1b0
> > [210757.664007] dispose_list+0x39/0x50
> > [210757.665083] prune_icache_sb+0x5c/0x80
> > [210757.666226] super_cache_scan+0x132/0x1b0
> > [210757.667373] do_shrink_slab+0x150/0x2a0
> > [210757.668510] shrink_slab+0x20c/0x2a0
> > [210757.669574] drop_slab_node+0x33/0x60
> > [210757.670646] drop_slab+0x3e/0x70
> > [210757.671633] drop_caches_sysctl_handler+0x75/0x85
> > [210757.672904] proc_sys_call_handler+0x1a2/0x1c0
> > [210757.674104] proc_sys_write+0x14/0x20
> > [210757.675189] __vfs_write+0x1b/0x40
> > [210757.676208] vfs_write+0xb9/0x1a0
> > [210757.677219] ksys_write+0x67/0xe0
> > [210757.678251] __x64_sys_write+0x1a/0x20
> >
> > 2. another thread is waiting for #1:
> > [210757.754646] schedule+0x42/0xb0
> > [210757.755680] rwsem_down_write_slowpath+0x244/0x4d0
> > [210757.757016] ? _cond_resched+0x19/0x30
> > [210757.758140] down_write+0x41/0x50
> > [210757.759223] prealloc_shrinker+0x6a/0x120
> > [210757.760397] alloc_super+0x275/0x2d0
> > [210757.761500] sget_fc+0x74/0x220
> > [210757.762547] ? set_anon_super+0x20/0x20
> > [210757.763711] ? shmem_create+0x20/0x20
> > [210757.764842] vfs_get_super+0x3d/0x100
> > [210757.765925] get_tree_nodev+0x16/0x20
> > [210757.767046] shmem_get_tree+0x15/0x20
> > [210757.768165] vfs_get_tree+0x2a/0xc0
> > [210757.769248] ? ns_capable+0x10/0x20
> > [210757.770361] do_mount+0x7b6/0x9c0
> > [210757.771423] ksys_mount+0x82/0xd0
> > [210757.772469] __x64_sys_mount+0x25/0x30
> > [210757.773592] do_syscall_64+0x57/0x190
> >
> > 3. thaw_super is waiting for #2, resulting in xfstests/generic/068 being stuck.
> > [210757.695823] __schedule+0x2e3/0x740
> > [210757.696897] ? sched_clock+0x9/0x10
> > [210757.697959] schedule+0x42/0xb0
> > [210757.698963] rwsem_down_write_slowpath+0x244/0x4d0
> > [210757.700247] down_write+0x41/0x50
> > [210757.701259] thaw_super+0x17/0x30
> > [210757.702235] do_vfs_ioctl+0x56f/0x670
> > [210757.703247] ? do_user_addr_fault+0x216/0x450
> > [210757.704395] ? _copy_to_user+0x2c/0x30
> > [210757.705443] ksys_ioctl+0x67/0x90
> > [210757.706419] __x64_sys_ioctl+0x1a/0x20
> > [210757.707437] do_syscall_64+0x57/0x190
> >
> > Signed-off-by: Jaegeuk Kim <[email protected]>
> > ---
> > fs/f2fs/inode.c | 2 --
> > 1 file changed, 2 deletions(-)
> >
> > diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c
> > index ab8e0c06c78c..882db4bd917b 100644
> > --- a/fs/f2fs/inode.c
> > +++ b/fs/f2fs/inode.c
> > @@ -778,7 +778,6 @@ void f2fs_evict_inode(struct inode *inode)
> > f2fs_remove_ino_entry(sbi, inode->i_ino, UPDATE_INO);
> > f2fs_remove_ino_entry(sbi, inode->i_ino, FLUSH_INO);
> > - sb_start_intwrite(inode->i_sb);
> > set_inode_flag(inode, FI_NO_ALLOC);
> > i_size_write(inode, 0);
> > retry:
> > @@ -809,7 +808,6 @@ void f2fs_evict_inode(struct inode *inode)
> > if (dquot_initialize_needed(inode))
> > set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR);
> > }
> > - sb_end_intwrite(inode->i_sb);
> > no_delete:
> > dquot_drop(inode);
1. waiting for f2fs_evict_inode
[ 5560.043945] __wait_on_freeing_inode+0xac/0xf0
[ 5560.045540] ? var_wake_function+0x30/0x30
[ 5560.047036] find_inode_fast+0x6d/0xc0
[ 5560.048473] iget_locked+0x79/0x230
[ 5560.049933] f2fs_iget+0x27/0x1200 [f2fs]
[ 5560.051496] f2fs_lookup+0x18c/0x3e0 [f2fs]
[ 5560.053069] __lookup_slow+0x84/0x150
[ 5560.054503] walk_component+0x141/0x1b0
[ 5560.055938] link_path_walk.part.0+0x23b/0x360
[ 5560.057541] ? end_bio_bh_io_sync+0x37/0x50
[ 5560.059086] path_parentat+0x3c/0x90
[ 5560.060492] filename_parentat+0xd7/0x1e0
[ 5560.062002] ? blk_mq_free_request+0x127/0x150
[ 5560.063576] do_renameat2+0xc1/0x5b0
--> sb_start_write(m->mnt_sb); -> __sb_start_write(sb, SB_FREEZE_WRITE);
[ 5560.064999] ? __check_object_size+0x13f/0x150
[ 5560.066559] ? strncpy_from_user+0x44/0x150
[ 5560.068038] ? getname_flags.part.0+0x4c/0x1b0
[ 5560.069617] __x64_sys_renameat2+0x51/0x60
2. waiting for sb_start_intwrite -> __sb_start_write(sb, SB_FREEZE_FS);
[ 5560.152447] percpu_rwsem_wait+0xaf/0x160
[ 5560.154000] ? percpu_down_write+0xd0/0xd0
[ 5560.155498] __percpu_down_read+0x4e/0x60
[ 5560.157000] f2fs_evict_inode+0x5a3/0x610 [f2fs]
[ 5560.158648] ? var_wake_function+0x30/0x30
[ 5560.160341] evict+0xd2/0x180
[ 5560.161728] prune_icache_sb+0x81/0xb0
--> inode_lru_isolate() -> inode->i_state |= I_FREEING;
[ 5560.163179] super_cache_scan+0x169/0x1f0
[ 5560.164675] do_shrink_slab+0x145/0x2b0
[ 5560.166121] shrink_slab+0x186/0x2d0
[ 5560.167481] drop_slab_node+0x4a/0x90
[ 5560.168876] drop_slab+0x3e/0x80
[ 5560.170178] drop_caches_sysctl_handler+0x75/0x90
[ 5560.171761] proc_sys_call_handler+0x149/0x280
[ 5560.173328] proc_sys_write+0x13/0x20
[ 5560.174667] new_sync_write+0x117/0x1b0
[ 5560.176120] vfs_write+0x1d5/0x270
[ 5560.177409] ksys_write+0x67/0xe0
Signed-off-by: Jaegeuk Kim <[email protected]>
---
Note, I found this call stack.
fs/f2fs/inode.c | 2 --
1 file changed, 2 deletions(-)
diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c
index ab8e0c06c78c..882db4bd917b 100644
--- a/fs/f2fs/inode.c
+++ b/fs/f2fs/inode.c
@@ -778,7 +778,6 @@ void f2fs_evict_inode(struct inode *inode)
f2fs_remove_ino_entry(sbi, inode->i_ino, UPDATE_INO);
f2fs_remove_ino_entry(sbi, inode->i_ino, FLUSH_INO);
- sb_start_intwrite(inode->i_sb);
set_inode_flag(inode, FI_NO_ALLOC);
i_size_write(inode, 0);
retry:
@@ -809,7 +808,6 @@ void f2fs_evict_inode(struct inode *inode)
if (dquot_initialize_needed(inode))
set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR);
}
- sb_end_intwrite(inode->i_sb);
no_delete:
dquot_drop(inode);
--
2.35.1.574.g5d30c73bfb-goog
On 2022/3/1 12:48, Jaegeuk Kim wrote:
> 1. waiting for f2fs_evict_inode
> [ 5560.043945] __wait_on_freeing_inode+0xac/0xf0
> [ 5560.045540] ? var_wake_function+0x30/0x30
> [ 5560.047036] find_inode_fast+0x6d/0xc0
> [ 5560.048473] iget_locked+0x79/0x230
> [ 5560.049933] f2fs_iget+0x27/0x1200 [f2fs]
> [ 5560.051496] f2fs_lookup+0x18c/0x3e0 [f2fs]
> [ 5560.053069] __lookup_slow+0x84/0x150
> [ 5560.054503] walk_component+0x141/0x1b0
> [ 5560.055938] link_path_walk.part.0+0x23b/0x360
> [ 5560.057541] ? end_bio_bh_io_sync+0x37/0x50
> [ 5560.059086] path_parentat+0x3c/0x90
> [ 5560.060492] filename_parentat+0xd7/0x1e0
> [ 5560.062002] ? blk_mq_free_request+0x127/0x150
> [ 5560.063576] do_renameat2+0xc1/0x5b0
> --> sb_start_write(m->mnt_sb); -> __sb_start_write(sb, SB_FREEZE_WRITE);
>
> [ 5560.064999] ? __check_object_size+0x13f/0x150
> [ 5560.066559] ? strncpy_from_user+0x44/0x150
> [ 5560.068038] ? getname_flags.part.0+0x4c/0x1b0
> [ 5560.069617] __x64_sys_renameat2+0x51/0x60
>
> 2. waiting for sb_start_intwrite -> __sb_start_write(sb, SB_FREEZE_FS);
It's still not clear that why __sb_start_write(sb, SB_FREEZE_FS) will be blocked,
as SB_FREEZE_FS and SB_FREEZE_WRITE points to different locks.
Thread A Thread B Thread C
- rename
- sb_start_write
- __sb_start_write(SB_FREEZE_WRITE)
...
- f2fs_lookup
...
- __wait_on_freeing_inode
- drop_slab
- prune_icache_sb
- inode_lru_isolate
:inode->i_state |= I_FREEING
- Is there any flow that it has already held
SB_FREEZE_FS and try to lock SB_FREEZE_WRITE?
- f2fs_evict_inode
- __sb_start_write(SB_FREEZE_FS)
Thanks,
>
> [ 5560.152447] percpu_rwsem_wait+0xaf/0x160
> [ 5560.154000] ? percpu_down_write+0xd0/0xd0
> [ 5560.155498] __percpu_down_read+0x4e/0x60
> [ 5560.157000] f2fs_evict_inode+0x5a3/0x610 [f2fs]
> [ 5560.158648] ? var_wake_function+0x30/0x30
> [ 5560.160341] evict+0xd2/0x180
> [ 5560.161728] prune_icache_sb+0x81/0xb0
> --> inode_lru_isolate() -> inode->i_state |= I_FREEING;
>
> [ 5560.163179] super_cache_scan+0x169/0x1f0
> [ 5560.164675] do_shrink_slab+0x145/0x2b0
> [ 5560.166121] shrink_slab+0x186/0x2d0
> [ 5560.167481] drop_slab_node+0x4a/0x90
> [ 5560.168876] drop_slab+0x3e/0x80
> [ 5560.170178] drop_caches_sysctl_handler+0x75/0x90
> [ 5560.171761] proc_sys_call_handler+0x149/0x280
> [ 5560.173328] proc_sys_write+0x13/0x20
> [ 5560.174667] new_sync_write+0x117/0x1b0
> [ 5560.176120] vfs_write+0x1d5/0x270
> [ 5560.177409] ksys_write+0x67/0xe0
>
> Signed-off-by: Jaegeuk Kim <[email protected]>
> ---
> Note, I found this call stack.
>
> fs/f2fs/inode.c | 2 --
> 1 file changed, 2 deletions(-)
>
> diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c
> index ab8e0c06c78c..882db4bd917b 100644
> --- a/fs/f2fs/inode.c
> +++ b/fs/f2fs/inode.c
> @@ -778,7 +778,6 @@ void f2fs_evict_inode(struct inode *inode)
> f2fs_remove_ino_entry(sbi, inode->i_ino, UPDATE_INO);
> f2fs_remove_ino_entry(sbi, inode->i_ino, FLUSH_INO);
>
> - sb_start_intwrite(inode->i_sb);
> set_inode_flag(inode, FI_NO_ALLOC);
> i_size_write(inode, 0);
> retry:
> @@ -809,7 +808,6 @@ void f2fs_evict_inode(struct inode *inode)
> if (dquot_initialize_needed(inode))
> set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR);
> }
> - sb_end_intwrite(inode->i_sb);
> no_delete:
> dquot_drop(inode);
>