Hello,
I've hit the following deadlock while running syzkaller on commit
ce5c2d2c256a4c8b523036537cd6be2d6af8f69d (Nov 7):
[ INFO: possible circular locking dependency detected ]
4.3.0+ #57 Not tainted
-------------------------------------------------------
syzkaller_execu/24882 is trying to acquire lock:
(&tty->atomic_write_lock){+.+.+.}, at: [<ffffffff81774876>]
tty_write_lock+0x46/0x70 drivers/tty/tty_io.c:1093
but task is already holding lock:
(&tty->termios_rwsem){++++.+}, at: [<ffffffff817864d7>]
n_tty_ioctl_helper+0x177/0x210 drivers/tty/tty_ioctl.c:1150
which lock already depends on the new lock.
the existing dependency chain (in reverse order) is:
-> #1 (&tty->termios_rwsem){++++.+}:
[<ffffffff81122501>] lock_acquire+0x101/0x1d0
kernel/locking/lockdep.c:3585
[<ffffffff821727a9>] down_read+0x39/0x50 kernel/locking/rwsem.c:22
[<ffffffff8177eb07>] n_tty_write+0x137/0xaa0 drivers/tty/n_tty.c:2362
[< inline >] do_tty_write drivers/tty/tty_io.c:1164
[<ffffffff8177832e>] tty_write+0x28e/0x4f0 drivers/tty/tty_io.c:1248
[<ffffffff81778631>] redirected_tty_write+0xa1/0xb0
drivers/tty/tty_io.c:1269
[<ffffffff812d788b>] __vfs_write+0xeb/0x2a0 fs/read_write.c:489
[<ffffffff812d8a63>] vfs_write+0x113/0x290 fs/read_write.c:538
[< inline >] SYSC_write fs/read_write.c:585
[<ffffffff812da27b>] SyS_write+0xbb/0x170 fs/read_write.c:577
[<ffffffff82175fd1>] entry_SYSCALL_64_fastpath+0x31/0x9a
arch/x86/entry/entry_64.S:187
-> #0 (&tty->atomic_write_lock){+.+.+.}:
[< inline >] __mutex_lock_common kernel/locking/mutex.c:518
[<ffffffff82170525>] mutex_lock_interruptible_nested+0xa5/0x660
kernel/locking/mutex.c:647
[<ffffffff81774876>] tty_write_lock+0x46/0x70 drivers/tty/tty_io.c:1093
[<ffffffff8177a5a4>] tty_send_xchar+0x94/0x130 drivers/tty/tty_io.c:1289
[<ffffffff81786507>] n_tty_ioctl_helper+0x1a7/0x210
drivers/tty/tty_ioctl.c:1158
[<ffffffff8177f6e9>] n_tty_ioctl+0xe9/0x1e0 drivers/tty/n_tty.c:2514
[<ffffffff8177979c>] tty_ioctl+0xa4c/0x1650 drivers/tty/tty_io.c:2945
[< inline >] vfs_ioctl fs/ioctl.c:43
[<ffffffff812fdf0f>] do_vfs_ioctl+0x53f/0x980 fs/ioctl.c:607
[< inline >] SYSC_ioctl fs/ioctl.c:622
[<ffffffff812fe3df>] SyS_ioctl+0x8f/0xc0 fs/ioctl.c:613
[<ffffffff82175fd1>] entry_SYSCALL_64_fastpath+0x31/0x9a
arch/x86/entry/entry_64.S:187
other info that might help us debug this:
Possible unsafe locking scenario:
CPU0 CPU1
---- ----
lock(&tty->termios_rwsem);
lock(&tty->atomic_write_lock);
lock(&tty->termios_rwsem);
lock(&tty->atomic_write_lock);
*** DEADLOCK ***
2 locks held by syzkaller_execu/24882:
#0: (&tty->ldisc_sem){++++++}, at: [<ffffffff8178699b>]
tty_ldisc_ref_wait+0x2b/0x90 drivers/tty/tty_ldisc.c:264
#1: (&tty->termios_rwsem){++++.+}, at: [<ffffffff817864d7>]
n_tty_ioctl_helper+0x177/0x210 drivers/tty/tty_ioctl.c:1150
stack backtrace:
CPU: 2 PID: 24882 Comm: syzkaller_execu Not tainted 4.3.0+ #57
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Bochs 01/01/2011
ffff88006be0c900 ffff88006b06f7d8 ffffffff816349c7 ffffffff837d05a0
ffffffff837d05a0 ffffffff837c7120 ffff88006b06f820 ffffffff8111c3ba
0000000100000001 ffffffff00000000 ffff88006be0c928 ffff88006be0c900
Call Trace:
[<ffffffff8111c3ba>] print_circular_bug+0x25a/0x2c0
kernel/locking/lockdep.c:1226
[< inline >] check_prev_add kernel/locking/lockdep.c:1853
[< inline >] check_prevs_add kernel/locking/lockdep.c:1958
[< inline >] validate_chain kernel/locking/lockdep.c:2144
[<ffffffff81120ee4>] __lock_acquire+0x2414/0x2600 kernel/locking/lockdep.c:3206
[<ffffffff81122501>] lock_acquire+0x101/0x1d0 kernel/locking/lockdep.c:3585
[< inline >] __mutex_lock_common kernel/locking/mutex.c:518
[<ffffffff82170525>] mutex_lock_interruptible_nested+0xa5/0x660
kernel/locking/mutex.c:647
[<ffffffff81774876>] tty_write_lock+0x46/0x70 drivers/tty/tty_io.c:1093
[<ffffffff8177a5a4>] tty_send_xchar+0x94/0x130 drivers/tty/tty_io.c:1289
[<ffffffff81786507>] n_tty_ioctl_helper+0x1a7/0x210
drivers/tty/tty_ioctl.c:1158
[<ffffffff8177f6e9>] n_tty_ioctl+0xe9/0x1e0 drivers/tty/n_tty.c:2514
[<ffffffff8177979c>] tty_ioctl+0xa4c/0x1650 drivers/tty/tty_io.c:2945
[< inline >] vfs_ioctl fs/ioctl.c:43
[<ffffffff812fdf0f>] do_vfs_ioctl+0x53f/0x980 fs/ioctl.c:607
[< inline >] SYSC_ioctl fs/ioctl.c:622
[<ffffffff812fe3df>] SyS_ioctl+0x8f/0xc0 fs/ioctl.c:613
[<ffffffff82175fd1>] entry_SYSCALL_64_fastpath+0x31/0x9a
arch/x86/entry/entry_64.S:187
The program that triggered it is (but I wasn't able to reproduce it
second time):
// autogenerated by syzkaller (http://github.com/google/syzkaller)
#include <syscall.h>
#include <string.h>
#include <stdint.h>
int main()
{
long r0 = syscall(SYS_mmap, 0x20000000ul, 0x1000ul, 0x2ul,
0x32ul, 0xfffffffffffffffful, 0x0ul);
long r1 = syscall(SYS_mmap, 0x20001000ul, 0x1000ul, 0x3ul,
0x32ul, 0xfffffffffffffffful, 0x0ul);
memcpy((void*)0x20000000, "\x2f\x64\x65\x76\x2f\x70\x74\x6d\x78", 9);
long r3 = syscall(SYS_open, 0x20000000ul, 0x81ul, 0x0ul, 0, 0, 0);
long r4 = syscall(SYS_mmap, 0x20002000ul, 0x1000ul, 0x3ul,
0x32ul, 0xfffffffffffffffful, 0x0ul);
long r5 = syscall(SYS_mmap, 0x20003000ul, 0x1000ul, 0x3ul,
0x32ul, 0xfffffffffffffffful, 0x0ul);
memcpy((void*)0x20003000, "\x4d", 1);
long r7 = syscall(SYS_write, r3, 0x20003000ul, 0x1ul, 0, 0, 0);
long r8 = syscall(SYS_ioctl, r3, 0x540aul, 0x2ul, 0, 0, 0);
long r9 = syscall(SYS_fcntl, r3, 0x406ul, r3, 0, 0, 0);
*(uint64_t*)0x20000000 = 0x20000bfd;
*(uint64_t*)0x20000008 = 0x1;
*(uint64_t*)0x20000010 = 0x20000255;
*(uint64_t*)0x20000018 = 0x0;
*(uint64_t*)0x20000020 = 0x20001fd3;
*(uint64_t*)0x20000028 = 0x1;
*(uint64_t*)0x20000030 = 0x20002f51;
*(uint64_t*)0x20000038 = 0x1;
memcpy((void*)0x20000bfd, "\x3a", 1);
memcpy((void*)0x20001fd3, "\x44", 1);
memcpy((void*)0x20002f51, "\x2d", 1);
long r21 = syscall(SYS_writev, r3, 0x20000000ul, 0x1ul, 0, 0, 0);
return 0;
}
Thanks
On 11/11/2015 03:35 AM, Dmitry Vyukov wrote:
> Hello,
>
> I've hit the following deadlock while running syzkaller on commit
> ce5c2d2c256a4c8b523036537cd6be2d6af8f69d (Nov 7):
>
> [ INFO: possible circular locking dependency detected ]
> 4.3.0+ #57 Not tainted
> -------------------------------------------------------
> syzkaller_execu/24882 is trying to acquire lock:
> (&tty->atomic_write_lock){+.+.+.}, at: [<ffffffff81774876>]
> tty_write_lock+0x46/0x70 drivers/tty/tty_io.c:1093
>
> but task is already holding lock:
> (&tty->termios_rwsem){++++.+}, at: [<ffffffff817864d7>]
> n_tty_ioctl_helper+0x177/0x210 drivers/tty/tty_ioctl.c:1150
>
> which lock already depends on the new lock.
>
> the existing dependency chain (in reverse order) is:
>
> -> #1 (&tty->termios_rwsem){++++.+}:
> [<ffffffff81122501>] lock_acquire+0x101/0x1d0 kernel/locking/lockdep.c:3585
> [<ffffffff821727a9>] down_read+0x39/0x50 kernel/locking/rwsem.c:22
> [<ffffffff8177eb07>] n_tty_write+0x137/0xaa0 drivers/tty/n_tty.c:2362
> [< inline >] do_tty_write drivers/tty/tty_io.c:1164
> [<ffffffff8177832e>] tty_write+0x28e/0x4f0 drivers/tty/tty_io.c:1248
> [<ffffffff81778631>] redirected_tty_write+0xa1/0xb0 drivers/tty/tty_io.c:1269
> [<ffffffff812d788b>] __vfs_write+0xeb/0x2a0 fs/read_write.c:489
> [<ffffffff812d8a63>] vfs_write+0x113/0x290 fs/read_write.c:538
> [< inline >] SYSC_write fs/read_write.c:585
> [<ffffffff812da27b>] SyS_write+0xbb/0x170 fs/read_write.c:577
> [<ffffffff82175fd1>] entry_SYSCALL_64_fastpath+0x31/0x9a
> arch/x86/entry/entry_64.S:187
>
> -> #0 (&tty->atomic_write_lock){+.+.+.}:
> [< inline >] __mutex_lock_common kernel/locking/mutex.c:518
> [<ffffffff82170525>] mutex_lock_interruptible_nested+0xa5/0x660 kernel/locking/mutex.c:647
> [<ffffffff81774876>] tty_write_lock+0x46/0x70 drivers/tty/tty_io.c:1093
> [<ffffffff8177a5a4>] tty_send_xchar+0x94/0x130 drivers/tty/tty_io.c:1289
> [<ffffffff81786507>] n_tty_ioctl_helper+0x1a7/0x210 drivers/tty/tty_ioctl.c:1158
> [<ffffffff8177f6e9>] n_tty_ioctl+0xe9/0x1e0 drivers/tty/n_tty.c:2514
> [<ffffffff8177979c>] tty_ioctl+0xa4c/0x1650 drivers/tty/tty_io.c:2945
> [< inline >] vfs_ioctl fs/ioctl.c:43
> [<ffffffff812fdf0f>] do_vfs_ioctl+0x53f/0x980 fs/ioctl.c:607
> [< inline >] SYSC_ioctl fs/ioctl.c:622
> [<ffffffff812fe3df>] SyS_ioctl+0x8f/0xc0 fs/ioctl.c:613
> [<ffffffff82175fd1>] entry_SYSCALL_64_fastpath+0x31/0x9a
> arch/x86/entry/entry_64.S:187
>
> other info that might help us debug this:
>
> Possible unsafe locking scenario:
>
> CPU0 CPU1
> ---- ----
> lock(&tty->termios_rwsem);
> lock(&tty->atomic_write_lock);
> lock(&tty->termios_rwsem);
> lock(&tty->atomic_write_lock);
>
> *** DEADLOCK ***
Thanks for the report, Dmitry.
Please re-test with the accompanying patch.
Regards,
Peter Hurley
--- >% ---
Subject: [PATCH] tty: Fix tty_send_xchar() lock order inversion
The correct lock order is atomic_write_lock => termios_rwsem, as
established by tty_write() => n_tty_write().
Fixes: c274f6ef1c666 ("tty: Hold termios_rwsem for tcflow(TCIxxx)")
Reported-by: Dmitry Vyukov <[email protected]>
Cc: <[email protected]> # v3.18+
Signed-off-by: Peter Hurley <[email protected]>
---
drivers/tty/tty_io.c | 4 ++++
drivers/tty/tty_ioctl.c | 4 ----
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c
index 2f8c21e..346a1a5 100644
--- a/drivers/tty/tty_io.c
+++ b/drivers/tty/tty_io.c
@@ -1283,18 +1283,22 @@ int tty_send_xchar(struct tty_struct *tty, char ch)
int was_stopped = tty->stopped;
if (tty->ops->send_xchar) {
+ down_read(&tty->termios_rwsem);
tty->ops->send_xchar(tty, ch);
+ up_read(&tty->termios_rwsem);
return 0;
}
if (tty_write_lock(tty, 0) < 0)
return -ERESTARTSYS;
+ down_read(&tty->termios_rwsem);
if (was_stopped)
start_tty(tty);
tty->ops->write(tty, &ch, 1);
if (was_stopped)
stop_tty(tty);
+ up_read(&tty->termios_rwsem);
tty_write_unlock(tty);
return 0;
}
diff --git a/drivers/tty/tty_ioctl.c b/drivers/tty/tty_ioctl.c
index b8c5c12..0ea3513 100644
--- a/drivers/tty/tty_ioctl.c
+++ b/drivers/tty/tty_ioctl.c
@@ -1140,16 +1140,12 @@ int n_tty_ioctl_helper(struct tty_struct *tty, struct file *file,
spin_unlock_irq(&tty->flow_lock);
break;
case TCIOFF:
- down_read(&tty->termios_rwsem);
if (STOP_CHAR(tty) != __DISABLED_CHAR)
retval = tty_send_xchar(tty, STOP_CHAR(tty));
- up_read(&tty->termios_rwsem);
break;
case TCION:
- down_read(&tty->termios_rwsem);
if (START_CHAR(tty) != __DISABLED_CHAR)
retval = tty_send_xchar(tty, START_CHAR(tty));
- up_read(&tty->termios_rwsem);
break;
default:
return -EINVAL;
--
2.6.3
On Wed, Nov 11, 2015 at 1:05 PM, Peter Hurley <[email protected]> wrote:
> On 11/11/2015 03:35 AM, Dmitry Vyukov wrote:
>> Hello,
>>
>> I've hit the following deadlock while running syzkaller on commit
>> ce5c2d2c256a4c8b523036537cd6be2d6af8f69d (Nov 7):
>>
>> [ INFO: possible circular locking dependency detected ]
>> 4.3.0+ #57 Not tainted
>> -------------------------------------------------------
>> syzkaller_execu/24882 is trying to acquire lock:
>> (&tty->atomic_write_lock){+.+.+.}, at: [<ffffffff81774876>]
>> tty_write_lock+0x46/0x70 drivers/tty/tty_io.c:1093
>>
>> but task is already holding lock:
>> (&tty->termios_rwsem){++++.+}, at: [<ffffffff817864d7>]
>> n_tty_ioctl_helper+0x177/0x210 drivers/tty/tty_ioctl.c:1150
>>
>> which lock already depends on the new lock.
>>
>> the existing dependency chain (in reverse order) is:
>>
>> -> #1 (&tty->termios_rwsem){++++.+}:
>> [<ffffffff81122501>] lock_acquire+0x101/0x1d0 kernel/locking/lockdep.c:3585
>> [<ffffffff821727a9>] down_read+0x39/0x50 kernel/locking/rwsem.c:22
>> [<ffffffff8177eb07>] n_tty_write+0x137/0xaa0 drivers/tty/n_tty.c:2362
>> [< inline >] do_tty_write drivers/tty/tty_io.c:1164
>> [<ffffffff8177832e>] tty_write+0x28e/0x4f0 drivers/tty/tty_io.c:1248
>> [<ffffffff81778631>] redirected_tty_write+0xa1/0xb0 drivers/tty/tty_io.c:1269
>> [<ffffffff812d788b>] __vfs_write+0xeb/0x2a0 fs/read_write.c:489
>> [<ffffffff812d8a63>] vfs_write+0x113/0x290 fs/read_write.c:538
>> [< inline >] SYSC_write fs/read_write.c:585
>> [<ffffffff812da27b>] SyS_write+0xbb/0x170 fs/read_write.c:577
>> [<ffffffff82175fd1>] entry_SYSCALL_64_fastpath+0x31/0x9a
>> arch/x86/entry/entry_64.S:187
>>
>> -> #0 (&tty->atomic_write_lock){+.+.+.}:
>> [< inline >] __mutex_lock_common kernel/locking/mutex.c:518
>> [<ffffffff82170525>] mutex_lock_interruptible_nested+0xa5/0x660 kernel/locking/mutex.c:647
>> [<ffffffff81774876>] tty_write_lock+0x46/0x70 drivers/tty/tty_io.c:1093
>> [<ffffffff8177a5a4>] tty_send_xchar+0x94/0x130 drivers/tty/tty_io.c:1289
>> [<ffffffff81786507>] n_tty_ioctl_helper+0x1a7/0x210 drivers/tty/tty_ioctl.c:1158
>> [<ffffffff8177f6e9>] n_tty_ioctl+0xe9/0x1e0 drivers/tty/n_tty.c:2514
>> [<ffffffff8177979c>] tty_ioctl+0xa4c/0x1650 drivers/tty/tty_io.c:2945
>> [< inline >] vfs_ioctl fs/ioctl.c:43
>> [<ffffffff812fdf0f>] do_vfs_ioctl+0x53f/0x980 fs/ioctl.c:607
>> [< inline >] SYSC_ioctl fs/ioctl.c:622
>> [<ffffffff812fe3df>] SyS_ioctl+0x8f/0xc0 fs/ioctl.c:613
>> [<ffffffff82175fd1>] entry_SYSCALL_64_fastpath+0x31/0x9a
>> arch/x86/entry/entry_64.S:187
>>
>> other info that might help us debug this:
>>
>> Possible unsafe locking scenario:
>>
>> CPU0 CPU1
>> ---- ----
>> lock(&tty->termios_rwsem);
>> lock(&tty->atomic_write_lock);
>> lock(&tty->termios_rwsem);
>> lock(&tty->atomic_write_lock);
>>
>> *** DEADLOCK ***
>
> Thanks for the report, Dmitry.
> Please re-test with the accompanying patch.
The patch fixes the deadlock. Thanks for the quick fix!
Tested-by: Dmitry Vyukov <[email protected]>
> Regards,
> Peter Hurley
>
> --- >% ---
> Subject: [PATCH] tty: Fix tty_send_xchar() lock order inversion
>
> The correct lock order is atomic_write_lock => termios_rwsem, as
> established by tty_write() => n_tty_write().
>
> Fixes: c274f6ef1c666 ("tty: Hold termios_rwsem for tcflow(TCIxxx)")
> Reported-by: Dmitry Vyukov <[email protected]>
> Cc: <[email protected]> # v3.18+
> Signed-off-by: Peter Hurley <[email protected]>
> ---
> drivers/tty/tty_io.c | 4 ++++
> drivers/tty/tty_ioctl.c | 4 ----
> 2 files changed, 4 insertions(+), 4 deletions(-)
>
> diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c
> index 2f8c21e..346a1a5 100644
> --- a/drivers/tty/tty_io.c
> +++ b/drivers/tty/tty_io.c
> @@ -1283,18 +1283,22 @@ int tty_send_xchar(struct tty_struct *tty, char ch)
> int was_stopped = tty->stopped;
>
> if (tty->ops->send_xchar) {
> + down_read(&tty->termios_rwsem);
> tty->ops->send_xchar(tty, ch);
> + up_read(&tty->termios_rwsem);
> return 0;
> }
>
> if (tty_write_lock(tty, 0) < 0)
> return -ERESTARTSYS;
>
> + down_read(&tty->termios_rwsem);
> if (was_stopped)
> start_tty(tty);
> tty->ops->write(tty, &ch, 1);
> if (was_stopped)
> stop_tty(tty);
> + up_read(&tty->termios_rwsem);
> tty_write_unlock(tty);
> return 0;
> }
> diff --git a/drivers/tty/tty_ioctl.c b/drivers/tty/tty_ioctl.c
> index b8c5c12..0ea3513 100644
> --- a/drivers/tty/tty_ioctl.c
> +++ b/drivers/tty/tty_ioctl.c
> @@ -1140,16 +1140,12 @@ int n_tty_ioctl_helper(struct tty_struct *tty, struct file *file,
> spin_unlock_irq(&tty->flow_lock);
> break;
> case TCIOFF:
> - down_read(&tty->termios_rwsem);
> if (STOP_CHAR(tty) != __DISABLED_CHAR)
> retval = tty_send_xchar(tty, STOP_CHAR(tty));
> - up_read(&tty->termios_rwsem);
> break;
> case TCION:
> - down_read(&tty->termios_rwsem);
> if (START_CHAR(tty) != __DISABLED_CHAR)
> retval = tty_send_xchar(tty, START_CHAR(tty));
> - up_read(&tty->termios_rwsem);
> break;
> default:
> return -EINVAL;
> --
> 2.6.3
>