On Mon, Feb 13, 2017 at 11:29 PM, Cong Wang <[email protected]> wrote:
> On Mon, Feb 13, 2017 at 11:19 AM, Andrey Konovalov
> <[email protected]> wrote:
>> Hi,
>>
>> I've got the following error report while fuzzing the kernel with syzkaller.
>>
>> On commit 926af6273fc683cd98cd0ce7bf0d04a02eed6742.
>>
>> A reproducer and .config are attached.
>> Note, that it takes quite some time to trigger the bug (up to 10 minutes).
>>
>> BUG: KASAN: use-after-free in dccp_feat_activate_values+0x967/0xab0
>> net/dccp/feat.c:1541 at addr ffff88003713be68
>> Read of size 8 by task syz-executor2/8457
>> CPU: 2 PID: 8457 Comm: syz-executor2 Not tainted 4.10.0-rc7+ #127
>> Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Bochs 01/01/2011
>> Call Trace:
>> <IRQ>
>> __dump_stack lib/dump_stack.c:15 [inline]
>> dump_stack+0x292/0x398 lib/dump_stack.c:51
>> kasan_object_err+0x1c/0x70 mm/kasan/report.c:162
>> print_address_description mm/kasan/report.c:200 [inline]
>> kasan_report_error mm/kasan/report.c:289 [inline]
>> kasan_report.part.1+0x20e/0x4e0 mm/kasan/report.c:311
>> kasan_report mm/kasan/report.c:332 [inline]
>> __asan_report_load8_noabort+0x29/0x30 mm/kasan/report.c:332
>> dccp_feat_activate_values+0x967/0xab0 net/dccp/feat.c:1541
>> dccp_create_openreq_child+0x464/0x610 net/dccp/minisocks.c:121
>> dccp_v6_request_recv_sock+0x1f6/0x1960 net/dccp/ipv6.c:457
>> dccp_check_req+0x335/0x5a0 net/dccp/minisocks.c:186
>> dccp_v6_rcv+0x69e/0x1d00 net/dccp/ipv6.c:711
>> ip6_input_finish+0x46d/0x17a0 net/ipv6/ip6_input.c:279
>> NF_HOOK include/linux/netfilter.h:257 [inline]
>> ip6_input+0xdb/0x590 net/ipv6/ip6_input.c:322
>> dst_input include/net/dst.h:507 [inline]
>> ip6_rcv_finish+0x289/0x890 net/ipv6/ip6_input.c:69
>> NF_HOOK include/linux/netfilter.h:257 [inline]
>> ipv6_rcv+0x12ec/0x23d0 net/ipv6/ip6_input.c:203
>> __netif_receive_skb_core+0x1ae5/0x3400 net/core/dev.c:4190
>> __netif_receive_skb+0x2a/0x170 net/core/dev.c:4228
>> process_backlog+0xe5/0x6c0 net/core/dev.c:4839
>> napi_poll net/core/dev.c:5202 [inline]
>> net_rx_action+0xe70/0x1900 net/core/dev.c:5267
>> __do_softirq+0x2fb/0xb7d kernel/softirq.c:284
>> do_softirq_own_stack+0x1c/0x30 arch/x86/entry/entry_64.S:902
>
>
> Seems there is a race condition between iterating dccp_feat_entry
> and freeing it, bh_lock_sock() seems not held in this path.
Cong, where exactly do we need to add bh_lock_sock()?
I am still seeing this on 4977ab6e92e267afe9d8f78438c3db330ca8434c
==================================================================
BUG: KASAN: use-after-free in dccp_feat_activate_values+0x94c/0xa90
net/dccp/feat.c:1541 at addr ffff880059f18998
Read of size 8 by task syz-executor/3952
CPU: 2 PID: 3952 Comm: syz-executor Not tainted 4.10.0+ #269
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Bochs 01/01/2011
Call Trace:
<IRQ>
__dump_stack lib/dump_stack.c:15 [inline]
dump_stack+0x2ee/0x3ef lib/dump_stack.c:51
kasan_object_err+0x1c/0x70 mm/kasan/report.c:166
print_address_description mm/kasan/report.c:204 [inline]
kasan_report_error mm/kasan/report.c:288 [inline]
kasan_report.part.2+0x198/0x440 mm/kasan/report.c:310
kasan_report mm/kasan/report.c:331 [inline]
__asan_report_load8_noabort+0x29/0x30 mm/kasan/report.c:331
dccp_feat_activate_values+0x94c/0xa90 net/dccp/feat.c:1541
dccp_create_openreq_child+0x464/0x610 net/dccp/minisocks.c:121
dccp_v6_request_recv_sock+0x1f1/0x17e0 net/dccp/ipv6.c:457
dccp_check_req+0x335/0x5a0 net/dccp/minisocks.c:186
dccp_v6_rcv+0x653/0x1d10 net/dccp/ipv6.c:711
ip6_input_finish+0x45b/0x1700 net/ipv6/ip6_input.c:279
NF_HOOK include/linux/netfilter.h:257 [inline]
ip6_input+0xdb/0x580 net/ipv6/ip6_input.c:322
dst_input include/net/dst.h:492 [inline]
ip6_rcv_finish+0x194/0x720 net/ipv6/ip6_input.c:69
NF_HOOK include/linux/netfilter.h:257 [inline]
ipv6_rcv+0x12df/0x2380 net/ipv6/ip6_input.c:203
__netif_receive_skb_core+0x1ac8/0x33f0 net/core/dev.c:4179
__netif_receive_skb+0x2a/0x170 net/core/dev.c:4217
process_backlog+0x11e/0x730 net/core/dev.c:4837
napi_poll net/core/dev.c:5171 [inline]
net_rx_action+0xf0a/0x1980 net/core/dev.c:5236
__do_softirq+0x31f/0xbe7 kernel/softirq.c:284
do_softirq_own_stack+0x1c/0x30 arch/x86/entry/entry_64.S:902
</IRQ>
do_softirq.part.21+0x2c0/0x300 kernel/softirq.c:328
do_softirq kernel/softirq.c:176 [inline]
__local_bh_enable_ip+0x24c/0x290 kernel/softirq.c:181
local_bh_enable include/linux/bottom_half.h:31 [inline]
rcu_read_unlock_bh include/linux/rcupdate.h:971 [inline]
ip6_finish_output2+0xb85/0x2380 net/ipv6/ip6_output.c:124
ip6_finish_output+0x2f9/0x950 net/ipv6/ip6_output.c:149
NF_HOOK_COND include/linux/netfilter.h:246 [inline]
ip6_output+0x1cb/0x8c0 net/ipv6/ip6_output.c:163
ip6_xmit+0xc36/0x1e80 include/net/dst.h:486
inet6_csk_xmit+0x320/0x5d0 net/ipv6/inet6_connection_sock.c:139
dccp_transmit_skb+0xac5/0x10e0 net/dccp/output.c:141
dccp_xmit_packet+0x20f/0x750 net/dccp/output.c:280
dccp_write_xmit+0x168/0x1d0 net/dccp/output.c:362
dccp_sendmsg+0x793/0xb00 net/dccp/proto.c:796
inet_sendmsg+0x164/0x5b0 net/ipv4/af_inet.c:761
sock_sendmsg_nosec net/socket.c:633 [inline]
sock_sendmsg+0xca/0x110 net/socket.c:643
SYSC_sendto+0x660/0x810 net/socket.c:1685
SyS_sendto+0x40/0x50 net/socket.c:1653
entry_SYSCALL_64_fastpath+0x1f/0xc2
RIP: 0033:0x4458d9
RSP: 002b:00007f59f536bb58 EFLAGS: 00000282 ORIG_RAX: 000000000000002c
RAX: ffffffffffffffda RBX: 000000000000001a RCX: 00000000004458d9
RDX: 0000000000000037 RSI: 0000000020e62000 RDI: 000000000000001a
RBP: 00000000006e1b90 R08: 0000000020e62fe0 R09: 0000000000000020
R10: 0000000000008000 R11: 0000000000000282 R12: 0000000000708150
R13: 0000000020d0aff8 R14: 0000000000000004 R15: 0000000000000000
Object at ffff880059f18980, in cache kmalloc-64 size: 64
Allocated:
PID = 3932
save_stack_trace+0x16/0x20 arch/x86/kernel/stacktrace.c:57
save_stack+0x43/0xd0 mm/kasan/kasan.c:502
set_track mm/kasan/kasan.c:514 [inline]
kasan_kmalloc+0xaa/0xd0 mm/kasan/kasan.c:605
kmem_cache_alloc_trace+0x10b/0x6e0 mm/slab.c:3637
kmalloc include/linux/slab.h:490 [inline]
dccp_feat_entry_new+0x20b/0x400 net/dccp/feat.c:467
dccp_feat_push_change+0x38/0x220 net/dccp/feat.c:487
__feat_register_sp+0x223/0x2f0 net/dccp/feat.c:741
dccp_feat_propagate_ccid+0x22b/0x2b0 net/dccp/feat.c:949
dccp_feat_server_ccid_dependencies+0x1b3/0x250 net/dccp/feat.c:1012
dccp_make_response+0x1f1/0xc50 net/dccp/output.c:423
dccp_v6_send_response+0x4ec/0xc20 net/dccp/ipv6.c:217
dccp_v6_conn_request+0xab2/0x1100 net/dccp/ipv6.c:377
dccp_rcv_state_process+0x50c/0x1650 net/dccp/input.c:606
dccp_v6_do_rcv+0x20d/0x350 net/dccp/ipv6.c:632
sk_backlog_rcv include/net/sock.h:896 [inline]
__sk_receive_skb+0x350/0xca0 net/core/sock.c:479
dccp_v6_rcv+0xb17/0x1d10 net/dccp/ipv6.c:742
ip6_input_finish+0x45b/0x1700 net/ipv6/ip6_input.c:279
NF_HOOK include/linux/netfilter.h:257 [inline]
ip6_input+0xdb/0x580 net/ipv6/ip6_input.c:322
dst_input include/net/dst.h:492 [inline]
ip6_rcv_finish+0x194/0x720 net/ipv6/ip6_input.c:69
NF_HOOK include/linux/netfilter.h:257 [inline]
ipv6_rcv+0x12df/0x2380 net/ipv6/ip6_input.c:203
__netif_receive_skb_core+0x1ac8/0x33f0 net/core/dev.c:4179
__netif_receive_skb+0x2a/0x170 net/core/dev.c:4217
process_backlog+0x11e/0x730 net/core/dev.c:4837
napi_poll net/core/dev.c:5171 [inline]
net_rx_action+0xf0a/0x1980 net/core/dev.c:5236
__do_softirq+0x31f/0xbe7 kernel/softirq.c:284
Freed:
PID = 24
save_stack_trace+0x16/0x20 arch/x86/kernel/stacktrace.c:57
save_stack+0x43/0xd0 mm/kasan/kasan.c:502
set_track mm/kasan/kasan.c:514 [inline]
kasan_slab_free+0x6f/0xb0 mm/kasan/kasan.c:578
__cache_free mm/slab.c:3513 [inline]
kfree+0xd3/0x250 mm/slab.c:3830
dccp_feat_entry_destructor.part.4+0x48/0x60 net/dccp/feat.c:418
dccp_feat_entry_destructor net/dccp/feat.c:416 [inline]
dccp_feat_list_pop net/dccp/feat.c:541 [inline]
dccp_feat_activate_values+0x57f/0xa90 net/dccp/feat.c:1543
dccp_create_openreq_child+0x464/0x610 net/dccp/minisocks.c:121
dccp_v6_request_recv_sock+0x1f1/0x17e0 net/dccp/ipv6.c:457
dccp_check_req+0x335/0x5a0 net/dccp/minisocks.c:186
dccp_v6_rcv+0x653/0x1d10 net/dccp/ipv6.c:711
ip6_input_finish+0x45b/0x1700 net/ipv6/ip6_input.c:279
NF_HOOK include/linux/netfilter.h:257 [inline]
ip6_input+0xdb/0x580 net/ipv6/ip6_input.c:322
dst_input include/net/dst.h:492 [inline]
ip6_rcv_finish+0x194/0x720 net/ipv6/ip6_input.c:69
NF_HOOK include/linux/netfilter.h:257 [inline]
ipv6_rcv+0x12df/0x2380 net/ipv6/ip6_input.c:203
__netif_receive_skb_core+0x1ac8/0x33f0 net/core/dev.c:4179
__netif_receive_skb+0x2a/0x170 net/core/dev.c:4217
process_backlog+0x11e/0x730 net/core/dev.c:4837
napi_poll net/core/dev.c:5171 [inline]
net_rx_action+0xf0a/0x1980 net/core/dev.c:5236
__do_softirq+0x31f/0xbe7 kernel/softirq.c:284
Memory state around the buggy address:
ffff880059f18880: 00 00 00 00 00 fc fc fc fc fc fc fc fc fc fc fc
ffff880059f18900: 00 00 00 00 00 fc fc fc fc fc fc fc fc fc fc fc
>ffff880059f18980: fb fb fb fb fb fb fb fb fc fc fc fc fc fc fc fc
^
ffff880059f18a00: fb fb fb fb fb fb fb fb fc fc fc fc fc fc fc fc
ffff880059f18a80: 00 00 00 00 00 fc fc fc fc fc fc fc fc fc fc fc
==================================================================
On Fri, Mar 3, 2017 at 3:11 PM, Dmitry Vyukov <[email protected]> wrote:
> On Mon, Feb 13, 2017 at 11:29 PM, Cong Wang <[email protected]> wrote:
>> On Mon, Feb 13, 2017 at 11:19 AM, Andrey Konovalov
>> <[email protected]> wrote:
>>> Hi,
>>>
>>> I've got the following error report while fuzzing the kernel with syzkaller.
>>>
>>> On commit 926af6273fc683cd98cd0ce7bf0d04a02eed6742.
>>>
>>> A reproducer and .config are attached.
>>> Note, that it takes quite some time to trigger the bug (up to 10 minutes).
>>>
>>> BUG: KASAN: use-after-free in dccp_feat_activate_values+0x967/0xab0
>>> net/dccp/feat.c:1541 at addr ffff88003713be68
>>> Read of size 8 by task syz-executor2/8457
>>> CPU: 2 PID: 8457 Comm: syz-executor2 Not tainted 4.10.0-rc7+ #127
>>> Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Bochs 01/01/2011
>>> Call Trace:
>>> <IRQ>
>>> __dump_stack lib/dump_stack.c:15 [inline]
>>> dump_stack+0x292/0x398 lib/dump_stack.c:51
>>> kasan_object_err+0x1c/0x70 mm/kasan/report.c:162
>>> print_address_description mm/kasan/report.c:200 [inline]
>>> kasan_report_error mm/kasan/report.c:289 [inline]
>>> kasan_report.part.1+0x20e/0x4e0 mm/kasan/report.c:311
>>> kasan_report mm/kasan/report.c:332 [inline]
>>> __asan_report_load8_noabort+0x29/0x30 mm/kasan/report.c:332
>>> dccp_feat_activate_values+0x967/0xab0 net/dccp/feat.c:1541
>>> dccp_create_openreq_child+0x464/0x610 net/dccp/minisocks.c:121
>>> dccp_v6_request_recv_sock+0x1f6/0x1960 net/dccp/ipv6.c:457
>>> dccp_check_req+0x335/0x5a0 net/dccp/minisocks.c:186
>>> dccp_v6_rcv+0x69e/0x1d00 net/dccp/ipv6.c:711
>>> ip6_input_finish+0x46d/0x17a0 net/ipv6/ip6_input.c:279
>>> NF_HOOK include/linux/netfilter.h:257 [inline]
>>> ip6_input+0xdb/0x590 net/ipv6/ip6_input.c:322
>>> dst_input include/net/dst.h:507 [inline]
>>> ip6_rcv_finish+0x289/0x890 net/ipv6/ip6_input.c:69
>>> NF_HOOK include/linux/netfilter.h:257 [inline]
>>> ipv6_rcv+0x12ec/0x23d0 net/ipv6/ip6_input.c:203
>>> __netif_receive_skb_core+0x1ae5/0x3400 net/core/dev.c:4190
>>> __netif_receive_skb+0x2a/0x170 net/core/dev.c:4228
>>> process_backlog+0xe5/0x6c0 net/core/dev.c:4839
>>> napi_poll net/core/dev.c:5202 [inline]
>>> net_rx_action+0xe70/0x1900 net/core/dev.c:5267
>>> __do_softirq+0x2fb/0xb7d kernel/softirq.c:284
>>> do_softirq_own_stack+0x1c/0x30 arch/x86/entry/entry_64.S:902
>>
>>
>> Seems there is a race condition between iterating dccp_feat_entry
>> and freeing it, bh_lock_sock() seems not held in this path.
>
>
>
> Cong, where exactly do we need to add bh_lock_sock()?
>
> I am still seeing this on 4977ab6e92e267afe9d8f78438c3db330ca8434c
>
> ==================================================================
> BUG: KASAN: use-after-free in dccp_feat_activate_values+0x94c/0xa90
> net/dccp/feat.c:1541 at addr ffff880059f18998
> Read of size 8 by task syz-executor/3952
> CPU: 2 PID: 3952 Comm: syz-executor Not tainted 4.10.0+ #269
> Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Bochs 01/01/2011
> Call Trace:
> <IRQ>
> __dump_stack lib/dump_stack.c:15 [inline]
> dump_stack+0x2ee/0x3ef lib/dump_stack.c:51
> kasan_object_err+0x1c/0x70 mm/kasan/report.c:166
> print_address_description mm/kasan/report.c:204 [inline]
> kasan_report_error mm/kasan/report.c:288 [inline]
> kasan_report.part.2+0x198/0x440 mm/kasan/report.c:310
> kasan_report mm/kasan/report.c:331 [inline]
> __asan_report_load8_noabort+0x29/0x30 mm/kasan/report.c:331
> dccp_feat_activate_values+0x94c/0xa90 net/dccp/feat.c:1541
> dccp_create_openreq_child+0x464/0x610 net/dccp/minisocks.c:121
> dccp_v6_request_recv_sock+0x1f1/0x17e0 net/dccp/ipv6.c:457
> dccp_check_req+0x335/0x5a0 net/dccp/minisocks.c:186
> dccp_v6_rcv+0x653/0x1d10 net/dccp/ipv6.c:711
> ip6_input_finish+0x45b/0x1700 net/ipv6/ip6_input.c:279
> NF_HOOK include/linux/netfilter.h:257 [inline]
> ip6_input+0xdb/0x580 net/ipv6/ip6_input.c:322
> dst_input include/net/dst.h:492 [inline]
> ip6_rcv_finish+0x194/0x720 net/ipv6/ip6_input.c:69
> NF_HOOK include/linux/netfilter.h:257 [inline]
> ipv6_rcv+0x12df/0x2380 net/ipv6/ip6_input.c:203
> __netif_receive_skb_core+0x1ac8/0x33f0 net/core/dev.c:4179
> __netif_receive_skb+0x2a/0x170 net/core/dev.c:4217
> process_backlog+0x11e/0x730 net/core/dev.c:4837
> napi_poll net/core/dev.c:5171 [inline]
> net_rx_action+0xf0a/0x1980 net/core/dev.c:5236
> __do_softirq+0x31f/0xbe7 kernel/softirq.c:284
> do_softirq_own_stack+0x1c/0x30 arch/x86/entry/entry_64.S:902
> </IRQ>
> do_softirq.part.21+0x2c0/0x300 kernel/softirq.c:328
> do_softirq kernel/softirq.c:176 [inline]
> __local_bh_enable_ip+0x24c/0x290 kernel/softirq.c:181
> local_bh_enable include/linux/bottom_half.h:31 [inline]
> rcu_read_unlock_bh include/linux/rcupdate.h:971 [inline]
> ip6_finish_output2+0xb85/0x2380 net/ipv6/ip6_output.c:124
> ip6_finish_output+0x2f9/0x950 net/ipv6/ip6_output.c:149
> NF_HOOK_COND include/linux/netfilter.h:246 [inline]
> ip6_output+0x1cb/0x8c0 net/ipv6/ip6_output.c:163
> ip6_xmit+0xc36/0x1e80 include/net/dst.h:486
> inet6_csk_xmit+0x320/0x5d0 net/ipv6/inet6_connection_sock.c:139
> dccp_transmit_skb+0xac5/0x10e0 net/dccp/output.c:141
> dccp_xmit_packet+0x20f/0x750 net/dccp/output.c:280
> dccp_write_xmit+0x168/0x1d0 net/dccp/output.c:362
> dccp_sendmsg+0x793/0xb00 net/dccp/proto.c:796
> inet_sendmsg+0x164/0x5b0 net/ipv4/af_inet.c:761
> sock_sendmsg_nosec net/socket.c:633 [inline]
> sock_sendmsg+0xca/0x110 net/socket.c:643
> SYSC_sendto+0x660/0x810 net/socket.c:1685
> SyS_sendto+0x40/0x50 net/socket.c:1653
> entry_SYSCALL_64_fastpath+0x1f/0xc2
> RIP: 0033:0x4458d9
> RSP: 002b:00007f59f536bb58 EFLAGS: 00000282 ORIG_RAX: 000000000000002c
> RAX: ffffffffffffffda RBX: 000000000000001a RCX: 00000000004458d9
> RDX: 0000000000000037 RSI: 0000000020e62000 RDI: 000000000000001a
> RBP: 00000000006e1b90 R08: 0000000020e62fe0 R09: 0000000000000020
> R10: 0000000000008000 R11: 0000000000000282 R12: 0000000000708150
> R13: 0000000020d0aff8 R14: 0000000000000004 R15: 0000000000000000
> Object at ffff880059f18980, in cache kmalloc-64 size: 64
> Allocated:
> PID = 3932
> save_stack_trace+0x16/0x20 arch/x86/kernel/stacktrace.c:57
> save_stack+0x43/0xd0 mm/kasan/kasan.c:502
> set_track mm/kasan/kasan.c:514 [inline]
> kasan_kmalloc+0xaa/0xd0 mm/kasan/kasan.c:605
> kmem_cache_alloc_trace+0x10b/0x6e0 mm/slab.c:3637
> kmalloc include/linux/slab.h:490 [inline]
> dccp_feat_entry_new+0x20b/0x400 net/dccp/feat.c:467
> dccp_feat_push_change+0x38/0x220 net/dccp/feat.c:487
> __feat_register_sp+0x223/0x2f0 net/dccp/feat.c:741
> dccp_feat_propagate_ccid+0x22b/0x2b0 net/dccp/feat.c:949
> dccp_feat_server_ccid_dependencies+0x1b3/0x250 net/dccp/feat.c:1012
> dccp_make_response+0x1f1/0xc50 net/dccp/output.c:423
> dccp_v6_send_response+0x4ec/0xc20 net/dccp/ipv6.c:217
> dccp_v6_conn_request+0xab2/0x1100 net/dccp/ipv6.c:377
> dccp_rcv_state_process+0x50c/0x1650 net/dccp/input.c:606
> dccp_v6_do_rcv+0x20d/0x350 net/dccp/ipv6.c:632
> sk_backlog_rcv include/net/sock.h:896 [inline]
> __sk_receive_skb+0x350/0xca0 net/core/sock.c:479
> dccp_v6_rcv+0xb17/0x1d10 net/dccp/ipv6.c:742
> ip6_input_finish+0x45b/0x1700 net/ipv6/ip6_input.c:279
> NF_HOOK include/linux/netfilter.h:257 [inline]
> ip6_input+0xdb/0x580 net/ipv6/ip6_input.c:322
> dst_input include/net/dst.h:492 [inline]
> ip6_rcv_finish+0x194/0x720 net/ipv6/ip6_input.c:69
> NF_HOOK include/linux/netfilter.h:257 [inline]
> ipv6_rcv+0x12df/0x2380 net/ipv6/ip6_input.c:203
> __netif_receive_skb_core+0x1ac8/0x33f0 net/core/dev.c:4179
> __netif_receive_skb+0x2a/0x170 net/core/dev.c:4217
> process_backlog+0x11e/0x730 net/core/dev.c:4837
> napi_poll net/core/dev.c:5171 [inline]
> net_rx_action+0xf0a/0x1980 net/core/dev.c:5236
> __do_softirq+0x31f/0xbe7 kernel/softirq.c:284
> Freed:
> PID = 24
> save_stack_trace+0x16/0x20 arch/x86/kernel/stacktrace.c:57
> save_stack+0x43/0xd0 mm/kasan/kasan.c:502
> set_track mm/kasan/kasan.c:514 [inline]
> kasan_slab_free+0x6f/0xb0 mm/kasan/kasan.c:578
> __cache_free mm/slab.c:3513 [inline]
> kfree+0xd3/0x250 mm/slab.c:3830
> dccp_feat_entry_destructor.part.4+0x48/0x60 net/dccp/feat.c:418
> dccp_feat_entry_destructor net/dccp/feat.c:416 [inline]
> dccp_feat_list_pop net/dccp/feat.c:541 [inline]
> dccp_feat_activate_values+0x57f/0xa90 net/dccp/feat.c:1543
> dccp_create_openreq_child+0x464/0x610 net/dccp/minisocks.c:121
> dccp_v6_request_recv_sock+0x1f1/0x17e0 net/dccp/ipv6.c:457
> dccp_check_req+0x335/0x5a0 net/dccp/minisocks.c:186
> dccp_v6_rcv+0x653/0x1d10 net/dccp/ipv6.c:711
> ip6_input_finish+0x45b/0x1700 net/ipv6/ip6_input.c:279
> NF_HOOK include/linux/netfilter.h:257 [inline]
> ip6_input+0xdb/0x580 net/ipv6/ip6_input.c:322
> dst_input include/net/dst.h:492 [inline]
> ip6_rcv_finish+0x194/0x720 net/ipv6/ip6_input.c:69
> NF_HOOK include/linux/netfilter.h:257 [inline]
> ipv6_rcv+0x12df/0x2380 net/ipv6/ip6_input.c:203
> __netif_receive_skb_core+0x1ac8/0x33f0 net/core/dev.c:4179
> __netif_receive_skb+0x2a/0x170 net/core/dev.c:4217
> process_backlog+0x11e/0x730 net/core/dev.c:4837
> napi_poll net/core/dev.c:5171 [inline]
> net_rx_action+0xf0a/0x1980 net/core/dev.c:5236
> __do_softirq+0x31f/0xbe7 kernel/softirq.c:284
> Memory state around the buggy address:
> ffff880059f18880: 00 00 00 00 00 fc fc fc fc fc fc fc fc fc fc fc
> ffff880059f18900: 00 00 00 00 00 fc fc fc fc fc fc fc fc fc fc fc
>>ffff880059f18980: fb fb fb fb fb fb fb fb fc fc fc fc fc fc fc fc
> ^
> ffff880059f18a00: fb fb fb fb fb fb fb fb fc fc fc fc fc fc fc fc
> ffff880059f18a80: 00 00 00 00 00 fc fc fc fc fc fc fc fc fc fc fc
> ==================================================================
Hit this while trying to reproduce another WARNING in networking
stack, so now I suspect that it may be an induced WARNING and not sure
if it worth pursuing spending more time on it...
WARNING: CPU: 1 PID: 29691 at net/core/stream.c:198
sk_stream_kill_queues+0x401/0x570 net/core/stream.c:198
Call Trace:
__dump_stack lib/dump_stack.c:15 [inline]
dump_stack+0x2ee/0x3ef lib/dump_stack.c:51
panic+0x1fb/0x412 kernel/panic.c:179
__warn+0x1c4/0x1e0 kernel/panic.c:539
warn_slowpath_null+0x2c/0x40 kernel/panic.c:582
sk_stream_kill_queues+0x401/0x570 net/core/stream.c:198
inet_csk_destroy_sock+0x175/0x450 net/ipv4/inet_connection_sock.c:708
dccp_close+0x951/0xc60 net/dccp/proto.c:1067
inet_release+0xed/0x1c0 net/ipv4/af_inet.c:425
inet6_release+0x50/0x70 net/ipv6/af_inet6.c:431
sock_release+0x8d/0x1e0 net/socket.c:599
sock_close+0x16/0x20 net/socket.c:1063
__fput+0x332/0x7f0 fs/file_table.c:208
____fput+0x15/0x20 fs/file_table.c:244
task_work_run+0x18a/0x260 kernel/task_work.c:116
exit_task_work include/linux/task_work.h:21 [inline]
do_exit+0x18ef/0x28b0 kernel/exit.c:841
do_group_exit+0x149/0x420 kernel/exit.c:945
get_signal+0x7e0/0x1820 kernel/signal.c:2313
do_signal+0xd2/0x2190 arch/x86/kernel/signal.c:807
exit_to_usermode_loop+0x200/0x2a0 arch/x86/entry/common.c:156
prepare_exit_to_usermode arch/x86/entry/common.c:190 [inline]
syscall_return_slowpath+0x4d3/0x570 arch/x86/entry/common.c:259
entry_SYSCALL_64_fastpath+0xc0/0xc2
On Fri, Mar 3, 2017 at 4:06 PM, Dmitry Vyukov <[email protected]> wrote:
> On Fri, Mar 3, 2017 at 3:48 PM, Eric Dumazet <[email protected]> wrote:
>> On Fri, 2017-03-03 at 06:32 -0800, Eric Dumazet wrote:
>>> On Fri, 2017-03-03 at 15:11 +0100, Dmitry Vyukov wrote:
>>> > On Mon, Feb 13, 2017 at 11:29 PM, Cong Wang <[email protected]> wrote:
>>> > > On Mon, Feb 13, 2017 at 11:19 AM, Andrey Konovalov
>>> > > <[email protected]> wrote:
>>> > >> Hi,
>>> > >>
>>> > >> I've got the following error report while fuzzing the kernel with syzkaller.
>>> > >>
>>> > >> On commit 926af6273fc683cd98cd0ce7bf0d04a02eed6742.
>>> > >>
>>> > >> A reproducer and .config are attached.
>>> > >> Note, that it takes quite some time to trigger the bug (up to 10 minutes).
>>> > >>
>>> > >> BUG: KASAN: use-after-free in dccp_feat_activate_values+0x967/0xab0
>>> > >> net/dccp/feat.c:1541 at addr ffff88003713be68
>>> > >> Read of size 8 by task syz-executor2/8457
>>> > >> CPU: 2 PID: 8457 Comm: syz-executor2 Not tainted 4.10.0-rc7+ #127
>>> > >> Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Bochs 01/01/2011
>>> > >> Call Trace:
>>> > >> <IRQ>
>>> > >> __dump_stack lib/dump_stack.c:15 [inline]
>>> > >> dump_stack+0x292/0x398 lib/dump_stack.c:51
>>> > >> kasan_object_err+0x1c/0x70 mm/kasan/report.c:162
>>> > >> print_address_description mm/kasan/report.c:200 [inline]
>>> > >> kasan_report_error mm/kasan/report.c:289 [inline]
>>> > >> kasan_report.part.1+0x20e/0x4e0 mm/kasan/report.c:311
>>> > >> kasan_report mm/kasan/report.c:332 [inline]
>>> > >> __asan_report_load8_noabort+0x29/0x30 mm/kasan/report.c:332
>>> > >> dccp_feat_activate_values+0x967/0xab0 net/dccp/feat.c:1541
>>> > >> dccp_create_openreq_child+0x464/0x610 net/dccp/minisocks.c:121
>>> > >> dccp_v6_request_recv_sock+0x1f6/0x1960 net/dccp/ipv6.c:457
>>> > >> dccp_check_req+0x335/0x5a0 net/dccp/minisocks.c:186
>>> > >> dccp_v6_rcv+0x69e/0x1d00 net/dccp/ipv6.c:711
>>> > >> ip6_input_finish+0x46d/0x17a0 net/ipv6/ip6_input.c:279
>>> > >> NF_HOOK include/linux/netfilter.h:257 [inline]
>>> > >> ip6_input+0xdb/0x590 net/ipv6/ip6_input.c:322
>>> > >> dst_input include/net/dst.h:507 [inline]
>>> > >> ip6_rcv_finish+0x289/0x890 net/ipv6/ip6_input.c:69
>>> > >> NF_HOOK include/linux/netfilter.h:257 [inline]
>>> > >> ipv6_rcv+0x12ec/0x23d0 net/ipv6/ip6_input.c:203
>>> > >> __netif_receive_skb_core+0x1ae5/0x3400 net/core/dev.c:4190
>>> > >> __netif_receive_skb+0x2a/0x170 net/core/dev.c:4228
>>> > >> process_backlog+0xe5/0x6c0 net/core/dev.c:4839
>>> > >> napi_poll net/core/dev.c:5202 [inline]
>>> > >> net_rx_action+0xe70/0x1900 net/core/dev.c:5267
>>> > >> __do_softirq+0x2fb/0xb7d kernel/softirq.c:284
>>> > >> do_softirq_own_stack+0x1c/0x30 arch/x86/entry/entry_64.S:902
>>> > >
>>> > >
>>> > > Seems there is a race condition between iterating dccp_feat_entry
>>> > > and freeing it, bh_lock_sock() seems not held in this path.
>>> >
>>> >
>>> >
>>> > Cong, where exactly do we need to add bh_lock_sock()?
>>> >
>>> > I am still seeing this on 4977ab6e92e267afe9d8f78438c3db330ca8434c
>>>
>>>
>>> I would try :
>>
>> Or something that would compile. I will take a deeper look after my
>> commute.
>
>
> Something that compiles is definitely better :)
> Reapplied.
>
>
>> diff --git a/net/dccp/ipv4.c b/net/dccp/ipv4.c
>> index 409d0cfd34474812c3bf74f26cd423a3d65ee441..56f883b301ccd610fc24efeac4fb47d3c2f95ecf 100644
>> --- a/net/dccp/ipv4.c
>> +++ b/net/dccp/ipv4.c
>> @@ -482,7 +482,11 @@ static int dccp_v4_send_response(const struct sock *sk, struct request_sock *req
>> if (dst == NULL)
>> goto out;
>>
>> + /* DCCP is not ready yet for lockless SYN processing */
>> + bh_lock_sock((struct sock *)sk);
>> skb = dccp_make_response(sk, dst, req);
>> + bh_unlock_sock((struct sock *)sk);
>> +
>> if (skb != NULL) {
>> const struct inet_request_sock *ireq = inet_rsk(req);
>> struct dccp_hdr *dh = dccp_hdr(skb);
>> diff --git a/net/dccp/ipv6.c b/net/dccp/ipv6.c
>> index 233b57367758c64c09ed40f7359cb8fcb1918d93..673f45f85b7c755c8165c6274ffb6b1fe5660683 100644
>> --- a/net/dccp/ipv6.c
>> +++ b/net/dccp/ipv6.c
>> @@ -214,7 +214,11 @@ static int dccp_v6_send_response(const struct sock *sk, struct request_sock *req
>> goto done;
>> }
>>
>> + /* DCCP is not ready yet for lockless SYN processing */
>> + bh_lock_sock((struct sock *)sk);
>> skb = dccp_make_response(sk, dst, req);
>> + bh_unlock_sock((struct sock *)sk);
>> +
>> if (skb != NULL) {
>> struct dccp_hdr *dh = dccp_hdr(skb);
>> struct ipv6_txoptions *opt;
>>
>>
The first bot that picked this up started spewing:
BUG: spinlock recursion on CPU#1, syz-executor2/9452
lock: 0xffff8801cd09abc8, .magic: dead4ead, .owner:
syz-executor2/9452, .owner_cpu: 1
CPU: 1 PID: 9452 Comm: syz-executor2 Not tainted 4.10.0+ #14
Hardware name: Google Google Compute Engine/Google Compute Engine,
BIOS Google 01/01/2011
Call Trace:
<IRQ>
__dump_stack lib/dump_stack.c:15 [inline]
dump_stack+0x2ee/0x3ef lib/dump_stack.c:51
spin_dump+0x14d/0x280 kernel/locking/spinlock_debug.c:67
spin_bug kernel/locking/spinlock_debug.c:75 [inline]
debug_spin_lock_before kernel/locking/spinlock_debug.c:84 [inline]
do_raw_spin_lock+0x3dd/0x5b0 kernel/locking/spinlock_debug.c:135
__raw_spin_lock include/linux/spinlock_api_smp.h:145 [inline]
_raw_spin_lock+0x3b/0x50 kernel/locking/spinlock.c:151
spin_lock include/linux/spinlock.h:302 [inline]
dccp_v6_send_response+0x4f0/0xc40 net/dccp/ipv6.c:217
dccp_v6_conn_request+0xab2/0x1100 net/dccp/ipv6.c:379
dccp_rcv_state_process+0x51d/0x1680 net/dccp/input.c:611
dccp_v6_do_rcv+0x20d/0x350 net/dccp/ipv6.c:634
sk_backlog_rcv include/net/sock.h:893 [inline]
__sk_receive_skb+0x350/0xca0 net/core/sock.c:479
dccp_v6_rcv+0xb17/0x1d10 net/dccp/ipv6.c:744
ip6_input_finish+0x45b/0x1700 net/ipv6/ip6_input.c:279
NF_HOOK include/linux/netfilter.h:257 [inline]
ip6_input+0xdb/0x580 net/ipv6/ip6_input.c:322
dst_input include/net/dst.h:507 [inline]
ip6_rcv_finish+0x194/0x720 net/ipv6/ip6_input.c:69
NF_HOOK include/linux/netfilter.h:257 [inline]
ipv6_rcv+0x12d6/0x23b0 net/ipv6/ip6_input.c:203
__netif_receive_skb_core+0x1fb9/0x3390 net/core/dev.c:4224
__netif_receive_skb+0x2a/0x170 net/core/dev.c:4262
process_backlog+0x11e/0x730 net/core/dev.c:4873
napi_poll net/core/dev.c:5236 [inline]
net_rx_action+0xeb4/0x1580 net/core/dev.c:5301
__do_softirq+0x31f/0xbe7 kernel/softirq.c:284
do_softirq_own_stack+0x1c/0x30 arch/x86/entry/entry_64.S:902
</IRQ>
do_softirq.part.21+0x2c0/0x300 kernel/softirq.c:328
do_softirq kernel/softirq.c:176 [inline]
__local_bh_enable_ip+0x24c/0x290 kernel/softirq.c:181
local_bh_enable include/linux/bottom_half.h:31 [inline]
rcu_read_unlock_bh include/linux/rcupdate.h:971 [inline]
ip6_finish_output2+0xb0c/0x22f0 net/ipv6/ip6_output.c:123
ip6_finish_output+0x2f9/0x950 net/ipv6/ip6_output.c:148
NF_HOOK_COND include/linux/netfilter.h:246 [inline]
ip6_output+0x1cb/0x8c0 net/ipv6/ip6_output.c:162
ip6_xmit+0xc36/0x1e90 include/net/dst.h:501
inet6_csk_xmit+0x320/0x5d0 net/ipv6/inet6_connection_sock.c:179
dccp_transmit_skb+0xac5/0x10e0 net/dccp/output.c:141
dccp_connect+0x376/0x680 net/dccp/output.c:563
dccp_v6_connect+0xcbe/0x18c0 net/dccp/ipv6.c:916
__inet_stream_connect+0x882/0xea0 net/ipv4/af_inet.c:604
inet_stream_connect+0x55/0xa0 net/ipv4/af_inet.c:665
SYSC_connect+0x251/0x580 net/socket.c:1579
SyS_connect+0x24/0x30 net/socket.c:1560
entry_SYSCALL_64_fastpath+0x1f/0xc2
RIP: 0033:0x44fb79
RSP: 002b:00007fa5aef1ab58 EFLAGS: 00000212 ORIG_RAX: 000000000000002a
RAX: ffffffffffffffda RBX: 0000000020e5afe0 RCX: 000000000044fb79
RDX: 0000000000000020 RSI: 0000000020e5afe0 RDI: 0000000000000005
RBP: 0000000000000005 R08: 0000000000000000 R09: 0000000000000000
R10: 0000000000000000 R11: 0000000000000212 R12: 0000000000708000
R13: 0000000000000006 R14: 0000000000000000 R15: 0000000000000000
On Fri, 2017-03-03 at 06:32 -0800, Eric Dumazet wrote:
> On Fri, 2017-03-03 at 15:11 +0100, Dmitry Vyukov wrote:
> > On Mon, Feb 13, 2017 at 11:29 PM, Cong Wang <[email protected]> wrote:
> > > On Mon, Feb 13, 2017 at 11:19 AM, Andrey Konovalov
> > > <[email protected]> wrote:
> > >> Hi,
> > >>
> > >> I've got the following error report while fuzzing the kernel with syzkaller.
> > >>
> > >> On commit 926af6273fc683cd98cd0ce7bf0d04a02eed6742.
> > >>
> > >> A reproducer and .config are attached.
> > >> Note, that it takes quite some time to trigger the bug (up to 10 minutes).
> > >>
> > >> BUG: KASAN: use-after-free in dccp_feat_activate_values+0x967/0xab0
> > >> net/dccp/feat.c:1541 at addr ffff88003713be68
> > >> Read of size 8 by task syz-executor2/8457
> > >> CPU: 2 PID: 8457 Comm: syz-executor2 Not tainted 4.10.0-rc7+ #127
> > >> Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Bochs 01/01/2011
> > >> Call Trace:
> > >> <IRQ>
> > >> __dump_stack lib/dump_stack.c:15 [inline]
> > >> dump_stack+0x292/0x398 lib/dump_stack.c:51
> > >> kasan_object_err+0x1c/0x70 mm/kasan/report.c:162
> > >> print_address_description mm/kasan/report.c:200 [inline]
> > >> kasan_report_error mm/kasan/report.c:289 [inline]
> > >> kasan_report.part.1+0x20e/0x4e0 mm/kasan/report.c:311
> > >> kasan_report mm/kasan/report.c:332 [inline]
> > >> __asan_report_load8_noabort+0x29/0x30 mm/kasan/report.c:332
> > >> dccp_feat_activate_values+0x967/0xab0 net/dccp/feat.c:1541
> > >> dccp_create_openreq_child+0x464/0x610 net/dccp/minisocks.c:121
> > >> dccp_v6_request_recv_sock+0x1f6/0x1960 net/dccp/ipv6.c:457
> > >> dccp_check_req+0x335/0x5a0 net/dccp/minisocks.c:186
> > >> dccp_v6_rcv+0x69e/0x1d00 net/dccp/ipv6.c:711
> > >> ip6_input_finish+0x46d/0x17a0 net/ipv6/ip6_input.c:279
> > >> NF_HOOK include/linux/netfilter.h:257 [inline]
> > >> ip6_input+0xdb/0x590 net/ipv6/ip6_input.c:322
> > >> dst_input include/net/dst.h:507 [inline]
> > >> ip6_rcv_finish+0x289/0x890 net/ipv6/ip6_input.c:69
> > >> NF_HOOK include/linux/netfilter.h:257 [inline]
> > >> ipv6_rcv+0x12ec/0x23d0 net/ipv6/ip6_input.c:203
> > >> __netif_receive_skb_core+0x1ae5/0x3400 net/core/dev.c:4190
> > >> __netif_receive_skb+0x2a/0x170 net/core/dev.c:4228
> > >> process_backlog+0xe5/0x6c0 net/core/dev.c:4839
> > >> napi_poll net/core/dev.c:5202 [inline]
> > >> net_rx_action+0xe70/0x1900 net/core/dev.c:5267
> > >> __do_softirq+0x2fb/0xb7d kernel/softirq.c:284
> > >> do_softirq_own_stack+0x1c/0x30 arch/x86/entry/entry_64.S:902
> > >
> > >
> > > Seems there is a race condition between iterating dccp_feat_entry
> > > and freeing it, bh_lock_sock() seems not held in this path.
> >
> >
> >
> > Cong, where exactly do we need to add bh_lock_sock()?
> >
> > I am still seeing this on 4977ab6e92e267afe9d8f78438c3db330ca8434c
>
>
> I would try :
Or something that would compile. I will take a deeper look after my
commute.
diff --git a/net/dccp/ipv4.c b/net/dccp/ipv4.c
index 409d0cfd34474812c3bf74f26cd423a3d65ee441..56f883b301ccd610fc24efeac4fb47d3c2f95ecf 100644
--- a/net/dccp/ipv4.c
+++ b/net/dccp/ipv4.c
@@ -482,7 +482,11 @@ static int dccp_v4_send_response(const struct sock *sk, struct request_sock *req
if (dst == NULL)
goto out;
+ /* DCCP is not ready yet for lockless SYN processing */
+ bh_lock_sock((struct sock *)sk);
skb = dccp_make_response(sk, dst, req);
+ bh_unlock_sock((struct sock *)sk);
+
if (skb != NULL) {
const struct inet_request_sock *ireq = inet_rsk(req);
struct dccp_hdr *dh = dccp_hdr(skb);
diff --git a/net/dccp/ipv6.c b/net/dccp/ipv6.c
index 233b57367758c64c09ed40f7359cb8fcb1918d93..673f45f85b7c755c8165c6274ffb6b1fe5660683 100644
--- a/net/dccp/ipv6.c
+++ b/net/dccp/ipv6.c
@@ -214,7 +214,11 @@ static int dccp_v6_send_response(const struct sock *sk, struct request_sock *req
goto done;
}
+ /* DCCP is not ready yet for lockless SYN processing */
+ bh_lock_sock((struct sock *)sk);
skb = dccp_make_response(sk, dst, req);
+ bh_unlock_sock((struct sock *)sk);
+
if (skb != NULL) {
struct dccp_hdr *dh = dccp_hdr(skb);
struct ipv6_txoptions *opt;
On Fri, 2017-03-03 at 16:06 +0100, Dmitry Vyukov wrote:
> Something that compiles is definitely better :)
> Reapplied.
Just to be clear : This is not the proper patch. This only reduces the
race.
bh_lock_sock() does not prevent a user process from owning the socket.
We need another protection, probably RCU based, or another spinlock
protecting the fields needed at SYNACK generation.
On Fri, Mar 3, 2017 at 7:12 AM, Dmitry Vyukov <[email protected]> wrote:
> The first bot that picked this up started spewing:
>
> BUG: spinlock recursion on CPU#1, syz-executor2/9452
Yes. The bug is not about locking the listener, but protecting fields
of struct dccp_request_sock
I will provide a patch, once I reach the office and after the breakfast ;)
On Fri, Mar 3, 2017 at 3:32 PM, Eric Dumazet <[email protected]> wrote:
>> > On Mon, Feb 13, 2017 at 11:19 AM, Andrey Konovalov
>> > <[email protected]> wrote:
>> >> Hi,
>> >>
>> >> I've got the following error report while fuzzing the kernel with syzkaller.
>> >>
>> >> On commit 926af6273fc683cd98cd0ce7bf0d04a02eed6742.
>> >>
>> >> A reproducer and .config are attached.
>> >> Note, that it takes quite some time to trigger the bug (up to 10 minutes).
>> >>
>> >> BUG: KASAN: use-after-free in dccp_feat_activate_values+0x967/0xab0
>> >> net/dccp/feat.c:1541 at addr ffff88003713be68
>> >> Read of size 8 by task syz-executor2/8457
>> >> CPU: 2 PID: 8457 Comm: syz-executor2 Not tainted 4.10.0-rc7+ #127
>> >> Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Bochs 01/01/2011
>> >> Call Trace:
>> >> <IRQ>
>> >> __dump_stack lib/dump_stack.c:15 [inline]
>> >> dump_stack+0x292/0x398 lib/dump_stack.c:51
>> >> kasan_object_err+0x1c/0x70 mm/kasan/report.c:162
>> >> print_address_description mm/kasan/report.c:200 [inline]
>> >> kasan_report_error mm/kasan/report.c:289 [inline]
>> >> kasan_report.part.1+0x20e/0x4e0 mm/kasan/report.c:311
>> >> kasan_report mm/kasan/report.c:332 [inline]
>> >> __asan_report_load8_noabort+0x29/0x30 mm/kasan/report.c:332
>> >> dccp_feat_activate_values+0x967/0xab0 net/dccp/feat.c:1541
>> >> dccp_create_openreq_child+0x464/0x610 net/dccp/minisocks.c:121
>> >> dccp_v6_request_recv_sock+0x1f6/0x1960 net/dccp/ipv6.c:457
>> >> dccp_check_req+0x335/0x5a0 net/dccp/minisocks.c:186
>> >> dccp_v6_rcv+0x69e/0x1d00 net/dccp/ipv6.c:711
>> >> ip6_input_finish+0x46d/0x17a0 net/ipv6/ip6_input.c:279
>> >> NF_HOOK include/linux/netfilter.h:257 [inline]
>> >> ip6_input+0xdb/0x590 net/ipv6/ip6_input.c:322
>> >> dst_input include/net/dst.h:507 [inline]
>> >> ip6_rcv_finish+0x289/0x890 net/ipv6/ip6_input.c:69
>> >> NF_HOOK include/linux/netfilter.h:257 [inline]
>> >> ipv6_rcv+0x12ec/0x23d0 net/ipv6/ip6_input.c:203
>> >> __netif_receive_skb_core+0x1ae5/0x3400 net/core/dev.c:4190
>> >> __netif_receive_skb+0x2a/0x170 net/core/dev.c:4228
>> >> process_backlog+0xe5/0x6c0 net/core/dev.c:4839
>> >> napi_poll net/core/dev.c:5202 [inline]
>> >> net_rx_action+0xe70/0x1900 net/core/dev.c:5267
>> >> __do_softirq+0x2fb/0xb7d kernel/softirq.c:284
>> >> do_softirq_own_stack+0x1c/0x30 arch/x86/entry/entry_64.S:902
>> >
>> >
>> > Seems there is a race condition between iterating dccp_feat_entry
>> > and freeing it, bh_lock_sock() seems not held in this path.
>>
>>
>>
>> Cong, where exactly do we need to add bh_lock_sock()?
>>
>> I am still seeing this on 4977ab6e92e267afe9d8f78438c3db330ca8434c
>
>
> I would try :
>
> diff --git a/net/dccp/ipv4.c b/net/dccp/ipv4.c
> index 409d0cfd34474812c3bf74f26cd423a3d65ee441..5a8b5ac5edaaf35428ab04cc810d98310bd169ed 100644
> --- a/net/dccp/ipv4.c
> +++ b/net/dccp/ipv4.c
> @@ -482,7 +482,9 @@ static int dccp_v4_send_response(const struct sock *sk, struct request_sock *req
> if (dst == NULL)
> goto out;
>
> + bh_lock_sock(sk);
> skb = dccp_make_response(sk, dst, req);
> + bh_unlock_sock(sk);
> if (skb != NULL) {
> const struct inet_request_sock *ireq = inet_rsk(req);
> struct dccp_hdr *dh = dccp_hdr(skb);
> diff --git a/net/dccp/ipv6.c b/net/dccp/ipv6.c
> index 233b57367758c64c09ed40f7359cb8fcb1918d93..e89cc88d14c22d411a91afab093e209fcbb816d8 100644
> --- a/net/dccp/ipv6.c
> +++ b/net/dccp/ipv6.c
> @@ -214,7 +214,9 @@ static int dccp_v6_send_response(const struct sock *sk, struct request_sock *req
> goto done;
> }
>
> + bh_lock_sock(sk);
> skb = dccp_make_response(sk, dst, req);
> + bh_unlock_sock(sk);
> if (skb != NULL) {
> struct dccp_hdr *dh = dccp_hdr(skb);
> struct ipv6_txoptions *opt;
Applied on bots. Thanks!
On Fri, 2017-03-03 at 15:11 +0100, Dmitry Vyukov wrote:
> On Mon, Feb 13, 2017 at 11:29 PM, Cong Wang <[email protected]> wrote:
> > On Mon, Feb 13, 2017 at 11:19 AM, Andrey Konovalov
> > <[email protected]> wrote:
> >> Hi,
> >>
> >> I've got the following error report while fuzzing the kernel with syzkaller.
> >>
> >> On commit 926af6273fc683cd98cd0ce7bf0d04a02eed6742.
> >>
> >> A reproducer and .config are attached.
> >> Note, that it takes quite some time to trigger the bug (up to 10 minutes).
> >>
> >> BUG: KASAN: use-after-free in dccp_feat_activate_values+0x967/0xab0
> >> net/dccp/feat.c:1541 at addr ffff88003713be68
> >> Read of size 8 by task syz-executor2/8457
> >> CPU: 2 PID: 8457 Comm: syz-executor2 Not tainted 4.10.0-rc7+ #127
> >> Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Bochs 01/01/2011
> >> Call Trace:
> >> <IRQ>
> >> __dump_stack lib/dump_stack.c:15 [inline]
> >> dump_stack+0x292/0x398 lib/dump_stack.c:51
> >> kasan_object_err+0x1c/0x70 mm/kasan/report.c:162
> >> print_address_description mm/kasan/report.c:200 [inline]
> >> kasan_report_error mm/kasan/report.c:289 [inline]
> >> kasan_report.part.1+0x20e/0x4e0 mm/kasan/report.c:311
> >> kasan_report mm/kasan/report.c:332 [inline]
> >> __asan_report_load8_noabort+0x29/0x30 mm/kasan/report.c:332
> >> dccp_feat_activate_values+0x967/0xab0 net/dccp/feat.c:1541
> >> dccp_create_openreq_child+0x464/0x610 net/dccp/minisocks.c:121
> >> dccp_v6_request_recv_sock+0x1f6/0x1960 net/dccp/ipv6.c:457
> >> dccp_check_req+0x335/0x5a0 net/dccp/minisocks.c:186
> >> dccp_v6_rcv+0x69e/0x1d00 net/dccp/ipv6.c:711
> >> ip6_input_finish+0x46d/0x17a0 net/ipv6/ip6_input.c:279
> >> NF_HOOK include/linux/netfilter.h:257 [inline]
> >> ip6_input+0xdb/0x590 net/ipv6/ip6_input.c:322
> >> dst_input include/net/dst.h:507 [inline]
> >> ip6_rcv_finish+0x289/0x890 net/ipv6/ip6_input.c:69
> >> NF_HOOK include/linux/netfilter.h:257 [inline]
> >> ipv6_rcv+0x12ec/0x23d0 net/ipv6/ip6_input.c:203
> >> __netif_receive_skb_core+0x1ae5/0x3400 net/core/dev.c:4190
> >> __netif_receive_skb+0x2a/0x170 net/core/dev.c:4228
> >> process_backlog+0xe5/0x6c0 net/core/dev.c:4839
> >> napi_poll net/core/dev.c:5202 [inline]
> >> net_rx_action+0xe70/0x1900 net/core/dev.c:5267
> >> __do_softirq+0x2fb/0xb7d kernel/softirq.c:284
> >> do_softirq_own_stack+0x1c/0x30 arch/x86/entry/entry_64.S:902
> >
> >
> > Seems there is a race condition between iterating dccp_feat_entry
> > and freeing it, bh_lock_sock() seems not held in this path.
>
>
>
> Cong, where exactly do we need to add bh_lock_sock()?
>
> I am still seeing this on 4977ab6e92e267afe9d8f78438c3db330ca8434c
I would try :
diff --git a/net/dccp/ipv4.c b/net/dccp/ipv4.c
index 409d0cfd34474812c3bf74f26cd423a3d65ee441..5a8b5ac5edaaf35428ab04cc810d98310bd169ed 100644
--- a/net/dccp/ipv4.c
+++ b/net/dccp/ipv4.c
@@ -482,7 +482,9 @@ static int dccp_v4_send_response(const struct sock *sk, struct request_sock *req
if (dst == NULL)
goto out;
+ bh_lock_sock(sk);
skb = dccp_make_response(sk, dst, req);
+ bh_unlock_sock(sk);
if (skb != NULL) {
const struct inet_request_sock *ireq = inet_rsk(req);
struct dccp_hdr *dh = dccp_hdr(skb);
diff --git a/net/dccp/ipv6.c b/net/dccp/ipv6.c
index 233b57367758c64c09ed40f7359cb8fcb1918d93..e89cc88d14c22d411a91afab093e209fcbb816d8 100644
--- a/net/dccp/ipv6.c
+++ b/net/dccp/ipv6.c
@@ -214,7 +214,9 @@ static int dccp_v6_send_response(const struct sock *sk, struct request_sock *req
goto done;
}
+ bh_lock_sock(sk);
skb = dccp_make_response(sk, dst, req);
+ bh_unlock_sock(sk);
if (skb != NULL) {
struct dccp_hdr *dh = dccp_hdr(skb);
struct ipv6_txoptions *opt;
On Fri, 2017-03-03 at 07:22 -0800, Eric Dumazet wrote:
> On Fri, Mar 3, 2017 at 7:12 AM, Dmitry Vyukov <[email protected]> wrote:
> > The first bot that picked this up started spewing:
> >
> > BUG: spinlock recursion on CPU#1, syz-executor2/9452
>
> Yes. The bug is not about locking the listener, but protecting fields
> of struct dccp_request_sock
>
> I will provide a patch, once I reach the office and after the breakfast ;)
OK here is what I suggest to fix the races.
diff --git a/include/linux/dccp.h b/include/linux/dccp.h
index 61d042bbbf607253033d9948b291cab2322814ba..68449293c4b6233c1a1d4133b1819376a9310225 100644
--- a/include/linux/dccp.h
+++ b/include/linux/dccp.h
@@ -163,6 +163,7 @@ struct dccp_request_sock {
__u64 dreq_isr;
__u64 dreq_gsr;
__be32 dreq_service;
+ spinlock_t dreq_lock;
struct list_head dreq_featneg;
__u32 dreq_timestamp_echo;
__u32 dreq_timestamp_time;
diff --git a/net/dccp/minisocks.c b/net/dccp/minisocks.c
index e267e6f4c9a5566b369a03a600a408e5bd41cbad..abd07a443219853b022bef41cb072e90ff8f07f0 100644
--- a/net/dccp/minisocks.c
+++ b/net/dccp/minisocks.c
@@ -142,6 +142,13 @@ struct sock *dccp_check_req(struct sock *sk, struct sk_buff *skb,
struct dccp_request_sock *dreq = dccp_rsk(req);
bool own_req;
+ /* TCP/DCCP listeners became lockless.
+ * DCCP stores complex state in its request_sock, so we need
+ * a protection for them, now this code runs without being protected
+ * by the parent (listener) lock.
+ */
+ spin_lock_bh(&dreq->dreq_lock);
+
/* Check for retransmitted REQUEST */
if (dccp_hdr(skb)->dccph_type == DCCP_PKT_REQUEST) {
@@ -156,7 +163,7 @@ struct sock *dccp_check_req(struct sock *sk, struct sk_buff *skb,
inet_rtx_syn_ack(sk, req);
}
/* Network Duplicate, discard packet */
- return NULL;
+ goto out;
}
DCCP_SKB_CB(skb)->dccpd_reset_code = DCCP_RESET_CODE_PACKET_ERROR;
@@ -182,20 +189,20 @@ struct sock *dccp_check_req(struct sock *sk, struct sk_buff *skb,
child = inet_csk(sk)->icsk_af_ops->syn_recv_sock(sk, skb, req, NULL,
req, &own_req);
- if (!child)
- goto listen_overflow;
-
- return inet_csk_complete_hashdance(sk, child, req, own_req);
+ if (child) {
+ child = inet_csk_complete_hashdance(sk, child, req, own_req);
+ goto out;
+ }
-listen_overflow:
- dccp_pr_debug("listen_overflow!\n");
DCCP_SKB_CB(skb)->dccpd_reset_code = DCCP_RESET_CODE_TOO_BUSY;
drop:
if (dccp_hdr(skb)->dccph_type != DCCP_PKT_RESET)
req->rsk_ops->send_reset(sk, skb);
inet_csk_reqsk_queue_drop(sk, req);
- return NULL;
+out:
+ spin_unlock_bh(&dreq->dreq_lock);
+ return child;
}
EXPORT_SYMBOL_GPL(dccp_check_req);
@@ -246,6 +253,7 @@ int dccp_reqsk_init(struct request_sock *req,
{
struct dccp_request_sock *dreq = dccp_rsk(req);
+ spin_lock_init(&dreq->dreq_lock);
inet_rsk(req)->ir_rmt_port = dccp_hdr(skb)->dccph_sport;
inet_rsk(req)->ir_num = ntohs(dccp_hdr(skb)->dccph_dport);
inet_rsk(req)->acked = 0;
On Fri, Mar 3, 2017 at 3:48 PM, Eric Dumazet <[email protected]> wrote:
> On Fri, 2017-03-03 at 06:32 -0800, Eric Dumazet wrote:
>> On Fri, 2017-03-03 at 15:11 +0100, Dmitry Vyukov wrote:
>> > On Mon, Feb 13, 2017 at 11:29 PM, Cong Wang <[email protected]> wrote:
>> > > On Mon, Feb 13, 2017 at 11:19 AM, Andrey Konovalov
>> > > <[email protected]> wrote:
>> > >> Hi,
>> > >>
>> > >> I've got the following error report while fuzzing the kernel with syzkaller.
>> > >>
>> > >> On commit 926af6273fc683cd98cd0ce7bf0d04a02eed6742.
>> > >>
>> > >> A reproducer and .config are attached.
>> > >> Note, that it takes quite some time to trigger the bug (up to 10 minutes).
>> > >>
>> > >> BUG: KASAN: use-after-free in dccp_feat_activate_values+0x967/0xab0
>> > >> net/dccp/feat.c:1541 at addr ffff88003713be68
>> > >> Read of size 8 by task syz-executor2/8457
>> > >> CPU: 2 PID: 8457 Comm: syz-executor2 Not tainted 4.10.0-rc7+ #127
>> > >> Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Bochs 01/01/2011
>> > >> Call Trace:
>> > >> <IRQ>
>> > >> __dump_stack lib/dump_stack.c:15 [inline]
>> > >> dump_stack+0x292/0x398 lib/dump_stack.c:51
>> > >> kasan_object_err+0x1c/0x70 mm/kasan/report.c:162
>> > >> print_address_description mm/kasan/report.c:200 [inline]
>> > >> kasan_report_error mm/kasan/report.c:289 [inline]
>> > >> kasan_report.part.1+0x20e/0x4e0 mm/kasan/report.c:311
>> > >> kasan_report mm/kasan/report.c:332 [inline]
>> > >> __asan_report_load8_noabort+0x29/0x30 mm/kasan/report.c:332
>> > >> dccp_feat_activate_values+0x967/0xab0 net/dccp/feat.c:1541
>> > >> dccp_create_openreq_child+0x464/0x610 net/dccp/minisocks.c:121
>> > >> dccp_v6_request_recv_sock+0x1f6/0x1960 net/dccp/ipv6.c:457
>> > >> dccp_check_req+0x335/0x5a0 net/dccp/minisocks.c:186
>> > >> dccp_v6_rcv+0x69e/0x1d00 net/dccp/ipv6.c:711
>> > >> ip6_input_finish+0x46d/0x17a0 net/ipv6/ip6_input.c:279
>> > >> NF_HOOK include/linux/netfilter.h:257 [inline]
>> > >> ip6_input+0xdb/0x590 net/ipv6/ip6_input.c:322
>> > >> dst_input include/net/dst.h:507 [inline]
>> > >> ip6_rcv_finish+0x289/0x890 net/ipv6/ip6_input.c:69
>> > >> NF_HOOK include/linux/netfilter.h:257 [inline]
>> > >> ipv6_rcv+0x12ec/0x23d0 net/ipv6/ip6_input.c:203
>> > >> __netif_receive_skb_core+0x1ae5/0x3400 net/core/dev.c:4190
>> > >> __netif_receive_skb+0x2a/0x170 net/core/dev.c:4228
>> > >> process_backlog+0xe5/0x6c0 net/core/dev.c:4839
>> > >> napi_poll net/core/dev.c:5202 [inline]
>> > >> net_rx_action+0xe70/0x1900 net/core/dev.c:5267
>> > >> __do_softirq+0x2fb/0xb7d kernel/softirq.c:284
>> > >> do_softirq_own_stack+0x1c/0x30 arch/x86/entry/entry_64.S:902
>> > >
>> > >
>> > > Seems there is a race condition between iterating dccp_feat_entry
>> > > and freeing it, bh_lock_sock() seems not held in this path.
>> >
>> >
>> >
>> > Cong, where exactly do we need to add bh_lock_sock()?
>> >
>> > I am still seeing this on 4977ab6e92e267afe9d8f78438c3db330ca8434c
>>
>>
>> I would try :
>
> Or something that would compile. I will take a deeper look after my
> commute.
Something that compiles is definitely better :)
Reapplied.
> diff --git a/net/dccp/ipv4.c b/net/dccp/ipv4.c
> index 409d0cfd34474812c3bf74f26cd423a3d65ee441..56f883b301ccd610fc24efeac4fb47d3c2f95ecf 100644
> --- a/net/dccp/ipv4.c
> +++ b/net/dccp/ipv4.c
> @@ -482,7 +482,11 @@ static int dccp_v4_send_response(const struct sock *sk, struct request_sock *req
> if (dst == NULL)
> goto out;
>
> + /* DCCP is not ready yet for lockless SYN processing */
> + bh_lock_sock((struct sock *)sk);
> skb = dccp_make_response(sk, dst, req);
> + bh_unlock_sock((struct sock *)sk);
> +
> if (skb != NULL) {
> const struct inet_request_sock *ireq = inet_rsk(req);
> struct dccp_hdr *dh = dccp_hdr(skb);
> diff --git a/net/dccp/ipv6.c b/net/dccp/ipv6.c
> index 233b57367758c64c09ed40f7359cb8fcb1918d93..673f45f85b7c755c8165c6274ffb6b1fe5660683 100644
> --- a/net/dccp/ipv6.c
> +++ b/net/dccp/ipv6.c
> @@ -214,7 +214,11 @@ static int dccp_v6_send_response(const struct sock *sk, struct request_sock *req
> goto done;
> }
>
> + /* DCCP is not ready yet for lockless SYN processing */
> + bh_lock_sock((struct sock *)sk);
> skb = dccp_make_response(sk, dst, req);
> + bh_unlock_sock((struct sock *)sk);
> +
> if (skb != NULL) {
> struct dccp_hdr *dh = dccp_hdr(skb);
> struct ipv6_txoptions *opt;
>
>
On Fri, Mar 3, 2017 at 5:00 PM, Eric Dumazet <[email protected]> wrote:
> On Fri, 2017-03-03 at 07:22 -0800, Eric Dumazet wrote:
>> On Fri, Mar 3, 2017 at 7:12 AM, Dmitry Vyukov <[email protected]> wrote:
>> > The first bot that picked this up started spewing:
>> >
>> > BUG: spinlock recursion on CPU#1, syz-executor2/9452
>>
>> Yes. The bug is not about locking the listener, but protecting fields
>> of struct dccp_request_sock
>>
>> I will provide a patch, once I reach the office and after the breakfast ;)
>
> OK here is what I suggest to fix the races.
I've applied the patch 2 days ago and it stopped happening since then.
Please mail an official patch.
Tested-by: Dmitry Vyukov <[email protected]>
> diff --git a/include/linux/dccp.h b/include/linux/dccp.h
> index 61d042bbbf607253033d9948b291cab2322814ba..68449293c4b6233c1a1d4133b1819376a9310225 100644
> --- a/include/linux/dccp.h
> +++ b/include/linux/dccp.h
> @@ -163,6 +163,7 @@ struct dccp_request_sock {
> __u64 dreq_isr;
> __u64 dreq_gsr;
> __be32 dreq_service;
> + spinlock_t dreq_lock;
> struct list_head dreq_featneg;
> __u32 dreq_timestamp_echo;
> __u32 dreq_timestamp_time;
> diff --git a/net/dccp/minisocks.c b/net/dccp/minisocks.c
> index e267e6f4c9a5566b369a03a600a408e5bd41cbad..abd07a443219853b022bef41cb072e90ff8f07f0 100644
> --- a/net/dccp/minisocks.c
> +++ b/net/dccp/minisocks.c
> @@ -142,6 +142,13 @@ struct sock *dccp_check_req(struct sock *sk, struct sk_buff *skb,
> struct dccp_request_sock *dreq = dccp_rsk(req);
> bool own_req;
>
> + /* TCP/DCCP listeners became lockless.
> + * DCCP stores complex state in its request_sock, so we need
> + * a protection for them, now this code runs without being protected
> + * by the parent (listener) lock.
> + */
> + spin_lock_bh(&dreq->dreq_lock);
> +
> /* Check for retransmitted REQUEST */
> if (dccp_hdr(skb)->dccph_type == DCCP_PKT_REQUEST) {
>
> @@ -156,7 +163,7 @@ struct sock *dccp_check_req(struct sock *sk, struct sk_buff *skb,
> inet_rtx_syn_ack(sk, req);
> }
> /* Network Duplicate, discard packet */
> - return NULL;
> + goto out;
> }
>
> DCCP_SKB_CB(skb)->dccpd_reset_code = DCCP_RESET_CODE_PACKET_ERROR;
> @@ -182,20 +189,20 @@ struct sock *dccp_check_req(struct sock *sk, struct sk_buff *skb,
>
> child = inet_csk(sk)->icsk_af_ops->syn_recv_sock(sk, skb, req, NULL,
> req, &own_req);
> - if (!child)
> - goto listen_overflow;
> -
> - return inet_csk_complete_hashdance(sk, child, req, own_req);
> + if (child) {
> + child = inet_csk_complete_hashdance(sk, child, req, own_req);
> + goto out;
> + }
>
> -listen_overflow:
> - dccp_pr_debug("listen_overflow!\n");
> DCCP_SKB_CB(skb)->dccpd_reset_code = DCCP_RESET_CODE_TOO_BUSY;
> drop:
> if (dccp_hdr(skb)->dccph_type != DCCP_PKT_RESET)
> req->rsk_ops->send_reset(sk, skb);
>
> inet_csk_reqsk_queue_drop(sk, req);
> - return NULL;
> +out:
> + spin_unlock_bh(&dreq->dreq_lock);
> + return child;
> }
>
> EXPORT_SYMBOL_GPL(dccp_check_req);
> @@ -246,6 +253,7 @@ int dccp_reqsk_init(struct request_sock *req,
> {
> struct dccp_request_sock *dreq = dccp_rsk(req);
>
> + spin_lock_init(&dreq->dreq_lock);
> inet_rsk(req)->ir_rmt_port = dccp_hdr(skb)->dccph_sport;
> inet_rsk(req)->ir_num = ntohs(dccp_hdr(skb)->dccph_dport);
> inet_rsk(req)->acked = 0;
>
>