Starting to see those on 3.8.4 (never saw in 3.8.2) stable kernel on a few systems
during LTP run,
[11297.597242] BUG: unable to handle kernel paging request at 00000000fffffffe
[11297.598022] IP: [] kmem_cache_alloc+0x68/0x1e0
[11297.598022] PGD 7b9eb067 PUD 0
[11297.598022] Oops: 0000 [#2] SMP
[11297.598022] Modules linked in: cmtp kernelcapi bnep scsi_transport_iscsi rfcomm l2tp_ppp l2tp_netlink l2tp_core hidp ipt_ULOG af_key nfc rds pppoe pppox ppp_generic slhc af_802154 atm ip6table_filter ip6_tables iptable_filter ip_tables btrfs zlib_deflate vfat fat nfs_layout_nfsv41_files nfsv4 auth_rpcgss nfsv3 nfs_acl nfsv2 nfs lockd sunrpc fscache nfnetlink_log nfnetlink bluetooth rfkill arc4 md4 nls_utf8 cifs dns_resolver nf_tproxy_core nls_koi8_u nls_cp932 ts_kmp sctp sg kvm_amd kvm virtio_balloon i2c_piix4 pcspkr xfs libcrc32c ata_generic pata_acpi cirrus drm_kms_helper ttm ata_piix virtio_net drm libata virtio_blk i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ipt_REJECT]
[11297.598022] CPU 1
[11297.598022] Pid: 14134, comm: ltp-pan Tainted: G D 3.8.4+ #1 Bochs Bochs
[11297.598022] RIP: 0010:[] [] kmem_cache_alloc+0x68/0x1e0
[11297.598022] RSP: 0018:ffff8800447dbdd0 EFLAGS: 00010246
[11297.598022] RAX: 0000000000000000 RBX: ffff88007c169970 RCX: 00000000018acdcd
[11297.598022] RDX: 000000000006c104 RSI: 00000000000080d0 RDI: ffff88007d04ac00
[11297.598022] RBP: ffff8800447dbe10 R08: 0000000000017620 R09: ffffffff810fe2e2
[11297.598022] R10: 0000000000000000 R11: 0000000000000000 R12: 00000000fffffffe
[11297.598022] R13: 00000000000080d0 R14: ffff88007d04ac00 R15: ffff88007d04ac00
[11297.598022] FS: 00007f09c29b4740(0000) GS:ffff88007fd00000(0000) knlGS:00000000f74d86c0
[11297.598022] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[11297.598022] CR2: 00000000fffffffe CR3: 0000000037213000 CR4: 00000000000006e0
[11297.598022] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[11297.598022] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[11297.598022] Process ltp-pan (pid: 14134, threadinfo ffff8800447da000, task ffff8800551ab2e0)
[11297.598022] Stack:
[11297.598022] ffffffff810fe2e2 ffffffff8108cf0f 0000000001200011 ffff88007c169970
[11297.598022] 0000000000000000 00007f09c29b4a10 0000000000000000 ffff88007c169970
[11297.598022] ffff8800447dbe30 ffffffff810fe2e2 0000000000000000 0000000001200011
[11297.598022] Call Trace:
[11297.598022] [] ? __delayacct_tsk_init+0x22/0x40
[11297.598022] [] ? prepare_creds+0xdf/0x190
[11297.598022] [] __delayacct_tsk_init+0x22/0x40
[11297.598022] [] copy_process.part.25+0x31f/0x13f0
[11297.598022] [] do_fork+0xa9/0x350
[11297.598022] [] sys_clone+0x16/0x20
[11297.598022] [] stub_clone+0x69/0x90
[11297.598022] [] ? system_call_fastpath+0x16/0x1b
[11297.598022] Code: 90 4d 89 fe 4d 8b 06 65 4c 03 04 25 c8 db 00 00 49 8b 50 08 4d 8b 20 4d 85 e4 0f 84 2b 01 00 00 49 63 46 20 4d 8b 06 41 f6 c0 0f <49> 8b 1c 04 0f 85 55 01 00 00 48 8d 4a 01 4c 89 e0 65 49 0f c7
[11297.598022] RIP [] kmem_cache_alloc+0x68/0x1e0
[11297.598022] RSP
[11297.598022] CR2: 00000000fffffffe
[11297.727799] ---[ end trace 037bde72f23b34d2 ]---
Never saw this in mainline but only something like this wondering could be related
(that kmem_cache_alloc also in the trace).
[12124.201919] INFO: task kworker/2:1:166 blocked for more than 120 seconds.
[12124.242758] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[12124.289801] kworker/2:1 D ffff88081fc54440 0 166 2 0x00000000
[12124.330784] ffff88081361ba68 0000000000000046 ffff880813568000 ffff88081361bfd8
[12124.373694] ffff88081361bfd8 ffff88081361bfd8 ffff8808144fb2e0 ffff880813568000
[12124.416896] 0000000000000000 ffff880813568000 ffff8808133f8930 0000000000000002
[12124.458674] Call Trace:
[12124.473291] [] schedule+0x29/0x70
[12124.502143] [] rwsem_down_failed_common+0xda/0x230
[12124.539311] [] rwsem_down_write_failed+0x13/0x20
[12124.575585] [] call_rwsem_down_write_failed+0x13/0x20
[12124.614129] [] ? down_write+0x32/0x40
[12124.644703] [] xlog_cil_push+0x89/0x3c0 [xfs]
[12124.680046] [] ? up+0x32/0x50
[12124.706083] [] ? flush_work+0x113/0x170
[12124.738078] [] xlog_cil_force_lsn+0xf7/0x160 [xfs]
[12124.776062] [] ? xfs_trans_free_items+0x88/0xb0 [xfs]
[12124.814503] [] _xfs_log_force_lsn+0x5a/0x2e0 [xfs]
[12124.851512] [] xfs_trans_commit+0x263/0x270 [xfs]
[12124.887996] [] xfs_fs_log_dummy+0x61/0x90 [xfs]
[12124.924015] [] ? xfs_log_need_covered+0x93/0xc0 [xfs]
[12124.963079] [] xfs_log_worker+0x48/0x50 [xfs]
[12124.997404] [] process_one_work+0x174/0x3d0
[12125.031408] [] worker_thread+0x10f/0x390
[12125.062936] [] ? busy_worker_rebind_fn+0xb0/0xb0
[12125.098924] [] kthread+0xc0/0xd0
[12125.126124] [] ? kthread_create_on_node+0x120/0x120
[12125.162995] [] ret_from_fork+0x7c/0xb0
[12125.193516] [] ? kthread_create_on_node+0x120/0x120
[12125.229431] INFO: task beah-beaker-bac:3331 blocked for more than 120 seconds.
[12125.269795] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disabfd5bb59b0 0000000000000086 ffff881fbf770000 ffff881fd5bb5fd8
[12129.329892] ffff881fd5bb5fd8 ffff8Trace:
[12134.346772] [] schedule+0x29/0x70
[12134.401771] 30
[12136.376743] [] ? kmem_cache_alloc+0x35/0x1e0
[12136.411867] [] rwsem_down_read_failed+0x15/0x17
[12136.448141] [] call_rwsem_down_read_failed+0x14/0x30
[12136.487438] [] ? kmem_alloc+0x67/0xf0 [xfs]
[12136.521108] [] ? down_read+0x24/0x2b
[12136.549333] [] xfs_log_commit_cil+0x1a6/0x4a0 [xfs]
[12136.586227] [] ? kmem_zone_alloc+0x67/0xf0 [xfs]
[12136.621792] [] xfs_trans_commit+0x134/0x270 [xfs]
[12136.658163] [] xfs_vn_update_time+0xf7/0x1a0 [xfs]
[12136.694257] [] update_time+0x23/0xc0
[12136.722821] [] ? mnt_clone_write+0x12/0x30
[12136.755240] [] file_update_time+0x98/0xf0
[12136.785989] [] xfs_file_aio_write_checks+0xdb/0xf0 [xfs]
[12136.825592] [] xfs_file_buffered_aio_write+0x7b/0x1a0 [xfs]
[12136.868827] [] xfs_file_aio_write+0xf9/0x160 [xfs]
[12136.907037] [] do_sync_write+0xa7/0xe0
[12136.939287] [] vfs_write+0xac/0x180
[12136.969067] [] sys_pwrite64+0x9a/0xb0
[12137.000528] [] system_call_fastpath+0x16/0x1b
[12137.036022] INFO: task master:3497 blocked for more than 120 seconds.
[12137.073730] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[12137.117609] master D ffff88101fd54440 0 3497 1 0x00000080
[12137.155846] ffff8808091a3a60 0000000000000082 ffff880813079970 ffff8808091a3fd8
[12137.197460] ffff8808091a3fd8 ffff8808091a3fd8 ffff8808145432e0 ffff880813079970
[12137.239430] ffff8808091a3a88 ffff880813079970 ffff8808133f8930 0000000000000001
[12137.279786] Call Trace:
[12137.293627] [] schedule+0x29/0x70
[12137.321390] [] rwsem_down_failed_common+0xda/0x230
[12137.357677] [] ? __enqueue_entity+0x78/0x80
[12137.390943] [] ? kmem_cache_alloc+0x35/0x1e0
[12137.426089] [] rwsem_down_read_failed+0x15/0x17
[12137.462395] [] call_rwsem_down_read_failed+0x14/0x30
[12137.502332] [] ? kmem_alloc+0x67/0xf0 [xfs]
[12137.537843] [] ? down_read+0x24/
Any idea?
CAI Qian
On Fri, 22 Mar 2013, CAI Qian wrote:
> Starting to see those on 3.8.4 (never saw in 3.8.2) stable kernel on a few systems
> during LTP run,
>
> [11297.597242] BUG: unable to handle kernel paging request at 00000000fffffffe
> [11297.598022] IP: [] kmem_cache_alloc+0x68/0x1e0
Is this repeatable? Do you have CONFIG_SLAB or CONFIG_SLUB enabled?
> [11297.598022] PGD 7b9eb067 PUD 0
> [11297.598022] Oops: 0000 [#2] SMP
> [11297.598022] Modules linked in: cmtp kernelcapi bnep scsi_transport_iscsi rfcomm l2tp_ppp l2tp_netlink l2tp_core hidp ipt_ULOG af_key nfc rds pppoe pppox ppp_generic slhc af_802154 atm ip6table_filter ip6_tables iptable_filter ip_tables btrfs zlib_deflate vfat fat nfs_layout_nfsv41_files nfsv4 auth_rpcgss nfsv3 nfs_acl nfsv2 nfs lockd sunrpc fscache nfnetlink_log nfnetlink bluetooth rfkill arc4 md4 nls_utf8 cifs dns_resolver nf_tproxy_core nls_koi8_u nls_cp932 ts_kmp sctp sg kvm_amd kvm virtio_balloon i2c_piix4 pcspkr xfs libcrc32c ata_generic pata_acpi cirrus drm_kms_helper ttm ata_piix virtio_net drm libata virtio_blk i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ipt_REJECT]
> [11297.598022] CPU 1
> [11297.598022] Pid: 14134, comm: ltp-pan Tainted: G D 3.8.4+ #1 Bochs Bochs
> [11297.598022] RIP: 0010:[] [] kmem_cache_alloc+0x68/0x1e0
> [11297.598022] RSP: 0018:ffff8800447dbdd0 EFLAGS: 00010246
> [11297.598022] RAX: 0000000000000000 RBX: ffff88007c169970 RCX: 00000000018acdcd
> [11297.598022] RDX: 000000000006c104 RSI: 00000000000080d0 RDI: ffff88007d04ac00
> [11297.598022] RBP: ffff8800447dbe10 R08: 0000000000017620 R09: ffffffff810fe2e2
> [11297.598022] R10: 0000000000000000 R11: 0000000000000000 R12: 00000000fffffffe
> [11297.598022] R13: 00000000000080d0 R14: ffff88007d04ac00 R15: ffff88007d04ac00
> [11297.598022] FS: 00007f09c29b4740(0000) GS:ffff88007fd00000(0000) knlGS:00000000f74d86c0
> [11297.598022] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
> [11297.598022] CR2: 00000000fffffffe CR3: 0000000037213000 CR4: 00000000000006e0
> [11297.598022] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
> [11297.598022] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
> [11297.598022] Process ltp-pan (pid: 14134, threadinfo ffff8800447da000, task ffff8800551ab2e0)
> [11297.598022] Stack:
> [11297.598022] ffffffff810fe2e2 ffffffff8108cf0f 0000000001200011 ffff88007c169970
> [11297.598022] 0000000000000000 00007f09c29b4a10 0000000000000000 ffff88007c169970
> [11297.598022] ffff8800447dbe30 ffffffff810fe2e2 0000000000000000 0000000001200011
> [11297.598022] Call Trace:
> [11297.598022] [] ? __delayacct_tsk_init+0x22/0x40
> [11297.598022] [] ? prepare_creds+0xdf/0x190
> [11297.598022] [] __delayacct_tsk_init+0x22/0x40
> [11297.598022] [] copy_process.part.25+0x31f/0x13f0
> [11297.598022] [] do_fork+0xa9/0x350
> [11297.598022] [] sys_clone+0x16/0x20
> [11297.598022] [] stub_clone+0x69/0x90
> [11297.598022] [] ? system_call_fastpath+0x16/0x1b
> [11297.598022] Code: 90 4d 89 fe 4d 8b 06 65 4c 03 04 25 c8 db 00 00 49 8b 50 08 4d 8b 20 4d 85 e4 0f 84 2b 01 00 00 49 63 46 20 4d 8b 06 41 f6 c0 0f <49> 8b 1c 04 0f 85 55 01 00 00 48 8d 4a 01 4c 89 e0 65 49 0f c7
> [11297.598022] RIP [] kmem_cache_alloc+0x68/0x1e0
> [11297.598022] RSP
> [11297.598022] CR2: 00000000fffffffe
> [11297.727799] ---[ end trace 037bde72f23b34d2 ]---
>
> Never saw this in mainline but only something like this wondering could be related
> (that kmem_cache_alloc also in the trace).
>
These are unrelated.
----- Original Message -----
> From: "David Rientjes" <[email protected]>
> To: "CAI Qian" <[email protected]>
> Cc: "linux-mm" kvack.org>, [email protected], "Oleg Nesterov" <[email protected]>
> Sent: Friday, March 22, 2013 5:35:34 PM
> Subject: Re: BUG at kmem_cache_alloc
>
> On Fri, 22 Mar 2013, CAI Qian wrote:
>
> > Starting to see those on 3.8.4 (never saw in 3.8.2) stable kernel
> > on a few systems
> > during LTP run,
> >
> > [11297.597242] BUG: unable to handle kernel paging request at
> > 00000000fffffffe
> > [11297.598022] IP: [] kmem_cache_alloc+0x68/0x1e0
>
> Is this repeatable? Do you have CONFIG_SLAB or CONFIG_SLUB enabled?
Saw it on 2 systems so far - one HP server and one KVM guest. Still
trying to reproduce. Used CONFIG_SLUB=y.
CAI Qian
>
> > [11297.598022] PGD 7b9eb067 PUD 0
> > [11297.598022] Oops: 0000 [#2] SMP
> > [11297.598022] Modules linked in: cmtp kernelcapi bnep
> > scsi_transport_iscsi rfcomm l2tp_ppp l2tp_netlink l2tp_core hidp
> > ipt_ULOG af_key nfc rds pppoe pppox ppp_generic slhc af_802154 atm
> > ip6table_filter ip6_tables iptable_filter ip_tables btrfs
> > zlib_deflate vfat fat nfs_layout_nfsv41_files nfsv4 auth_rpcgss
> > nfsv3 nfs_acl nfsv2 nfs lockd sunrpc fscache nfnetlink_log
> > nfnetlink bluetooth rfkill arc4 md4 nls_utf8 cifs dns_resolver
> > nf_tproxy_core nls_koi8_u nls_cp932 ts_kmp sctp sg kvm_amd kvm
> > virtio_balloon i2c_piix4 pcspkr xfs libcrc32c ata_generic
> > pata_acpi cirrus drm_kms_helper ttm ata_piix virtio_net drm libata
> > virtio_blk i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod
> > [last unloaded: ipt_REJECT]
> > [11297.598022] CPU 1
> > [11297.598022] Pid: 14134, comm: ltp-pan Tainted: G D
> > 3.8.4+ #1 Bochs Bochs
> > [11297.598022] RIP: 0010:[] [] kmem_cache_alloc+0x68/0x1e0
> > [11297.598022] RSP: 0018:ffff8800447dbdd0 EFLAGS: 00010246
> > [11297.598022] RAX: 0000000000000000 RBX: ffff88007c169970 RCX:
> > 00000000018acdcd
> > [11297.598022] RDX: 000000000006c104 RSI: 00000000000080d0 RDI:
> > ffff88007d04ac00
> > [11297.598022] RBP: ffff8800447dbe10 R08: 0000000000017620 R09:
> > ffffffff810fe2e2
> > [11297.598022] R10: 0000000000000000 R11: 0000000000000000 R12:
> > 00000000fffffffe
> > [11297.598022] R13: 00000000000080d0 R14: ffff88007d04ac00 R15:
> > ffff88007d04ac00
> > [11297.598022] FS: 00007f09c29b4740(0000)
> > GS:ffff88007fd00000(0000) knlGS:00000000f74d86c0
> > [11297.598022] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
> > [11297.598022] CR2: 00000000fffffffe CR3: 0000000037213000 CR4:
> > 00000000000006e0
> > [11297.598022] DR0: 0000000000000000 DR1: 0000000000000000 DR2:
> > 0000000000000000
> > [11297.598022] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7:
> > 0000000000000400
> > [11297.598022] Process ltp-pan (pid: 14134, threadinfo
> > ffff8800447da000, task ffff8800551ab2e0)
> > [11297.598022] Stack:
> > [11297.598022] ffffffff810fe2e2 ffffffff8108cf0f 0000000001200011
> > ffff88007c169970
> > [11297.598022] 0000000000000000 00007f09c29b4a10 0000000000000000
> > ffff88007c169970
> > [11297.598022] ffff8800447dbe30 ffffffff810fe2e2 0000000000000000
> > 0000000001200011
> > [11297.598022] Call Trace:
> > [11297.598022] [] ? __delayacct_tsk_init+0x22/0x40
> > [11297.598022] [] ? prepare_creds+0xdf/0x190
> > [11297.598022] [] __delayacct_tsk_init+0x22/0x40
> > [11297.598022] [] copy_process.part.25+0x31f/0x13f0
> > [11297.598022] [] do_fork+0xa9/0x350
> > [11297.598022] [] sys_clone+0x16/0x20
> > [11297.598022] [] stub_clone+0x69/0x90
> > [11297.598022] [] ? system_call_fastpath+0x16/0x1b
> > [11297.598022] Code: 90 4d 89 fe 4d 8b 06 65 4c 03 04 25 c8 db 00
> > 00 49 8b 50 08 4d 8b 20 4d 85 e4 0f 84 2b 01 00 00 49 63 46 20 4d
> > 8b 06 41 f6 c0 0f <49> 8b 1c 04 0f 85 55 01 00 00 48 8d 4a 01 4c
> > 89 e0 65 49 0f c7
> > [11297.598022] RIP [] kmem_cache_alloc+0x68/0x1e0
> > [11297.598022] RSP
> > [11297.598022] CR2: 00000000fffffffe
> > [11297.727799] ---[ end trace 037bde72f23b34d2 ]---
> >
> > Never saw this in mainline but only something like this wondering
> > could be related
> > (that kmem_cache_alloc also in the trace).
> >
>
> These are unrelated.
>
> --
> To unsubscribe, send a message with 'unsubscribe linux-mm' in
> the body to [email protected]. For more info on Linux MM,
> see: http://www.linux-mm.org/ .
> Don't email: href=mailto:"[email protected]"> [email protected]
>
----- Original Message -----
> From: "CAI Qian" <[email protected]>
> To: "David Rientjes" <[email protected]>
> Cc: "linux-mm" <[email protected]>, [email protected], "Oleg Nesterov" <[email protected]>
> Sent: Monday, March 25, 2013 1:37:50 PM
> Subject: Re: BUG at kmem_cache_alloc
>
>
>
> ----- Original Message -----
> > From: "David Rientjes" <[email protected]>
> > To: "CAI Qian" <[email protected]>
> > Cc: "linux-mm" kvack.org>, [email protected], "Oleg
> > Nesterov" <[email protected]>
> > Sent: Friday, March 22, 2013 5:35:34 PM
> > Subject: Re: BUG at kmem_cache_alloc
> >
> > On Fri, 22 Mar 2013, CAI Qian wrote:
> >
> > > Starting to see those on 3.8.4 (never saw in 3.8.2) stable kernel
> > > on a few systems
> > > during LTP run,
> > >
> > > [11297.597242] BUG: unable to handle kernel paging request at
> > > 00000000fffffffe
> > > [11297.598022] IP: [] kmem_cache_alloc+0x68/0x1e0
> >
> > Is this repeatable? Do you have CONFIG_SLAB or CONFIG_SLUB
> > enabled?
> Saw it on 2 systems so far - one HP server and one KVM guest. Still
This happened again during trinity run. Bisecting is in-progress...
CAI Qian
[ 9236.418633] BUG: unable to handle kernel paging request at 0000000000010000
[ 9236.420033] IP: [<ffffffff8118a008>] kmem_cache_alloc+0x68/0x200
[ 9236.421405] PGD 147308067 PUD 149456067 PMD 0
[ 9236.422476] Oops: 0000 [#1] SMP
[ 9236.423182] Modules linked in: tun(F+) cmtp(F) kernelcapi(F) hidp(F) rfcomm(F) bnep(F) l2tp_ppp(F) l2tp_netlink(F) l2tp_core(F) ipt_ULOG(F) scsi_transport_iscsi(F) af_802154(F) rds(F) af_key(F) pppoe(F) pppox(F) ppp_generic(F) slhc(F) nfc(F) atm(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) btrfs(F) zlib_deflate(F) vfat(F) fat(F) nfs_layout_nfsv41_files(F) nfsv4(F) auth_rpcgss(F) nfsv3(F) nfs_acl(F) nfsv2(F) nfs(F) lockd(F) sunrpc(F) fscache(F) nfnetlink_log(F) nfnetlink(F) bluetooth(F) rfkill(F) arc4(F) md4(F) nls_utf8(F) cifs(F) dns_resolver(F) nf_tproxy_core(F) nls_koi8_u(F) nls_cp932(F) ts_kmp(F) sctp(F) fuse(F) sg(F) kvm_amd(F) kvm(F) amd64_edac_mod(F) edac_mce_amd(F) bnx2x(F) serio_raw(F) edac_core(F) netxen_nic(F) mdio(F) k10temp(F) microcode(F) i2c_piix4(F) ipmi_si(F) ipmi_msghandler(F) shpchp(F) hpwdt(F) hpilo(F) xfs(F) libcrc32c(F) sd_mod(F) crc_t10dif(F) radeon(F) i2c_algo_bit(F) drm_kms_helper(F) sata_svw(F) ttm(F) libata(F) drm(F) i2c_core(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F) [last unloaded: ipt_REJECT]
[ 9236.444373] CPU 3
[ 9236.444885] Pid: 3495, comm: modprobe Tainted: GF W 3.8.4 #1 HP ProLiant BL495c G5
[ 9236.446808] RIP: 0010:[<ffffffff8118a008>] [<ffffffff8118a008>] kmem_cache_alloc+0x68/0x200
[ 9236.448757] RSP: 0018:ffff8801206c3c88 EFLAGS: 00010246
[ 9236.449946] RAX: 0000000000000000 RBX: 000000000000000f RCX: 0000000000000124
[ 9236.451531] RDX: 000000000002c644 RSI: 00000000000080d0 RDI: ffff88014b040c00
[ 9236.453123] RBP: ffff8801206c3cd8 R08: 0000000000017690 R09: ffffffff81215c94
[ 9236.454735] R10: 0000000000004380 R11: 0000000000000001 R12: 0000000000010000
[ 9236.456373] R13: 00000000000080d0 R14: ffff88014b040c00 R15: ffff88014b040c00
[ 9236.457993] FS: 00007f92ce2d5740(0000) GS:ffff88014fd80000(0000) knlGS:00000000f75546c0
[ 9236.459803] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[ 9236.461081] CR2: 0000000000010000 CR3: 00000001360c9000 CR4: 00000000000007e0
[ 9236.462682] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 9236.464279] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[ 9236.465875] Process modprobe (pid: 3495, threadinfo ffff8801206c2000, task ffff8801463065c0)
[ 9236.467747] Stack:
[ 9236.468260] ffff8801206c3ca8 ffffffff812156c5 ffffffff81215c94 ffffffff812156c5
[ 9236.469903] ffff8801495405ae 000000000000000f 0000000000000002 ffff8800a2862d10
[ 9236.471569] 0000000000000000 0000000000008124 ffff8801206c3d28 ffffffff81215c94
[ 9236.473315] Call Trace:
[ 9236.473862] [<ffffffff812156c5>] ? sysfs_link_sibling+0xb5/0xe0
[ 9236.475216] [<ffffffff81215c94>] ? sysfs_new_dirent+0x54/0x110
[ 9236.476571] [<ffffffff812156c5>] ? sysfs_link_sibling+0xb5/0xe0
[ 9236.477970] [<ffffffff81215c94>] sysfs_new_dirent+0x54/0x110
[ 9236.479253] [<ffffffff81215ebc>] ? sysfs_add_one+0x2c/0x100
[ 9236.480548] [<ffffffff8121517b>] sysfs_add_file_mode+0x6b/0xe0
[ 9236.481889] [<ffffffff81217f70>] internal_create_group+0xd0/0x230
[ 9236.483288] [<ffffffff81218103>] sysfs_create_group+0x13/0x20
[ 9236.484613] [<ffffffff810c90dc>] load_module+0x138c/0x1660
[ 9236.485865] [<ffffffff8131abe0>] ? ddebug_proc_open+0xc0/0xc0
[ 9236.487174] [<ffffffff810c9487>] sys_init_module+0xd7/0x120
[ 9236.488498] [<ffffffff8161f7d9>] system_call_fastpath+0x16/0x1b
[ 9236.489797] Code: 90 4d 89 fe 4d 8b 06 65 4c 03 04 25 c8 db 00 00 49 8b 50 08 4d 8b 20 4d 85 e4 0f 84 1c 01 00 00 49 63 46 20 4d 8b 06 41 f6 c0 0f <49> 8b 1c 04 0f 85 46 01 00 00 48 8d 4a 01 4c 89 e0 65 49 0f c7
[ 9236.494164] RIP [<ffffffff8118a008>] kmem_cache_alloc+0x68/0x200
[ 9236.495550] RSP <ffff8801206c3c88>
[ 9236.496287] CR2: 0000000000010000
[ 9236.513347] ---[ end trace 3567090873e2c5de ]---
[watchdog] 9407744 iterations. [F:8626700 S:781126]
[ 9237.335465] BUG: unable to handle kernel paging request at 0000000000010000
[ 9237.336924] IP: [<ffffffff8118a008>] kmem_cache_alloc+0x68/0x200
[ 9237.338280] PGD 1489d8067 PUD 1489da067 PMD 0
[ 9237.339197] Oops: 0000 [#2] SMP
[ 9237.339945] Modules linked in: tun(F+) cmtp(F) kernelcapi(F) hidp(F) rfcomm(F) bnep(F) l2tp_ppp(F) l2tp_netlink(F) l2tp_core(F) ipt_ULOG(F) scsi_transport_iscsi(F) af_802154(F) rds(F) af_key(F) pppoe(F) pppox(F) ppp_generic(F) slhc(F) nfc(F) atm(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) btrfs(F) zlib_deflate(F) vfat(F) fat(F) nfs_layout_nfsv41_files(F) nfsv4(F) auth_rpcgss(F) nfsv3(F) nfs_acl(F) nfsv2(F) nfs(F) lockd(F) sunrpc(F) fscache(F) nfnetlink_log(F) nfnetlink(F) bluetooth(F) rfkill(F) arc4(F) md4(F) nls_utf8(F) cifs(F) dns_resolver(F) nf_tproxy_core(F) nls_koi8_u(F) nls_cp932(F) ts_kmp(F) sctp(F) fuse(F) sg(F) kvm_amd(F) kvm(F) amd64_edac_mod(F) edac_mce_amd(F) bnx2x(F) serio_raw(F) edac_core(F) netxen_nic(F) mdio(F) k10temp(F) microcode(F) i2c_piix4(F) ipmi_si(F) ipmi_msghandler(F) shpchp(F) hpwdt(F) hpilo(F) xfs(F) libcrc32c(F) sd_mod(F) crc_t10dif(F) radeon(F) i2c_algo_bit(F) drm_kms_helper(F) sata_svw(F) ttm(F) libata(F) drm(F) i2c_core(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F) [last unloaded: ipt_REJECT]
[ 9237.361067] CPU 3
[ 9237.361512] Pid: 4191, comm: rhts-test-runne Tainted: GF D W 3.8.4 #1 HP ProLiant BL495c G5
[ 9237.363861] RIP: 0010:[<ffffffff8118a008>] [<ffffffff8118a008>] kmem_cache_alloc+0x68/0x200
[ 9237.365823] RSP: 0018:ffff8801208d9dc0 EFLAGS: 00010246
[ 9237.367069] RAX: 0000000000000000 RBX: ffff8801463065c0 RCX: 0000000000000000
[ 9237.368381] RDX: 000000000002c644 RSI: 00000000000080d0 RDI: ffff88014b040c00
[ 9237.369678] RBP: ffff8801208d9e10 R08: 0000000000017690 R09: ffffffff810fe6e2
[ 9237.370967] R10: 0000000000000000 R11: ffffffffffffffe2 R12: 0000000000010000
[ 9237.372213] R13: 00000000000080d0 R14: ffff88014b040c00 R15: ffff88014b040c00
[ 9237.373498] FS: 00007fc53f905740(0000) GS:ffff88014fd80000(0000) knlGS:00000000f75546c0
[ 9237.374947] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[ 9237.376071] CR2: 0000000000010000 CR3: 000000010817d000 CR4: 00000000000007e0
[ 9237.377370] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 9237.378652] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[ 9237.379950] Process rhts-test-runne (pid: 4191, threadinfo ffff8801208d8000, task ffff880146898000)
[ 9237.381520] Stack:
[ 9237.381913] ffff8801208d9dd0 ffffffff8129aea6 ffffffff810fe6e2 ffffffff8108d29f
[ 9237.383499] 0000000001200011 ffff8801463065c0 0000000000000000 00007fc53f905a10
[ 9237.385169] 0000000000000000 ffff8801463065c0 ffff8801208d9e30 ffffffff810fe6e2
[ 9237.386871] Call Trace:
[ 9237.387484] [<ffffffff8129aea6>] ? security_prepare_creds+0x16/0x20
[ 9237.388905] [<ffffffff810fe6e2>] ? __delayacct_tsk_init+0x22/0x40
[ 9237.390303] [<ffffffff8108d29f>] ? prepare_creds+0xdf/0x190
[ 9237.391592] [<ffffffff810fe6e2>] __delayacct_tsk_init+0x22/0x40
[ 9237.392936] [<ffffffff810605df>] copy_process.part.25+0x31f/0x13f0
[ 9237.394297] [<ffffffff8129abd6>] ? security_file_alloc+0x16/0x20
[ 9237.395677] [<ffffffff811be652>] ? __alloc_fd+0x42/0x110
[ 9237.396905] [<ffffffff810617a9>] do_fork+0xa9/0x350
[ 9237.398102] [<ffffffff811be750>] ? get_unused_fd_flags+0x30/0x40
[ 9237.399466] [<ffffffff811be78e>] ? __fd_install+0x2e/0x60
[ 9237.400731] [<ffffffff81061ad6>] sys_clone+0x16/0x20
[ 9237.401866] [<ffffffff8161fb39>] stub_clone+0x69/0x90
[ 9237.403050] [<ffffffff8161f7d9>] ? system_call_fastpath+0x16/0x1b
[ 9237.404441] Code: 90 4d 89 fe 4d 8b 06 65 4c 03 04 25 c8 db 00 00 49 8b 50 08 4d 8b 20 4d 85 e4 0f 84 1c 01 00 00 49 63 46 20 4d 8b 06 41 f6 c0 0f <49> 8b 1c 04 0f 85 46 01 00 00 48 8d 4a 01 4c 89 e0 65 49 0f c7
[ 9237.408602] RIP [<ffffffff8118a008>] kmem_cache_alloc+0x68/0x200
[ 9237.410080] RSP <ffff8801208d9dc0>
[ 9237.410862] CR2: 0000000000010000
[ 9237.411687] ---[ end trace 3567090873e2c5df ]---
2013-03-25 03:10:52,681 rhts_task task_exited: INFO task_exited([Failure instance: Traceback (failure with no frames): <class 'twisted.internet.error.ProcessTerminated'>: A process has ended with a probable error condition: process ended with exit code 137.
])
2013-03-25 03:10:52,710 rhts_task on_exit: INFO quitting...
2013-03-25 03:10:52,716 rhts_task task_ended: INFO task_ended([Failure instance: Traceback (failure with no frames): <class 'twisted.internet.error.ProcessTerminated'>: A process has ended with a probable error condition: process ended with exit code 137.
])
[ 9237.541330] BUG: unable to handle kernel paging request at 0000000000010000
[ 9237.542719] IP: [<ffffffff8118a008>] kmem_cache_alloc+0x68/0x200
[ 9237.544095] PGD 133477067 PUD 148e5b067 PMD 0
[ 9237.545166] Oops: 0000 [#3] SMP
[ 9237.545868] Modules linked in: tun(F+) cmtp(F) kernelcapi(F) hidp(F ppoe(F) pppox(F) ppp_generic(F) slhc(F) nfc(F) atm(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) btrfs(F) zlib_deflate(F) vfat(F) fat(F) nfs_layout_nfsv41_files(F) nfsv4(F) auth_rpcgss(F) nfsv3(F) nfs_acl(F) nfsv2(F) nfs(F) lockd(F) sunrpc(F) fscache(F) nfnetlink_log(F) nfnetlink(F) bluetooth(F) rfkill(F) arc4(F) md4(F) nls_utf8(F) cifs(F) dns_resolver(F) nf_tproxy_core(F) nls_koi8_u(F) nls_cp932(F) ts_kmp(F) sctp(F) fuse(F) sg(F) kvm_amd(F) kvm(F) amd64_edac_mod(F) edac_mce_amd(F) bnx2x(F) serio_raw(F) edac_core(F) netxen_nic(F) mdio(F) k10temp(F) microcode(F) i2c_piix4(F) ipmi_si(F) ipmi_msghandler(F) shpchp(F) hpwdt(F) hpilo(F) xfs(F) libcrc32c(F) sd_mod(F) crc_t10dif(F) radeon(F) i2c_algo_bit(F) drm_kms_helper(F) sata_svw(F) ttm(F) libata(F) drm(F) i2c_core(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F) [last unloaded: ipt_REJECT]
[ 9237.567229] CPU 3
[ 9237.567676] Pid: 3508, comm: kworker/u:0 Tainted: GF D W 3.8.4 #1 HP ProLiant BL495c G5
[ 9237.569270] RIP: 0010:[<ffffffff8118a008>] [<ffffffff8118a008>] kmem_cache_alloc+0x68/0x200
[ 9237.570846] RSP: 0000:ffff880146cc3d98 EFLAGS: 00010246
[ 9237.571992] RAX: 0000000000000000 RBX: ffff880146304c50 RCX: 0000000000000000
[ 9237.573580] RDX: 000000000002c644 RSI: 00000000000080d0 RDI: ffff88014b040c00
[ 9237.575195] RBP: ffff880146cc3de8 R08: 0000000000017690 R09: ffffffff810fe6e2
[ 9237.576854] R10: ffff88014b001308 R11: ffffffffffffffe2 R12: 0000000000010000
[ 9237.578440] R13: 00000000000080d0 R14: ffff88014b040c00 R15: ffff88014b040c00
[ 9237.580063] FS: 00007f11cf46f740(0000) GS:ffff88014fd80000(0000) knlGS:00000000f75546c0
[ 9237.581853] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[ 9237.583140] CR2: 0000000000010000 CR3: 0000000141ad5000 CR4: 00000000000007e0
[ 9237.584740] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 9237.586319] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[ 9237.587939] Process kworker/u:0 (pid: 3508, threadinfo ffff880146cc2000, task ffff8801322ab2e0)
[ 9237.589905] Stack:
[ 9237.590333] ffff880146cc3da8 ffffffff8129aea6 ffffffff810fe6e2 ffffffff8108d29f
[ 9237.592093] 0000000000800111 ffff880146304c50 0000000000000000 0000000000000000
[ 9237.593748] 0000000000000000 ffff880146304c50 ffff880146cc3e08 ffffffff810fe6e2
[ 9237.595407] Call Trace:
[ 9237.596021] [<ffffffff8129aea6>] ? security_prepare_creds+0x16/0x20
[ 9237.597139] [<ffffffff810fe6e2>] ? __delayacct_tsk_init+0x22/0x40
[ 9237.598256] [<ffffffff8108d29f>] ? prepare_creds+0xdf/0x190
[ 9237.599344] [<ffffffff810fe6e2>] __delayacct_tsk_init+0x22/0x40
[ 9237.600433] [<ffffffff810605df>] copy_process.part.25+0x31f/0x13f0
[ 9237.601573] [<ffffffff8107cf40>] ? proc_cap_handler+0x1b0/0x1b0
[ 9237.602623] [<ffffffff810617a9>] do_fork+0xa9/0x350
[ 9237.603512] [<ffffffff81061a76>] kernel_thread+0x26/0x30
[ 9237.604495] [<ffffffff8107c738>] wait_for_helper+0x68/0xa0
[ 9237.605516] [<ffffffff81096387>] ? schedule_tail+0x27/0xb0
[ 9237.606526] [<ffffffff8107c6d0>] ? __call_usermodehelper+0xb0/0xb0
[ 9237.607674] [<ffffffff8161f72c>] ret_from_fork+0x7c/0xb0
[ 9237.608706] [<ffffffff8107c6d0>] ? __call_usermodehelper+0xb0/0xb0
[ 9237.619523] Code: 90 4d 89 fe 4d 8b 06 65 4c 03 04 25 c8 db 00 00 49 8b 50 08 4d 8b 20 4d 85 e4 0f 84 1c 01 00 00 49 63 46 20 4d 8b 06 41 f6 c0 0f <49> 8b 1c 04 0f 85 46 01 00 00 48 8d 4a 01 4c 89 e0 65 49 0f c7
[ 9237.648837] RIP [<ffffffff8118a008>] kmem_cache_alloc+0x68/0x200
[ 9237.650087] RSP <ffff880146cc3d98>
[ 9237.650832] CR2: 0000000000010000
[ 9237.666224] ---[ end trace 3567090873e2c5e0 ]---
[watchdog] 9465269 iterations. [F:8684132 S:781220]
[ 9238.441169] BUG: unable to handle kernel paging request at 0000000000010000
[ 9238.443191] IP: [<ffffffff8118a008>] kmem_cache_alloc+0x68/0x200
[ 9238.444509] PGD 129a98067 PUD 1363ef067 PMD 0
[ 9238.445574] Oops: 0000 [#4] SMP
[ 9238.446286] Modules linked in: tun(F+) cmtp(F) kernelcapi(F) hidp(F) rfcomm(F) bnep(F) l2tp_ppp(F) l2tp_netlink(F) l2tp_core(F) ipt_ULOG(F) scsi_transport_iscsi(F) af_802154(F) rds(F) af_key(F) pppoe(F) pppox(F) ppp_generic(F) slhc(F) nfc(F) atm(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) btrfs(F) zlib_deflate(F) vfat(F) fat(F) nfs_layout_nfsv41_files(F) nfsv4(F) auth_rpcgss(F) nfsv3(F) nfs_acl(F) nfsv2(F) nfs(F) lockd(F) sunrpc(F) fscache(F) nfnetlink_log(F) nfnetlink(F) bluetooth(F) rfkill(F) arc4(F) md4(F) nls_utf8(F) cifs(F) dns_resolver(F) nf_tproxy_core(F) nls_koi8_u(F) nls_cp932(F) ts_kmp(F) sctp(F) fuse(F) sg(F) kvm_amd(F) kvm(F) amd64_edac_mod(F) edac_mce_amd(F) bnx2x(F) serio_raw(F) edac_core(F) netxen_nic(F) mdio(F) k10temp(F) microcode(F) i2c_piix4(F) ipmi_si(F) ipmi_msghandler(F) shpchp(F) hpwdt(F) hpilo(F) xfs(F) libcrc32c(F) sd_mod(F) crc_t10dif(F) radeon(F) i2c_algo_bit(F) drm_kms_helper(F) sata_svw(F) ttm(F) libata(F) drm(F) i2c_core(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F) [last unloaded: ipt_REJECT]
[ 9238.467980] CPU 3
[ 9238.468382] Pid: 27728, comm: kworker/u:0 Tainted: GF D W 3.8.4 #1 HP ProLiant BL495c G5
[ 9238.470307] RIP: 0010:[<ffffffff8118a008>] [<ffffffff8118a008>] kmem_cache_alloc+0x68/0x200
[ 9238.471832] RSP: 0018:ffff8800a326bc38 EFLAGS: 00010246
[ 9238.472807] RAX: 0000000000000000 RBX: ffff880146301970 RCX: 0000000000000000
[ 9238.474105] RDX: 000000000002c644 RSI: 00000000000080d0 RDI: ffff88014b040c00
[ 9238.475447] RBP: ffff8800a326bc88 R08: 0000000000017690 R09: ffffffff810fe6e2
[ 9238.476730] R10: ffffffffffffffff R11: ffffffffffffffe2 R12: 0000000000010000
[ 9238.478034] R13: 00000000000080d0 R14: ffff88014b040c00 R15: ffff88014b040c00
[ 9238.479330] FS: 00007f00036a3740(0000) GS:ffff88014fd80000(0000) knlGS:00000000f75546c0
[ 9238.480773] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[ 9238.481823] CR2: 0000000000010000 CR3: 0000000135563000 CR4: 00000000000007e0
[ 9238.483124] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 9238.484411] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[ 9238.485734] Process kworker/u:0 (pid: 27728, threadinfo ffff8800a326a000, task ffff8801321ee5c0)
[ 9238.487310] Stack:
[ 9238.487722] ffff8800a326bc48 ffffffff8129aea6 ffffffff810fe6e2 ffffffff8108d29f
[ 9238.489162] 0000000000800711 ffff880146301970 0000000000000000 0000000000000000
[ 9238.490800] 0000000000000000 ffff880146301970 ffff8800a326bca8 ffffffff810fe6e2
[ 9238.492409] Call Trace:
[ 9238.492985] [<ffffffff8129aea6>] ? security_prepare_creds+0x16/0x20
[ 9238.494412] [<ffffffff810fe6e2>] ? __delayacct_tsk_init+0x22/0x40
[ 9238.495854] [<ffffffff8108d29f>] ? prepare_creds+0xdf/0x190
[ 9238.497133] [<ffffffff810fe6e2>] __delayacct_tsk_init+0x22/0x40
[ 9238.498488] [<ffffffff810605df>] copy_process.part.25+0x31f/0x13f0
[ 9238.499895] [<ffffffff8107c6d0>] ? __call_usermodehelper+0xb0/0xb0
[ 9238.501307] [<ffffffff810617a9>] do_fork+0xa9/0x350
[ 9238.502432] [<ffffffff8101358e>] ? __switch_to+0x13e/0x4a0
[ 9238.503698] [<ffffffff8110ad65>] ? tracing_is_on+0x15/0x30
[ 9238.504962] [<ffffffff81061a76>] kernel_thread+0x26/0x30
[ 9238.506246] [<ffffffff8107c69c>] __call_usermodehelper+0x7c/0xb0
[ 9238.507595] [<ffffffff8107e5b4>] process_one_work+0x164/0x490
[ 9238.508912] [<ffffffff81080c3e>] worker_thread+0x15e/0x450
[ 9238.510173] [<ffffffff81080ae0>] ? busy_worker_rebind_fn+0x110/0x110
[ 9238.511605] [<ffffffff81085f80>] kthread+0xc0/0xd0
[ 9238.512719] [<ffffffff81085ec0>] ? kthread_create_on_node+0x120/0x120
[ 9238.514200] [<ffffffff8161f72c>] ret_from_fork+0x7c/0xb0
[ 9238.515423] [<ffffffff81085ec0>] ? kthread_create_on_node+0x120/0x120
[ 9238.516909] Code: 90 4d 89 fe 4d 8b 06 65 4c 03 04 25 c8 db 00 00 49 8b 50 08 4d 8b 20 4d 85 e4 0f 84 1c 01 00 00 49 63 46 20 4d 8b 06 41 f6 c0 0f <49> 8b 1c 04 0f 85 46 01 00 00 48 8d 4a 01 4c 89 e0 65 49 0f c7
[ 9238.521164] RIP [<ffffffff8118a008>] kmem_cache_alloc+0x68/0x200
[ 9238.522547] RSP <ffff8800a326bc38>
[ 9238.523367] CR2: 0000000000010000
[ 9238.524218] ---[ end trace 3567090873e2c5e1 ]---
[ 9238.525360] BUG: unable to handle kernel paging request at ffffffffffffffd8
[ 9238.526891] IP: [<ffffffff81086370>] kthread_data+0x10/0x20
[ 9238.528114] PGD 18f8067 PUD 18f9067 PMD 0
[ 9238.529096] Oops: 0000 [#5] SMP
[ 9238.529780] Modules linked in: tun(F+) cmtp(F) kernelcapi(F) hidp(F) rfcomm(F) bnep(F) l2tp_ppp(F) l2tp_netlink(F) l2tp_core(F) ipt_ULOG(F) scsi_transport_iscsi(F) af_802154(F) rds(F) af_key(F) pppoe(F) pppox(F) ppp_generic(F) slhc(F) nfc(F) atm(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) btrfs(F) zlib_deflate(F) vfat(F) fat(F) nfs_layout_nfsv41_files(F) nfsv4(F) auth_rpcgss(F) nfsv3(F) nfs_acl(F) nfsv2(F) nfs(F) lockd(F) sunrpc(F) fscache(F) nfnetlink_log(F) nfnetlink(F) bluetooth(F) rfkill(F) arc4(F) md4(F) nls_utf8(F) cifs(F) dns_resolver(F) nf_tproxy_core(F) nls_koi8_u(F) nls_cp932(F) ts_kmp(F) sctp(F) fuse(F) sg(F) kvm_amd(F) kvm(F) amd64_edac_mod(F) edac_mce_amd(F) bnx2x(F) serio_raw(F) edac_core(F) netxen_nic(F) mdio(F) k10temp(F) microcode(F) i2c_piix4(F) ipmi_si(F) ipmi_msghandler(F) shpchp(F) hpwdt(F) hpilo(F) xfs(F) libcrc32c(F) sd_mod(F) crc_t10dif(F) radeon(F) i2c_algo_bit(F) drm_kms_helper(F) sata_svw(F) ttm(F) libata(F) drm(F) i2c_core(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F) [last unloaded: ipt_REJECT]
[ 9238.549662] CPU 3
[ 9238.549997] Pid: 27728, comm: kworker/u:0 Tainted: GF D W 3.8.4 #1 HP ProLiant BL495c G5
[ 9238.551600] RIP: 0010:[<ffffffff81086370>] [<ffffffff81086370>] kthread_data+0x10/0x20
[ 9238.553155] RSP: 0018:ffff8800a326b828 EFLAGS: 00010092
[ 9238.554136] RAX: 0000000000000000 RBX: 0000000000000003 RCX: 000000000000000d
[ 9238.555426] RDX: 0000000000000004 RSI: 0000000000000003 RDI: ffff8801321ee5c0
[ 9238.556724] RBP: ffff8800a326b828 R08: ffff8801321ee630 R09: 000000000000010b
[ 9238.558025] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88014fd94180
[ 9238.559317] R13: 0000000000000003 R14: ffff8801321ee5b0 R15: ffff8801321ee5c0
[ 9238.560598] FS: 00007f00036a3740(0000) GS:ffff88014fd80000(0000) knlGS:00000000f75546c0
[ 9238.562074] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[ 9238.563162] CR2: ffffffffffffffd8 CR3: 0000000135563000 CR4: 00000000000007e0
[ 9238.564448] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 9238.565729] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[ 9238.567030] Process kworker/u:0 (pid: 27728, threadinfo ffff8800a326a000, task ffff8801321ee5c0)
[ 9238.568600] Stack:
[ 9238.569023] ffff8800a326b848 ffffffff81081775 ffff8800a326b848 ffff8801321eeb90
[ 9238.570483] ffff8800a326b8b8 ffffffff81615af2 ffff8801321ee5c0 ffff8800a326bfd8
[ 9238.572187] ffff8800a326bfd8 ffff8800a326bfd8 ffff8801321ee5c0 ffff8801321ee5c0
[ 9238.583863] Call Trace:
[ 9238.584293] [<ffffffff81081775>] wq_worker_sleeping+0x15/0xc0
[ 9238.590261] [<ffffffff81615af2>] __schedule+0x5c2/0x7b0
[ 9238.591244] [<ffffffff81616009>] schedule+0x29/0x70
[ 9238.602045] [<ffffffff810684df>] do_exit+0x6af/0x9f0
[ 9238.603146] [<ffffffff816183ae>] oops_end+0x9e/0xe0
[ 9238.604304] [<ffffffff8160acbd>] no_context+0x253/0x27e
[ 9238.614916] [<ffffffff810b4e6c>] ? ktime_get_ts+0x4c/0xf0
[ 9238.616189] [<ffffffff8160aea8>] __bad_area_nosemaphore+0x1c0/0x1df
[ 9238.627046] [<ffffffff81310811>] ? list_del+0x11/0x40
[ 9238.628185] [<ffffffff8160aeda>] bad_area_nosemaphore+0x13/0x15
[ 9238.638969] [<ffffffff8161b04e>] __do_page_fault+0x38e/0x4d0
[ 9238.644921] [<ffffffff8119cef1>] ? mem_cgroup_bad_page_check+0x21/0x30
[ 9238.651155] [<ffffffff812fbe59>] ? cpumask_next_and+0x29/0x50
[ 9238.662141] [<ffffffff8161b19e>] do_page_fault+0xe/0x10
[ 9238.663318] [<ffffffff816177d8>] page_fault+0x28/0x30
[ 9238.668961] [<ffffffff810fe6e2>] ? __delayacct_tsk_init+0x22/0x40
[ 9238.680192] [<ffffffff8118a008>] ? kmem_cache_alloc+0x68/0x200
[ 9238.691111] [<ffffffff81189fd5>] ? kmem_cache_alloc+0x35/0x200
[ 9238.692393] [<ffffffff8129aea6>] ? security_prepare_creds+0x16/0x20
[ 9238.703313] [<ffffffff810fe6e2>] ? __delayacct_tsk_init+0x22/0x40
[ 9238.704694] [<ffffffff8108d29f>] ? prepare_creds+0xdf/0x190
[ 9238.715769] [<ffffffff810fe6e2>] __delayacct_tsk_init+0x22/0x40
[ 9238.716857] [<ffffffff810605df>] copy_process.part.25+0x31f/0x13f0
[ 9238.727647] [<ffffffff8107c6d0>] ? __call_usermodehelper+0xb0/0xb0
[ 9238.729076] [<ffffffff810617a9>] do_fork+0xa9/0x350
[ 9238.739650] [<ffffffff8101358e>] ? __switch_to+0x13e/0x4a0
[ 9238.740659] [<ffffffff8110ad65>] ? tracing_is_on+0x15/0x30
[ 9238.751661] [<ffffffff81061a76>] kernel_thread+0x26/0x30
[ 9238.752639] [<ffffffff8107c69c>] __call_usermodehelper+0x7c/0xb0
[ 9238.763691] [<ffffffff8107e5b4>] process_one_work+0x164/0x490
[ 9238.764993] [<ffffffff81080c3e>] worker_thread+0x15e/0x450
[ 9238.775749] [<ffffffff81080ae0>] ? busy_worker_rebind_fn+0x110/0x110
[ 9238.786726] [<ffffffff81085f80>] kthread+0xc0/0xd0
[ 9238.787811] [<ffffffff81085ec0>] ? kthread_create_on_node+0x120/0x120
[ 9238.798777] [<ffffffff8161f72c>] ret_from_fork+0x7c/0xb0
[ 9238.799887] [<ffffffff81085ec0>] ? kthread_create_on_node+0x120/0x120
[ 9238.810874] Code: 00 48 89 e5 5d 48 8b 40 c8 48 c1 e8 02 83 e0 01 c3 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 48 8b 87 78 05 00 00 55 48 89 e5 <48> 8b 40 d8 5d c3 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90
[ 9238.824739] RIP [<ffffffff81086370>] kthread_data+0x10/0x20
[ 9238.835318] RSP <ffff8800a326b828>
[ 9238.836050] CR2: ffffffffffffffd8
[ 9238.836704] ---[ end trace 3567090873e2c5e2 ]---
[ 9238.847352] Fixing recursive fault but reboot is needed!
[ 9251.888205] bnx2x 0000:03:00.0 ksdev0: MDC/MDIO access timeout
[ 9251.911020] bnx2x 0000:03:00.0 ksdev0: NIC Link is Down
[ 9270.036495] Kernel panic - not syncing: Watchdog detected hard LOCKUP on cpu 0
[ 9271.140360] Shutting down cpus with NMI
[ 9271.141082] drm_kms_helper: panic occurred, switching back to text console
> trying to reproduce. Used CONFIG_SLUB=y.
> CAI Qian
> >
> > > [11297.598022] PGD 7b9eb067 PUD 0
> > > [11297.598022] Oops: 0000 [#2] SMP
> > > [11297.598022] Modules linked in: cmtp kernelcapi bnep
> > > scsi_transport_iscsi rfcomm l2tp_ppp l2tp_netlink l2tp_core hidp
> > > ipt_ULOG af_key nfc rds pppoe pppox ppp_generic slhc af_802154
> > > atm
> > > ip6table_filter ip6_tables iptable_filter ip_tables btrfs
> > > zlib_deflate vfat fat nfs_layout_nfsv41_files nfsv4 auth_rpcgss
> > > nfsv3 nfs_acl nfsv2 nfs lockd sunrpc fscache nfnetlink_log
> > > nfnetlink bluetooth rfkill arc4 md4 nls_utf8 cifs dns_resolver
> > > nf_tproxy_core nls_koi8_u nls_cp932 ts_kmp sctp sg kvm_amd kvm
> > > virtio_balloon i2c_piix4 pcspkr xfs libcrc32c ata_generic
> > > pata_acpi cirrus drm_kms_helper ttm ata_piix virtio_net drm
> > > libata
> > > virtio_blk i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod
> > > [last unloaded: ipt_REJECT]
> > > [11297.598022] CPU 1
> > > [11297.598022] Pid: 14134, comm: ltp-pan Tainted: G D
> > > 3.8.4+ #1 Bochs Bochs
> > > [11297.598022] RIP: 0010:[] [] kmem_cache_alloc+0x68/0x1e0
> > > [11297.598022] RSP: 0018:ffff8800447dbdd0 EFLAGS: 00010246
> > > [11297.598022] RAX: 0000000000000000 RBX: ffff88007c169970 RCX:
> > > 00000000018acdcd
> > > [11297.598022] RDX: 000000000006c104 RSI: 00000000000080d0 RDI:
> > > ffff88007d04ac00
> > > [11297.598022] RBP: ffff8800447dbe10 R08: 0000000000017620 R09:
> > > ffffffff810fe2e2
> > > [11297.598022] R10: 0000000000000000 R11: 0000000000000000 R12:
> > > 00000000fffffffe
> > > [11297.598022] R13: 00000000000080d0 R14: ffff88007d04ac00 R15:
> > > ffff88007d04ac00
> > > [11297.598022] FS: 00007f09c29b4740(0000)
> > > GS:ffff88007fd00000(0000) knlGS:00000000f74d86c0
> > > [11297.598022] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
> > > [11297.598022] CR2: 00000000fffffffe CR3: 0000000037213000 CR4:
> > > 00000000000006e0
> > > [11297.598022] DR0: 0000000000000000 DR1: 0000000000000000 DR2:
> > > 0000000000000000
> > > [11297.598022] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7:
> > > 0000000000000400
> > > [11297.598022] Process ltp-pan (pid: 14134, threadinfo
> > > ffff8800447da000, task ffff8800551ab2e0)
> > > [11297.598022] Stack:
> > > [11297.598022] ffffffff810fe2e2 ffffffff8108cf0f
> > > 0000000001200011
> > > ffff88007c169970
> > > [11297.598022] 0000000000000000 00007f09c29b4a10
> > > 0000000000000000
> > > ffff88007c169970
> > > [11297.598022] ffff8800447dbe30 ffffffff810fe2e2
> > > 0000000000000000
> > > 0000000001200011
> > > [11297.598022] Call Trace:
> > > [11297.598022] [] ? __delayacct_tsk_init+0x22/0x40
> > > [11297.598022] [] ? prepare_creds+0xdf/0x190
> > > [11297.598022] [] __delayacct_tsk_init+0x22/0x40
> > > [11297.598022] [] copy_process.part.25+0x31f/0x13f0
> > > [11297.598022] [] do_fork+0xa9/0x350
> > > [11297.598022] [] sys_clone+0x16/0x20
> > > [11297.598022] [] stub_clone+0x69/0x90
> > > [11297.598022] [] ? system_call_fastpath+0x16/0x1b
> > > [11297.598022] Code: 90 4d 89 fe 4d 8b 06 65 4c 03 04 25 c8 db 00
> > > 00 49 8b 50 08 4d 8b 20 4d 85 e4 0f 84 2b 01 00 00 49 63 46 20 4d
> > > 8b 06 41 f6 c0 0f <49> 8b 1c 04 0f 85 55 01 00 00 48 8d 4a 01 4c
> > > 89 e0 65 49 0f c7
> > > [11297.598022] RIP [] kmem_cache_alloc+0x68/0x1e0
> > > [11297.598022] RSP
> > > [11297.598022] CR2: 00000000fffffffe
> > > [11297.727799] ---[ end trace 037bde72f23b34d2 ]---
> > >
> > > Never saw this in mainline but only something like this wondering
> > > could be related
> > > (that kmem_cache_alloc also in the trace).
> > >
> >
> > These are unrelated.
> >
> > --
> > To unsubscribe, send a message with 'unsubscribe linux-mm' in
> > the body to [email protected]. For more info on Linux MM,
> > see: http://www.linux-mm.org/ .
> > Don't email: href=mailto:"[email protected]"> [email protected]
> >
>
Please enable CONFIG_SLUB_DEBUG_ON or run the kernel with slub_debug on
the command line to get detailed diagnostics as to what causes this.
----- Original Message -----
> From: "Christoph Lameter" <[email protected]>
> To: "CAI Qian" <[email protected]>
> Cc: "David Rientjes" <[email protected]>, "linux-mm" <[email protected]>, [email protected], "Oleg
> Nesterov" <[email protected]>
> Sent: Tuesday, March 26, 2013 2:00:16 AM
> Subject: Re: BUG at kmem_cache_alloc
>
>
> Please enable CONFIG_SLUB_DEBUG_ON or run the kernel with slub_debug
> on
> the command line to get detailed diagnostics as to what causes this.
>
Still running and will update ASAP. One thing I noticed was that trinity
threw out this error before the kernel crash.
[19380] Random reseed: 644697889
trinity(19380): Randomness reseeded to 0x266d4f21
trinity: trinity(19380) Randomness reseeded to 0x266d4f21
[19380] Random reseed: 1927643389
trinity(19380): Randomness reseeded to 0x72e580fd
trinity: trinity(19380) Randomness reseeded to 0x72e580fd
[watchdog] 9381710 iterations. [F:8140812 S:1240290]
[watchdog] 9383499 iterations. [F:8142333 S:1240558]
Session terminated, killing shell...
BUG!:
CHILD (pid:28825) GOT REPARENTED! parent pid:19380. Watchdog pid:19379
BUG!:
Last syscalls:
[0] pid:28515 call:settimeofday callno:10356
[1] pid:28822 call:setgid callno:322
[2] pid:28581 call:init_module callno:3622
[3] pid:28825 call:readlinkat callno:403
child 28581 exiting
child 28515 exiting
...killed.
Then, some tests in LTP called epoll triggered it eventually.
[ 9788.955733] BUG: unable to handle kernel paging request at 00000000fffffff7
[ 9788.956687] IP: [<ffffffff811876a8>] kmem_cache_alloc+0x68/0x1e0
[ 9788.956687] PGD bebd3067 PUD 0
[ 9788.956687] Oops: 0000 [#1] SMP
[ 9788.956687] Modules linked in: l2tp_ppp l2tp_netlink l2tp_core tun cmtp kernelcapi bnep fuse rfcomm hidp ipt_ULOG rds af_key pppoe pppox ppp_generic slhc af_802154 nfc atm ip6table_filter ip6_tables iptable_filter ip_tables btrfs zlib_deflate vfat fat nfs_layout_nfsv41_files nfsv4 auth_rpcgss nfsv3 nfs_acl nfsv2 nfs lockd sunrpc fscache nfnetlink_log nfnetlink bluetooth rfkill arc4 md4 nls_utf8 cifs dns_resolver nf_tproxy_core nls_koi8_u nls_cp932 ts_kmp sctp sg i5000_edac coretemp edac_core kvm_intel iTCO_wdt iTCO_vendor_support kvm lpc_ich ipmi_si ipmi_msghandler i5k_amb mfd_core hpilo hpwdt shpchp serio_raw microcode pcspkr xfs sd_mod crc_t10dif sr_mod cdrom ata_generic hpsa pata_acpi radeon i2c_algo_bit drm_kms_helper ttm drm ata_piix libata i2c_core bnx2 bnx2x cciss 3w_9xxx libcrc32c dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libiscsi_tcp qla4xxx libiscsi scsi_transport_iscsi iscsi_ibft iscsi_boot_sysfs [last unloaded: ipt_REJECT]
[ 9788.956687] CPU 0
[ 9788.956687] Pid: 25412, comm: epoll-ltp Tainted: G W I 3.8.4+ #1 HP ProLiant DL380 G5
[ 9788.956687] RIP: 0010:[<ffffffff811876a8>] [<ffffffff811876a8>] kmem_cache_alloc+0x68/0x1e0
[ 9788.956687] RSP: 0018:ffff8800bbee9dd0 EFLAGS: 00010246
[ 9788.956687] RAX: 0000000000000000 RBX: ffff8801a356e5c0 RCX: 0000000000000000
[ 9788.956687] RDX: 0000000000036b0a RSI: 00000000000080d0 RDI: ffff8801a906ad00
[ 9788.956687] RBP: ffff8800bbee9e10 R08: 00000000000176b0 R09: ffffffff810fe2e2
[ 9788.956687] R10: 0000000000000016 R11: ffffffffffffffdc R12: 00000000fffffff7
[ 9788.956687] R13: 00000000000080d0 R14: ffff8801a906ad00 R15: ffff8801a906ad00
[ 9788.956687] FS: 00007fbf42058740(0000) GS:ffff8801afc00000(0000) knlGS:0000000000000000
[ 9788.956687] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[ 9788.956687] CR2: 00000000fffffff7 CR3: 00000000aece6000 CR4: 00000000000407f0
[ 9788.956687] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 9788.956687] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[ 9788.956687] Process epoll-ltp (pid: 25412, threadinfo ffff8800bbee8000, task ffff8801a6524c50)
[ 9788.956687] Stack:
[ 9788.956687] ffffffff810fe2e2 ffffffff8108cf0f 0000000001200011 ffff8801a356e5c0
[ 9788.956687] 0000000000000000 00007fbf42058a10 0000000000000000 ffff8801a356e5c0
[ 9788.956687] ffff8800bbee9e30 ffffffff810fe2e2 0000000000000000 0000000001200011
[ 9788.956687] Call Trace:
[ 9788.956687] [<ffffffff810fe2e2>] ? __delayacct_tsk_init+0x22/0x40
[ 9788.956687] [<ffffffff8108cf0f>] ? prepare_creds+0xdf/0x190
[ 9788.956687] [<ffffffff810fe2e2>] __delayacct_tsk_init+0x22/0x40
[ 9788.956687] [<ffffffff8106027f>] copy_process.part.25+0x31f/0x13f0
[ 9788.956687] [<ffffffff8106765b>] ? do_wait+0x12b/0x250
[ 9788.956687] [<ffffffff81097f3e>] ? wake_up_new_task+0xfe/0x160
[ 9788.956687] [<ffffffff81061449>] do_fork+0xa9/0x350
[ 9788.956687] [<ffffffff81068810>] ? sys_wait4+0x80/0xf0
[ 9788.956687] [<ffffffff81061776>] sys_clone+0x16/0x20
[ 9788.956687] [<ffffffff8161a7f9>] stub_clone+0x69/0x90
[ 9788.956687] [<ffffffff8161a499>] ? system_call_fastpath+0x16/0x1b
[ 9788.956687] Code: 90 4d 89 fe 4d 8b 06 65 4c 03 04 25 c8 db 00 00 49 8b 50 08 4d 8b 20 4d 85 e4 0f 84 2b 01 00 00 49 63 46 20 4d 8b 06 41 f6 c0 0f <49> 8b 1c 04 0f 85 55 01 00 00 48 8d 4a 01 4c 89 e0 65 49 0f c7
[ 9788.956687] RIP [<ffffffff811876a8>] kmem_cache_alloc+0x68/0x1e0
[ 9788.956687] RSP <ffff8800bbee9dd0>
[ 9788.956687] CR2: 00000000fffffff7
[ 9789.029177] ---[ end trace 001669df502cd1ce ]---
CAI Qian
On Tue, Mar 26, 2013 at 05:32:27AM -0400, CAI Qian wrote:
> Still running and will update ASAP. One thing I noticed was that trinity
> threw out this error before the kernel crash.
>
> BUG!:
> CHILD (pid:28825) GOT REPARENTED! parent pid:19380. Watchdog pid:19379
>
> BUG!:
> Last syscalls:
> [0] pid:28515 call:settimeofday callno:10356
> [1] pid:28822 call:setgid callno:322
> [2] pid:28581 call:init_module callno:3622
> [3] pid:28825 call:readlinkat callno:403
> child 28581 exiting
> child 28515 exiting
> ...killed.
When this happens, it usually means that the parent segfaulted.
I've been trying to reproduce a few reports of this for a while
without success. If you get time, running trinity inside gdb should
be enough to get a useful backtrace.
(Or run with -D, and collect coredumps [there will a lot], and match the
core to the pid of the process we're interested in)
Dave
----- Original Message -----
> From: "Dave Jones" <[email protected]>
> To: "CAI Qian" <[email protected]>
> Cc: "Christoph Lameter" <[email protected]>, "David Rientjes" <[email protected]>, "linux-mm" <[email protected]>,
> [email protected], "Oleg Nesterov" <[email protected]>
> Sent: Wednesday, March 27, 2013 3:53:44 AM
> Subject: Re: BUG at kmem_cache_alloc
>
> On Tue, Mar 26, 2013 at 05:32:27AM -0400, CAI Qian wrote:
>
> > Still running and will update ASAP. One thing I noticed was that
> > trinity
> > threw out this error before the kernel crash.
> >
> > BUG!:
> > CHILD (pid:28825) GOT REPARENTED! parent pid:19380. Watchdog
> > pid:19379
> >
> > BUG!:
> > Last syscalls:
> > [0] pid:28515 call:settimeofday callno:10356
> > [1] pid:28822 call:setgid callno:322
> > [2] pid:28581 call:init_module callno:3622
> > [3] pid:28825 call:readlinkat callno:403
> > child 28581 exiting
> > child 28515 exiting
> > ...killed.
>
> When this happens, it usually means that the parent segfaulted.
> I've been trying to reproduce a few reports of this for a while
> without success. If you get time, running trinity inside gdb should
> be enough to get a useful backtrace.
>
> (Or run with -D, and collect coredumps [there will a lot], and match
> the
> core to the pid of the process we're interested in)
>
> Dave
>
While reproducing this, it triggered something else with SLUB_DEBUG_ON.
CAI Qian
[87295.499233] general protection fault: 0000 [#1] SMP
[87295.500228] Modules linked in: binfmt_misc fuse tun cmtp kernelcapi rfcomm bnep hidp scsi_transport_iscsi nfnetlink ipt_ULOG nfc bluetooth rfkill af_key atm lockd sunrpc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_MASQUERADE ip6table_mangle ip6t_REJECT nf_conntrack_ipv6 nf_defrag_ipv6 iptable_nat nf_nat_ipv4 nf_nat iptable_mangle ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter ip_tables sg kvm_amd kvm microcode amd64_edac_mod edac_mce_amd pcspkr serio_raw edac_core k10temp bnx2x netxen_nic mdio i2c_piix4 i2c_core hpilo shpchp ipmi_si ipmi_msghandler hpwdt xfs libcrc32c sd_mod crc_t10dif sata_svw libata dm_mirror dm_region_hash dm_log dm_mod
[87295.515752] CPU 1
[87295.516184] Pid: 23211, comm: trinity-main Tainted: G W 3.8.4 #4 HP ProLiant BL495c G5
[87295.517810] RIP: 0010:[<ffffffff812e0b43>] [<ffffffff812e0b43>] rb_next+0x23/0x50
[87295.519254] RSP: 0018:ffff880127f5de58 EFLAGS: 00010202
[87295.520398] RAX: 6b6b6b6b6b6b6b6b RBX: 0000000000000000 RCX: ffff88014181d9c8
[87295.521996] RDX: 6b6b6b6b6b6b6b6b RSI: ffff88014181a6e0 RDI: ffff88014181d9e0
[87295.523606] RBP: ffff880127f5de58 R08: 0000000000003d7b R09: 0000000000000008
[87295.525201] R10: ffffffff81197360 R11: 0000000000000246 R12: ffff8801314f3180
[87295.526793] R13: 0000000000000000 R14: 000000000000000f R15: ffff88014181d9c8
[87295.528465] FS: 00007f94bbc0f740(0000) GS:ffff88014fc80000(0000) knlGS:0000000000000000
[87295.530271] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[87295.531578] CR2: 0000000001f53008 CR3: 00000001129f5000 CR4: 00000000000007e0
[87295.533210] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[87295.534797] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[87295.536402] Process trinity-main (pid: 23211, threadinfo ffff880127f5c000, task ffff8801418e98a0)
[87295.538368] Stack:
[87295.538793] ffff880127f5ded8 ffffffff811f8220 0000000000000008 0000000000003d7b
[87295.540579] ffff880127f50001 ffff8801314f3190 0000000000020000 ffffffff81197360
[87295.542313] ffff880127f5df40 ffff88014181a6e0 ffff880127f5ded8 ffff8801314f3180
[87295.543959] Call Trace:
[87295.544513] [<ffffffff811f8220>] sysfs_readdir+0x150/0x280
[87295.545774] [<ffffffff81197360>] ? fillonedir+0x100/0x100
[87295.547004] [<ffffffff81197360>] ? fillonedir+0x100/0x100
[87295.548268] [<ffffffff81197238>] vfs_readdir+0xb8/0xe0
[87295.549446] [<ffffffff811a159b>] ? set_close_on_exec+0x3b/0x70
[87295.550832] [<ffffffff8119758f>] sys_getdents+0x8f/0x110
[87295.552068] [<ffffffff815e6419>] system_call_fastpath+0x16/0x1b
[87295.553433] Code: 48 89 70 10 eb a9 66 90 55 48 8b 17 48 89 e5 48 39 d7 74 3b 48 8b 47 08 48 85 c0 75 0e eb 1f 66 0f 1f 84 00 00 00 00 00 48 89 d0 <48> 8b 50 10 48 85 d2 75 f4 5d c3 66 90 48 8b 10 48 89 c7 48 89
[87295.557829] RIP [<ffffffff812e0b43>] rb_next+0x23/0x50
[87295.558960] RSP <ffff880127f5de58>
[87295.560213] ---[ end trace d5f25cc963b1f1d9 ]---
[watchdog] Triggering periodic reseed.
On Thu, Mar 28, 2013 at 03:49:37AM -0400, CAI Qian wrote:
> While reproducing this, it triggered something else with SLUB_DEBUG_ON.
> CAI Qian
>
> [87295.499233] general protection fault: 0000 [#1] SMP
> [87295.500228] Modules linked in: binfmt_misc fuse tun cmtp kernelcapi rfcomm bnep hidp scsi_transport_iscsi nfnetlink ipt_ULOG nfc bluetooth rfkill af_key atm lockd sunrpc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_MASQUERADE ip6table_mangle ip6t_REJECT nf_conntrack_ipv6 nf_defrag_ipv6 iptable_nat nf_nat_ipv4 nf_nat iptable_mangle ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter ip_tables sg kvm_amd kvm microcode amd64_edac_mod edac_mce_amd pcspkr serio_raw edac_core k10temp bnx2x netxen_nic mdio i2c_piix4 i2c_core hpilo shpchp ipmi_si ipmi_msghandler hpwdt xfs libcrc32c sd_mod crc_t10dif sata_svw libata dm_mirror dm_region_hash dm_log dm_mod
> [87295.515752] CPU 1
> [87295.516184] Pid: 23211, comm: trinity-main Tainted: G W 3.8.4 #4 HP ProLiant BL495c G5
> [87295.517810] RIP: 0010:[<ffffffff812e0b43>] [<ffffffff812e0b43>] rb_next+0x23/0x50
> [87295.519254] RSP: 0018:ffff880127f5de58 EFLAGS: 00010202
> [87295.520398] RAX: 6b6b6b6b6b6b6b6b RBX: 0000000000000000 RCX: ffff88014181d9c8
> [87295.521996] RDX: 6b6b6b6b6b6b6b6b RSI: ffff88014181a6e0 RDI: ffff88014181d9e0
> [87295.523606] RBP: ffff880127f5de58 R08: 0000000000003d7b R09: 0000000000000008
> [87295.525201] R10: ffffffff81197360 R11: 0000000000000246 R12: ffff8801314f3180
> [87295.526793] R13: 0000000000000000 R14: 000000000000000f R15: ffff88014181d9c8
> [87295.528465] FS: 00007f94bbc0f740(0000) GS:ffff88014fc80000(0000) knlGS:0000000000000000
> [87295.530271] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
> [87295.531578] CR2: 0000000001f53008 CR3: 00000001129f5000 CR4: 00000000000007e0
> [87295.533210] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
> [87295.534797] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
> [87295.536402] Process trinity-main (pid: 23211, threadinfo ffff880127f5c000, task ffff8801418e98a0)
> [87295.538368] Stack:
> [87295.538793] ffff880127f5ded8 ffffffff811f8220 0000000000000008 0000000000003d7b
> [87295.540579] ffff880127f50001 ffff8801314f3190 0000000000020000 ffffffff81197360
> [87295.542313] ffff880127f5df40 ffff88014181a6e0 ffff880127f5ded8 ffff8801314f3180
> [87295.543959] Call Trace:
> [87295.544513] [<ffffffff811f8220>] sysfs_readdir+0x150/0x280
> [87295.545774] [<ffffffff81197360>] ? fillonedir+0x100/0x100
> [87295.547004] [<ffffffff81197360>] ? fillonedir+0x100/0x100
> [87295.548268] [<ffffffff81197238>] vfs_readdir+0xb8/0xe0
> [87295.549446] [<ffffffff811a159b>] ? set_close_on_exec+0x3b/0x70
> [87295.550832] [<ffffffff8119758f>] sys_getdents+0x8f/0x110
> [87295.552068] [<ffffffff815e6419>] system_call_fastpath+0x16/0x1b
> [87295.553433] Code: 48 89 70 10 eb a9 66 90 55 48 8b 17 48 89 e5 48 39 d7 74 3b 48 8b 47 08 48 85 c0 75 0e eb 1f 66 0f 1f 84 00 00 00 00 00 48 89 d0 <48> 8b 50 10 48 85 d2 75 f4 5d c3 66 90 48 8b 10 48 89 c7 48 89
> [87295.557829] RIP [<ffffffff812e0b43>] rb_next+0x23/0x50
> [87295.558960] RSP <ffff880127f5de58>
> [87295.560213] ---[ end trace d5f25cc963b1f1d9 ]---
> [watchdog] Triggering periodic reseed.
That's fixed by the patch below from Ming Lei.
diff --git a/fs/sysfs/dir.c b/fs/sysfs/dir.c
index 2fbdff6..014ed97 100644
--- a/fs/sysfs/dir.c
+++ b/fs/sysfs/dir.c
@@ -280,6 +280,11 @@ void release_sysfs_dirent(struct sysfs_dirent * sd)
* sd->s_parent won't change beneath us.
*/
parent_sd = sd->s_parent;
+ if(!(sd->s_flags & SYSFS_FLAG_REMOVED)) {
+ printk("%s-%d sysfs_dirent use after free: %s-%s\n",
+ __func__, __LINE__, parent_sd->s_name, sd->s_name);
+ dump_stack();
+ }
if (sysfs_type(sd) == SYSFS_KOBJ_LINK)
sysfs_put(sd->s_symlink.target_sd);
@@ -962,6 +967,12 @@ static struct sysfs_dirent *sysfs_dir_pos(const void *ns,
int valid = !(pos->s_flags & SYSFS_FLAG_REMOVED) &&
pos->s_parent == parent_sd &&
hash == pos->s_hash;
+
+ if ((atomic_read(&pos->s_count) == 1)) {
+ printk("%s-%d sysfs_dirent use after free: %s(%s)-%s, %lld-%u\n",
+ __func__, __LINE__, parent_sd->s_name, pos->s_parent->s_name,
+ pos->s_name, hash, pos->s_hash);
+ }
sysfs_put(pos);
if (!valid)
pos = NULL;
@@ -1020,6 +1031,8 @@ static int sysfs_readdir(struct file * filp, void * dirent, filldir_t filldir)
ino = parent_sd->s_ino;
if (filldir(dirent, ".", 1, filp->f_pos, ino, DT_DIR) == 0)
filp->f_pos++;
+ else
+ return 0;
}
if (filp->f_pos == 1) {
if (parent_sd->s_parent)
@@ -1028,6 +1041,8 @@ static int sysfs_readdir(struct file * filp, void * dirent, filldir_t filldir)
ino = parent_sd->s_ino;
if (filldir(dirent, "..", 2, filp->f_pos, ino, DT_DIR) == 0)
filp->f_pos++;
+ else
+ return 0;
}
mutex_lock(&sysfs_mutex);
for (pos = sysfs_dir_pos(ns, parent_sd, filp->f_pos, pos);
@@ -1058,10 +1073,21 @@ static int sysfs_readdir(struct file * filp, void * dirent, filldir_t filldir)
return 0;
}
+static loff_t sysfs_dir_llseek(struct file *file, loff_t offset, int whence)
+{
+ struct inode *inode = file_inode(file);
+ loff_t ret;
+
+ mutex_lock(&inode->i_mutex);
+ ret = generic_file_llseek(file, offset, whence);
+ mutex_unlock(&inode->i_mutex);
+
+ return ret;
+}
const struct file_operations sysfs_dir_operations = {
.read = generic_read_dir,
.readdir = sysfs_readdir,
.release = sysfs_dir_release,
- .llseek = generic_file_llseek,
+ .llseek = sysfs_dir_llseek,
};