OK, so the minimal workload to trigger this I found so far was to
run trinity, ltp and then xfstests. I have been able to easily
reproduced on 3 servers so far, and I'll post full logs here for
LKML and linux-mm as this may unrelated to XFS only. As far as
I can tell from the previous testing results, this has never been
reproduced back in 3.9 GA time. This seems also been reproduced
on 3.10-rc3 mostly on s390x so far.
CAI Qian
1)
[10948.104344] XFS (dm-2): Mounting Filesystem
[10948.257335] XFS (dm-2): Ending clean mount
[10949.753951] XFS (dm-0): Mounting Filesystem
[10950.012824] XFS (dm-0): Ending clean mount
[10951.115722] BUG: unable to handle kernel paging request at ffff88003a9ca478
[10951.122727] IP: [<ffffffff810f7f47>] __call_rcu.constprop.47+0x77/0x230
[10951.129372] PGD 1ddf067 PUD 1de0067 PMD 3da02063 PTE 889b2731bac1a751
[10951.135887] Oops: 000b [#1] SMP
[10951.139157] Modules linked in: binfmt_misc(F) fuse(F) snd_seq_dummy(F) rfcomm(F) hidp(F) l2tp_ppp(F) l2tp_netlink(F) l2tp_core(F) cmtp(F) kernelcapi(F) tun(F) bnep(F) ipt_ULOG(F) scsi_transport_iscsi(F) nfnetlink(F) rds(F) bluetooth(F) af_key(F) pppoe(F) pppox(F) ppp_generic(F) slhc(F) atm(F) nfc(F) af_802154(F) lockd(F) sunrpc(F) nf_conntrack_netbios_ns(F) nf_conntrack_broadcast(F) ipt_MASQUERADE(F) ip6table_nat(F) nf_nat_ipv6(F) ip6table_mangle(F) ip6t_REJECT(F) nf_conntrack_ipv6(F) nf_defrag_ipv6(F) iptable_nat(F) nf_nat_ipv4(F) nf_nat(F) iptable_mangle(F) ipt_REJECT(F) nf_conntrack_ipv4(F) nf_defrag_ipv4(F) xt_conntrack(F) nf_conntrack(F) ebtable_filter(F) ebtables(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) sg(F) snd_hda_codec_hdmi(F) snd_hda_codec_realtek(F) snd_hda_intel(F) snd_hda_codec(F) snd_hwdep(F) snd_seq(F) snd_seq_device(F) snd_pcm(F) hp_wmi(F) sparse_keymap(F) e1000e(F) rfkill(F) iTCO_wdt(F) snd_page_alloc(F) snd_timer(F) snd(F) iTCO_vendor_support(F) soundcore(F) mei(F) ptp(F) i2c_i801(F) pps_core(F) lpc_ich(F) pcspkr(F) mfd_core(F) microcode(F) xfs(F) libcrc32c(F) sr_mod(F) sd_mod(F) cdrom(F) crc_t10dif(F) nouveau(F) video(F) mxm_wmi(F) i2c_algo_bit(F) drm_kms_helper(F) ata_generic(F) ttm(F) ahci(F) libahci(F) pata_acpi(F) drm(F) libata(F) i2c_core(F) wmi(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F)
[10951.260278] CPU 1
[10951.262122] Pid: 0, comm: swapper/1 Tainted: GF 3.9.4 #1 Hewlett-Packard HP Z210 Workstation/1587h
[10951.272238] RIP: 0010:[<ffffffff810f7f47>] [<ffffffff810f7f47>] __call_rcu.constprop.47+0x77/0x230
[10951.281307] RSP: 0018:ffff88007d083e00 EFLAGS: 00010046
[10951.286620] RAX: ffff88003a9ca478 RBX: ffff88007d08ec80 RCX: 0000000000000000
[10951.293763] RDX: ffffffff81947c80 RSI: ffffffff8108a350 RDI: ffff88005dbcd890
[10951.300909] RBP: ffff88007d083e28 R08: 0000000000000000 R09: 0000000000000000
[10951.308051] R10: ffff88007cc2c480 R11: ffffea000106ab00 R12: ffff88005dbcd890
[10951.315187] R13: 0000000000000246 R14: ffff880041aad520 R15: ffff88007d08ec80
[10951.322324] FS: 0000000000000000(0000) GS:ffff88007d080000(0000) knlGS:0000000000000000
[10951.330422] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[10951.336176] CR2: ffff88003a9ca478 CR3: 00000000018fa000 CR4: 00000000000407e0
[10951.343319] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[10951.350463] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[10951.357604] Process swapper/1 (pid: 0, threadinfo ffff880079990000, task ffff88007996b580)
[10951.365872] Stack:
[10951.367885] ffff88007996b580 000000010012000b ffff880079955040 ffff880079955040
[10951.375347] ffff8800799559f8 ffff88007d083e38 ffffffff810f8157 ffff88007d083e48
[10951.382812] ffffffff8108a4af ffff88007d083e60 ffffffff8108a958 ffff880079955040
[10951.390276] Call Trace:
[10951.392723] <IRQ>
[10951.394658] [<ffffffff810f8157>] call_rcu_sched+0x17/0x20
[10951.400353] [<ffffffff8108a4af>] __put_cred+0x3f/0x50
[10951.405499] [<ffffffff8108a958>] exit_creds+0x58/0x60
[10951.410647] [<ffffffff8105e0ca>] __put_task_struct+0x4a/0x140
[10951.416485] [<ffffffff8106519c>] delayed_put_task_struct+0x3c/0x80
[10951.422759] [<ffffffff810f833d>] rcu_process_callbacks+0x1dd/0x550
[10951.429035] [<ffffffff8106991f>] __do_softirq+0xef/0x230
[10951.434442] [<ffffffff814b8800>] ? cpufreq_p4_cpu_init+0x290/0x290
[10951.440718] [<ffffffff81069be5>] irq_exit+0xa5/0xb0
[10951.445691] [<ffffffff8161b0ee>] smp_apic_timer_interrupt+0x6e/0x9c
[10951.452055] [<ffffffff81619fdd>] apic_timer_interrupt+0x6d/0x80
[10951.458065] <EOI>
[10951.459995] [<ffffffff814b9181>] ? cpuidle_wrap_enter+0x41/0x80
[10951.466218] [<ffffffff814b91d0>] cpuidle_enter_tk+0x10/0x20
[10951.471885] [<ffffffff814b8f32>] cpuidle_idle_call+0xb2/0x1e0
[10951.477730] [<ffffffff8101c015>] cpu_idle+0xe5/0x140
[10951.482790] [<ffffffff815f9a9e>] start_secondary+0x24f/0x251
[10951.488542] Code: c8 db 00 00 48 83 7b 48 00 0f 84 f1 00 00 00 48 8b 43 78 48 83 c0 01 84 c9 48 89 43 78 74 05 48 83 43 70 01 0f ae f0 48 8b 43 48 <4c> 89 20 4c 89 63 48 48 c7 c1 20 eb 00 00 65 48 03 0c 25 c8 db
[10951.508531] RIP [<ffffffff810f7f47>] __call_rcu.constprop.47+0x77/0x230
[10951.515258] RSP <ffff88007d083e00>
[10951.518748] CR2: ffff88003a9ca478
[10951.527448] ---[ end trace 1dba1edcdfef3999 ]---
[10951.527453] BUG: unable to handle kernel paging request at ffff88003db52000
[10951.527459] IP: [<ffffffff81302301>] memmove+0x51/0x1a0
[10951.527462] PGD 1ddf067 PUD 1de0067 PMD 626ab063 PTE 800000003db52161
[10951.527464] Oops: 0003 [#2] SMP
[10951.527487] Modules linked in: binfmt_misc(F) fuse(F) snd_seq_dummy(F) rfcomm(F) hidp(F) l2tp_ppp(F) l2tp_netlink(F) l2tp_core(F) cmtp(F) kernelcapi(F) tun(F) bnep(F) ipt_ULOG(F) scsi_transport_iscsi(F) nfnetlink(F) rds(F) bluetooth(F) af_key(F) pppoe(F) pppox(F) ppp_generic(F) slhc(F) atm(F) nfc(F) af_802154(F) lockd(F) sunrpc(F) nf_conntrack_netbios_ns(F) nf_conntrack_broadcast(F) ipt_MASQUERADE(F) ip6table_nat(F) nf_nat_ipv6(F) ip6table_mangle(F) ip6t_REJECT(F) nf_conntrack_ipv6(F) nf_defrag_ipv6(F) iptable_nat(F) nf_nat_ipv4(F) nf_nat(F) iptable_mangle(F) ipt_REJECT(F) nf_conntrack_ipv4(F) nf_defrag_ipv4(F) xt_conntrack(F) nf_conntrack(F) ebtable_filter(F) ebtables(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) sg(F) snd_hda_codec_hdmi(F) snd_hda_codec_realtek(F) snd_hda_intel(F) snd_hda_codec(F) snd_hwdep(F) snd_seq(F) snd_seq_device(F) snd_pcm(F) hp_wmi(F) sparse_keymap(F) e1000e(F) rfkill(F) iTCO_wdt(F) snd_page_alloc(F) snd_timer(F) snd(F) iTCO_vendor_support(F) soundcore(F) mei(F) ptp(F) i2c_i801(F) pps_core(F) lpc_ich(F) pcspkr(F) mfd_core(F) microcode(F) xfs(F) libcrc32c(F) sr_mod(F) sd_mod(F) cdrom(F) crc_t10dif(F) nouveau(F) video(F) mxm_wmi(F) i2c_algo_bit(F) drm_kms_helper(F) ata_generic(F) ttm(F) ahci(F) libahci(F) pata_acpi(F) drm(F) libata(F) i2c_core(F) wmi(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F)
[10951.527508] CPU 0
[10951.527509] Pid: 8341, comm: attr Tainted: GF D 3.9.4 #1 Hewlett-Packard HP Z210 Workstation/1587h
[10951.527513] RIP: 0010:[<ffffffff81302301>] [<ffffffff81302301>] memmove+0x51/0x1a0
[10951.527514] RSP: 0018:ffff88003106fad8 EFLAGS: 00010286
[10951.527515] RAX: ffff88003d95a2b0 RBX: ffff880042652000 RCX: 000000000000d00b
[10951.527516] RDX: ffffffffffe08278 RSI: ffff88003db523e8 RDI: ffff88003db51ff0
[10951.527517] RBP: ffff88003106fb30 R08: 15e8a066d060c7c7 R09: 481974030000012c
[10951.527518] R10: b883660a74c08548 R11: a066db00dd048b48 R12: ffff88003d95a290
[10951.527519] R13: 00000000000003d8 R14: 0000000000001800 R15: 000000000000007b
[10951.527521] FS: 00007feda92ea740(0000) GS:ffff88007d000000(0000) knlGS:0000000000000000
[10951.527522] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[10951.527523] CR2: ffff88003db52000 CR3: 000000005ab9f000 CR4: 00000000000407f0
[10951.527524] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[10951.527525] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[10951.527527] Process attr (pid: 8341, threadinfo ffff88003106e000, task ffff88003a9c8000)
[10951.527527] Stack:
[10951.527530] ffffffffa028c1a1 0000000000000000 0000000000000000 ffff88003d95a2b0
[10951.527532] ffff88003d95a290 0000000000000018 ffff88003d95a290 0000000000000000
[10951.527534] ffff880077bd4a40 ffff880063fe63c0 ffff880042c9fb60 ffff88003106fb68
[10951.527535] Call Trace:
[10951.527563] [<ffffffffa028c1a1>] ? xfs_attr_leaf_moveents.isra.2+0x91/0x280 [xfs]
[10951.527582] [<ffffffffa028c467>] xfs_attr_leaf_compact+0xd7/0x130 [xfs]
[10951.527602] [<ffffffffa028da2e>] xfs_attr_leaf_add+0xce/0x170 [xfs]
[10951.527619] [<ffffffffa0289850>] xfs_attr_leaf_addname+0xc0/0x3d0 [xfs]
[10951.527638] [<ffffffffa0297d4e>] ? xfs_bmap_one_block+0x3e/0xa0 [xfs]
[10951.527656] [<ffffffffa028a78c>] xfs_attr_set_int+0x30c/0x420 [xfs]
[10951.527660] [<ffffffff811be9f4>] ? setxattr+0xa4/0x1c0
[10951.527678] [<ffffffffa028ad1f>] xfs_attr_set+0x7f/0x90 [xfs]
[10951.527696] [<ffffffffa0280a12>] xfs_xattr_set+0x42/0x70 [xfs]
[10951.527699] [<ffffffff811bdef2>] generic_setxattr+0x62/0x80
[10951.527702] [<ffffffff811be743>] __vfs_setxattr_noperm+0x63/0x1b0
[10951.527705] [<ffffffff811be945>] vfs_setxattr+0xb5/0xc0
[10951.527707] [<ffffffff811bea76>] setxattr+0x126/0x1c0
[10951.527711] [<ffffffff8118358d>] ? kmem_cache_free+0x1cd/0x1e0
[10951.527713] [<ffffffff811a72b2>] ? final_putname+0x22/0x50
[10951.527715] [<ffffffff811a74cb>] ? putname+0x2b/0x40
[10951.527717] [<ffffffff811ab96f>] ? user_path_at_empty+0x5f/0x90
[10951.527720] [<ffffffff8119e5c9>] ? __sb_start_write+0x49/0x100
[10951.527723] [<ffffffff811bedef>] sys_lsetxattr+0x8f/0xd0
[10951.527727] [<ffffffff81619359>] system_call_fastpath+0x16/0x1b
[10951.527746] Code: 00 00 72 05 40 38 fe 74 41 48 83 ea 20 48 83 ea 20 4c 8b 1e 4c 8b 56 08 4c 8b 4e 10 4c 8b 46 18 48 8d 76 20 4c 89 1f 4c 89 57 08 <4c> 89 4f 10 4c 89 47 18 48 8d 7f 20 73 d4 48 83 c2 20 e9 a8 00
[10951.527749] RIP [<ffffffff81302301>] memmove+0x51/0x1a0
[10951.527750] RSP <ffff88003106fad8>
[10951.527750] CR2: ffff88003db52000
[10951.527752] ---[ end trace 1dba1edcdfef399a ]---
[10951.527814] BUG: unable to handle kernel paging request at ffff88003a9d2018
[10951.527819] IP: [<ffffffff81095092>] try_to_wake_up+0x52/0x2d0
[10951.527824] PGD 1ddf067 PUD 1de0067 PMD 3da02063 PTE 2d00000000000000
[10951.527826] Oops: 0000 [#3] SMP
[10951.527847] Modules linked in: binfmt_misc(F) fuse(F) snd_seq_dummy(F) rfcomm(F) hidp(F) l2tp_ppp(F) l2tp_netlink(F) l2tp_core(F) cmtp(F) kernelcapi(F) tun(F) bnep(F) ipt_ULOG(F) scsi_transport_iscsi(F) nfnetlink(F) rds(F) bluetooth(F) af_key(F) pppoe(F) pppox(F) ppp_generic(F) slhc(F) atm(F) nfc(F) af_802154(F) lockd(F) sunrpc(F) nf_conntrack_netbios_ns(F) nf_conntrack_broadcast(F) ipt_MASQUERADE(F) ip6table_nat(F) nf_nat_ipv6(F) ip6table_mangle(F) ip6t_REJECT(F) nf_conntrack_ipv6(F) nf_defrag_ipv6(F) iptable_nat(F) nf_nat_ipv4(F) nf_nat(F) iptable_mangle(F) ipt_REJECT(F) nf_conntrack_ipv4(F) nf_defrag_ipv4(F) xt_conntrack(F) nf_conntrack(F) ebtable_filter(F) ebtables(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) sg(F) snd_hda_codec_hdmi(F) snd_hda_codec_realtek(F) snd_hda_intel(F) snd_hda_codec(F) snd_hwdep(F) snd_seq(F) snd_seq_device(F) snd_pcm(F) hp_wmi(F) sparse_keymap(F) e1000e(F) rfkill(F) iTCO_wdt(F) snd_page_alloc(F) snd_timer(F) snd(F) iTCO_vendor_support(F) soundcore(F) mei(F) ptp(F) i2c_i801(F) pps_core(F) lpc_ich(F) pcspkr(F) mfd_core(F) microcode(F) xfs(F) libcrc32c(F) sr_mod(F) sd_mod(F) cdrom(F) crc_t10dif(F) nouveau(F) video(F) mxm_wmi(F) i2c_algo_bit(F) drm_kms_helper(F) ata_generic(F) ttm(F) ahci(F) libahci(F) pata_acpi(F) drm(F) libata(F) i2c_core(F) wmi(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F)
[10951.527868] CPU 0
[10951.527868] Pid: 8341, comm: attr Tainted: GF D 3.9.4 #1 Hewlett-Packard HP Z210 Workstation/1587h
[10951.527871] RIP: 0010:[<ffffffff81095092>] [<ffffffff81095092>] try_to_wake_up+0x52/0x2d0
[10951.527873] RSP: 0018:ffff88007d003d98 EFLAGS: 00010046
[10951.527874] RAX: ffff88003a9d2000 RBX: ffff8800647f8000 RCX: 0000000000000000
[10951.527875] RDX: 000000003e463e46 RSI: 0000000000000003 RDI: ffff8800647f87ec
[10951.527876] RBP: ffff88007d003dd8 R08: ffff88007d013d10 R09: ffffffff81c75303
[10951.527877] R10: 0000000000000000 R11: 000000000000000c R12: ffff8800647f87ec
[10951.527878] R13: 0000000000000000 R14: 0000000000000046 R15: 0000000000014400
[10951.527879] FS: 00007feda92ea740(0000) GS:ffff88007d000000(0000) knlGS:0000000000000000
[10951.527880] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[10951.527881] CR2: ffff88003a9d2018 CR3: 000000005ab9f000 CR4: 00000000000407f0
[10951.527882] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[10951.527883] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[10951.527884] Process attr (pid: 8341, threadinfo ffff88003106e000, task ffff88003a9c8000)
[10951.527885] Stack:
[10951.527887] 0000008000000001 ffff88007d003db0 0000000000000003 ffff8800647f8000
[10951.527889] ffff88007d017e00 ffff88007d010f08 0000000000000000 ffff88007cc428a0
[10951.527891] ffff88007d003df0 ffffffff81095333 ffff88007d013d00 ffff88007d003e00
[10951.527892] Call Trace:
[10951.527893] <IRQ>
[10951.527896] [<ffffffff81095333>] wake_up_process+0x23/0x40
[10951.527899] [<ffffffff8107c064>] wake_up_worker+0x24/0x30
[10951.527901] [<ffffffff8107c533>] insert_work+0x43/0x50
[10951.527904] [<ffffffff8107c6e8>] __queue_work+0x118/0x270
[10951.527906] [<ffffffff8107c840>] ? __queue_work+0x270/0x270
[10951.527909] [<ffffffff8107c858>] delayed_work_timer_fn+0x18/0x20
[10951.527911] [<ffffffff8106f106>] call_timer_fn+0x36/0x110
[10951.527913] [<ffffffff8107c840>] ? __queue_work+0x270/0x270
[10951.527916] [<ffffffff81071431>] run_timer_softirq+0x151/0x290
[10951.527920] [<ffffffff8106991f>] __do_softirq+0xef/0x230
[10951.527923] [<ffffffff81069be5>] irq_exit+0xa5/0xb0
[10951.527925] [<ffffffff8161b0ee>] smp_apic_timer_interrupt+0x6e/0x9c
[10951.527928] [<ffffffff81619fdd>] apic_timer_interrupt+0x6d/0x80
[10951.527929] <EOI>
[10951.527932] [<ffffffff810c8c0f>] ? acct_collect+0x16f/0x1b0
[10951.527934] [<ffffffff81066e01>] do_exit+0x7e1/0xa20
[10951.527937] [<ffffffff81604989>] ? printk+0x67/0x69
[10951.527941] [<ffffffff81612034>] oops_end+0xa4/0xe0
[10951.527943] [<ffffffff81604212>] no_context+0x263/0x270
[10951.527945] [<ffffffff81604292>] __bad_area_nosemaphore+0x73/0x1cc
[10951.527947] [<ffffffff816043fe>] bad_area_nosemaphore+0x13/0x15
[10951.527949] [<ffffffff81614c0e>] __do_page_fault+0x31e/0x510
[10951.527967] [<ffffffffa0284497>] ? kmem_alloc+0x67/0xf0 [xfs]
[10951.527971] [<ffffffff81020c2b>] ? save_stack_trace+0x2b/0x50
[10951.527974] [<ffffffff81180911>] ? set_track+0x61/0x1b0
[10951.527976] [<ffffffff8118176d>] ? init_object+0x3d/0x70
[10951.527979] [<ffffffff81614e0e>] do_page_fault+0xe/0x10
[10951.527982] [<ffffffff81611498>] page_fault+0x28/0x30
[10951.527985] [<ffffffff81302301>] ? memmove+0x51/0x1a0
[10951.528003] [<ffffffffa028c1a1>] ? xfs_attr_leaf_moveents.isra.2+0x91/0x280 [xfs]
[10951.528020] [<ffffffffa028c467>] xfs_attr_leaf_compact+0xd7/0x130 [xfs]
[10951.528038] [<ffffffffa028da2e>] xfs_attr_leaf_add+0xce/0x170 [xfs]
[10951.528055] [<ffffffffa0289850>] xfs_attr_leaf_addname+0xc0/0x3d0 [xfs]
[10951.528073] [<ffffffffa0297d4e>] ? xfs_bmap_one_block+0x3e/0xa0 [xfs]
[10951.528090] [<ffffffffa028a78c>] xfs_attr_set_int+0x30c/0x420 [xfs]
[10951.528093] [<ffffffff811be9f4>] ? setxattr+0xa4/0x1c0
[10951.528111] [<ffffffffa028ad1f>] xfs_attr_set+0x7f/0x90 [xfs]
[10951.528128] [<ffffffffa0280a12>] xfs_xattr_set+0x42/0x70 [xfs]
[10951.528131] [<ffffffff811bdef2>] generic_setxattr+0x62/0x80
[10951.528134] [<ffffffff811be743>] __vfs_setxattr_noperm+0x63/0x1b0
[10951.528137] [<ffffffff811be945>] vfs_setxattr+0xb5/0xc0
[10951.528140] [<ffffffff811bea76>] setxattr+0x126/0x1c0
[10951.528142] [<ffffffff8118358d>] ? kmem_cache_free+0x1cd/0x1e0
[10951.528144] [<ffffffff811a72b2>] ? final_putname+0x22/0x50
[10951.528146] [<ffffffff811a74cb>] ? putname+0x2b/0x40
[10951.528148] [<ffffffff811ab96f>] ? user_path_at_empty+0x5f/0x90
[10951.528150] [<ffffffff8119e5c9>] ? __sb_start_write+0x49/0x100
[10951.528154] [<ffffffff811bedef>] sys_lsetxattr+0x8f/0xd0
[10951.528157] [<ffffffff81619359>] system_call_fastpath+0x16/0x1b
[10951.528175] Code: 31 ff 4c 89 e7 e8 3f bd 57 00 8b 75 d0 49 89 c6 48 8b 03 85 c6 0f 84 9f 01 00 00 8b 4b 2c 48 8b 43 08 49 c7 c7 00 44 01 00 85 c9 <44> 8b 48 18 74 7a 49 c7 c7 00 44 01 00 44 89 c8 4c 89 fa 48 89
[10951.528178] RIP [<ffffffff81095092>] try_to_wake_up+0x52/0x2d0
[10951.528179] RSP <ffff88007d003d98>
[10951.528180] CR2: ffff88003a9d2018
[10951.528181] ---[ end trace 1dba1edcdfef399b ]---
[10951.528182] Kernel panic - not syncing: Fatal exception in interrupt
[10952.594747] Shutting down cpus with NMI
[10952.598598] drm_kms_helper: panic occurred, switching back to text console
2)
[20914.575680] XFS (dm-2): Mounting Filesystem
[20914.653966] XFS (dm-2): Ending clean mount
[20915.812263] XFS (dm-0): Mounting Filesystem
[20916.628209] XFS (dm-0): Ending clean mount
[20917.997069] divide error: 0000 [#1] SMP
[20917.998029] Modules linked in: binfmt_misc(F) cmtp(F) kernelcapi(F) tun(F) l2tp_ppp(F) l2tp_netlink(F) l2tp_core(F) bnep(F) hidp(F) nfnetlink(F) scsi_transport_iscsi(F) ipt_ULOG(F) af_802154(F) atm(F) rds(F) af_key(F) bluetooth(F) rfkill(F) pppoe(F) pppox(F) ppp_generic(F) slhc(F) nfc(F) lockd(F) sunrpc(F) nf_conntrack_netbios_ns(F) nf_conntrack_broadcast(F) ipt_MASQUERADE(F) ip6table_nat(F) nf_nat_ipv6(F) ip6table_mangle(F) ip6t_REJECT(F) nf_conntrack_ipv6(F) nf_defrag_ipv6(F) iptable_nat(F) nf_nat_ipv4(F) nf_nat(F) iptable_mangle(F) ipt_REJECT(F) nf_conntrack_ipv4(F) nf_defrag_ipv4(F) xt_conntrack(F) nf_conntrack(F) ebtable_filter(F) ebtables(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) sg(F) bnx2(F) kvm_amd(F) amd64_edac_mod(F) edac_mce_amd(F) kvm(F) pcspkr(F) serio_raw(F) shpchp(F) i2c_piix4(F) edac_core(F) k8temp(F) xfs(F) libcrc32c(F) sd_mod(F) crc_t10dif(F) radeon(F) i2c_algo_bit(F) drm_kms_helper(F) mptsas(F) ttm(F) scsi_transport_sas(F) drm(F) mptscsih(F) i2c_core(F) mptbase(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F)
[20917.998029] CPU 0
[20917.998029] Pid: 10, comm: rcu_sched Tainted: GF 3.9.4 #1 IBM BladeCenter LS41 -[7972AC1]-/Server Blade
[20917.998029] RIP: 0010:[<ffffffff8109ed2b>] [<ffffffff8109ed2b>] update_sd_lb_stats+0x2bb/0x5b0
[20917.998029] RSP: 0018:ffff88013a35fa60 EFLAGS: 00010016
[20917.998029] RAX: 0000000000044800 RBX: 0000000000000000 RCX: ffff880439f5ba18
[20917.998029] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000008
[20917.998029] RBP: ffff88013a35fb28 R08: 0000000000000112 R09: 0000000000000000
[20917.998029] R10: 0000000000000000 R11: 0000000000000001 R12: 0000000000000008
[20917.998029] R13: 0000000000000000 R14: 0000000000000001 R15: ffff88013a35fc50
[20917.998029] FS: 00007fb8dd06e740(0000) GS:ffff88013bc00000(0000) knlGS:0000000000000000
[20917.998029] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[20917.998029] CR2: 0000000001890840 CR3: 00000002205ce000 CR4: 00000000000007f0
[20917.998029] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[20917.998029] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[20917.998029] Process rcu_sched (pid: 10, threadinfo ffff88013a35e000, task ffff88013a365040)
[20917.998029] Stack:
[20917.998029] 0100000000000000 ffff88013a35fce4 ffff88013a35fb38 00000000ffffffff
[20917.998029] ffff880139c21fb0 0000000000014400 0000000000014400 0000000000000000
[20917.998029] 0000000000000001 0000000000000112 ffff880139c21fc8 0000000000000000
[20917.998029] Call Trace:
[20917.998029] [<ffffffff8109f056>] find_busiest_group+0x36/0x4b0
[20917.998029] [<ffffffff8109f5d8>] load_balance+0x108/0x7d0
[20917.998029] [<ffffffff8108fa7c>] ? update_rq_clock.part.67+0x1c/0x170
[20917.998029] [<ffffffff810a0142>] idle_balance+0x182/0x2f0
[20917.998029] [<ffffffff8160f1ac>] __schedule+0x7bc/0x7d0
[20917.998029] [<ffffffff8160f1e9>] schedule+0x29/0x70
[20917.998029] [<ffffffff8160d8e2>] schedule_timeout+0x172/0x2c0
[20917.998029] [<ffffffff810f709a>] ? rcu_report_qs_rnp+0x7a/0x130
[20917.998029] [<ffffffff81070210>] ? __internal_add_timer+0x130/0x130
[20917.998029] [<ffffffff810f8ce3>] rcu_gp_kthread+0x313/0x560
[20917.998029] [<ffffffff810850e0>] ? wake_up_bit+0x30/0x30
[20917.998029] [<ffffffff810f89d0>] ? rcu_gp_fqs+0x80/0x80
[20917.998029] [<ffffffff81084240>] kthread+0xc0/0xd0
[20917.998029] [<ffffffff81084180>] ? insert_kthread_work+0x40/0x40
[20917.998029] [<ffffffff816192ac>] ret_from_fork+0x7c/0xb0
[20917.998029] [<ffffffff81084180>] ? insert_kthread_work+0x40/0x40
[20917.998029] Code: b5 50 ff ff ff 49 8b 3f e8 b3 fb ff ff 0f 1f 00 48 8b 85 58 ff ff ff 48 8b 48 10 4c 8b 45 a0 8b 71 04 31 d2 4c 89 c0 48 c1 e0 0a <48> f7 f6 48 8b 75 a8 48 85 f6 48 89 c7 49 89 c1 48 89 45 98 0f
[20917.998029] RIP [<ffffffff8109ed2b>] update_sd_lb_stats+0x2bb/0x5b0
[20917.998029] RSP <ffff88013a35fa60>
[20917.998029] ---[ end trace 9d1c03412e3d1bf5 ]---
[20918.017044] divide error: 0000 [#2] SMP
[20918.017044] Modules linked in: binfmt_misc(F) cmtp(F) kernelcapi(F) tun(F) l2tp_ppp(F) l2tp_netlink(F) l2tp_core(F) bnep(F) hidp(F) nfnetlink(F) scsi_transport_iscsi(F) ipt_ULOG(F) af_802154(F) atm(F) rds(F) af_key(F) bluetooth(F) rfkill(F) pppoe(F) pppox(F) ppp_generic(F) slhc(F) nfc(F) lockd(F) sunrpc(F) nf_conntrack_netbios_ns(F) nf_conntrack_broadcast(F) ipt_MASQUERADE(F) ip6table_nat(F) nf_nat_ipv6(F) ip6table_mangle(F) ip6t_REJECT(F) nf_conntrack_ipv6(F) nf_defrag_ipv6(F) iptable_nat(F) nf_nat_ipv4(F) nf_nat(F) iptable_mangle(F) ipt_REJECT(F) nf_conntrack_ipv4(F) nf_defrag_ipv4(F) xt_conntrack(F) nf_conntrack(F) ebtable_filter(F) ebtables(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) sg(F) bnx2(F) kvm_amd(F) amd64_edac_mod(F) edac_mce_amd(F) kvm(F) pcspkr(F) serio_raw(F) shpchp(F) i2c_piix4(F) edac_core(F) k8temp(F) xfs(F) libcrc32c(F) sd_mod(F) crc_t10dif(F) radeon(F) i2c_algo_bit(F) drm_kms_helper(F) mptsas(F) ttm(F) scsi_transport_sas(F) drm(F) mptscsih(F) i2c_core(F) mptbase(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F)
[20918.017044] CPU 2
[20918.017044] Pid: 0, comm: swapper/2 Tainted: GF D 3.9.4 #1 IBM BladeCenter LS41 -[7972AC1]-/Server Blade
[20918.017044] RIP: 0010:[<ffffffff8109ed2b>] [<ffffffff8109ed2b>] update_sd_lb_stats+0x2bb/0x5b0
[20918.017044] RSP: 0018:ffff88033bc03c00 EFLAGS: 00010216
[20918.017044] RAX: 0000000000040800 RBX: 0000000000000000 RCX: ffff880439f5ba18
[20918.017044] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000008
[20918.017044] RBP: ffff88033bc03cc8 R08: 0000000000000102 R09: 0000000000000005
[20918.017044] R10: 0000000000000000 R11: 0000000000000001 R12: 0000000000000008
[20918.017044] R13: 0000000000000000 R14: 0000000000000001 R15: ffff88033bc03df0
[20918.017044] FS: 00007fb8dd06e740(0000) GS:ffff88033bc00000(0000) knlGS:0000000000000000
[20918.017044] CS: 0010 DS: 0000 ES: 0000 CR[-- MARK -- Thu May 30 10:40:00 2013]
3)
[10721.161888] XFS (dm-0): Ending clean mount
[10722.090855] XFS (dm-2): Mounting Filesystem
[10722.143682] XFS (dm-2): Ending clean mount
[10722.584327] XFS (dm-0): Mounting Filesystem
[10722.949519] XFS (dm-0): Ending clean mount
[10723.579704] BUG: unable to handle kernel paging request at ffff8801b55cc000
[10723.619185] IP: [<ffffffff81302305>] memmove+0x55/0x1a0
[10723.647690] PGD 1ddf067 PUD 20bdfa067 PMD 1b5734063 PTE 80000001b55cc161
[10723.685013] Oops: 0003 [#1] SMP
[10723.701968] Modules linked in: binfmt_misc(F) bnep(F) hidp(F) tun(F) fuse(F) cmtp(F) kernelcapi(F) l2tp_ppp(F) l2tp_netlink(F) l2tp_core(F) rfcomm(F) nfnetlink(F) scsi_transport_iscsi(F) ipt_ULOG(F) af_802154(F) atm(F) bluetooth(F) rfkill(F) af_key(F) nfc(F) pppoe(F) pppox(F) ppp_generic(F) slhc(F) rds(F) lockd(F) sunrpc(F) nf_conntrack_netbios_ns(F) nf_conntrack_broadcast(F) ipt_MASQUERADE(F) ip6table_nat(F) nf_nat_ipv6(F) ip6table_mangle(F) ip6t_REJECT(F) nf_conntrack_ipv6(F) nf_defrag_ipv6(F) iptable_nat(F) nf_nat_ipv4(F) nf_nat(F) iptable_mangle(F) ipt_REJECT(F) nf_conntrack_ipv4(F) nf_defrag_ipv4(F) xt_conntrack(F) nf_conntrack(F) ebtable_filter(F) ebtables(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) sg(F) ixgbe(F) iTCO_wdt(F) mdio(F) iTCO_vendor_support(F) dca(F) e1000e(F) lpc_ich(F) serio_raw(F) ptp(F) mfd_core(F) hpilo(F) pcspkr(F) hpwdt(F) pps_core(F) microcode(F) xfs(F) libcrc32c(F) sd_mod(F) crc_t10dif(F) ata_generic(F) mgag200(F)[10724.128502] sleep[6604]: segfault at 0 ip (null) sp 00007fff96b37a30 error 14 in sleep[400000+6000]
[10724.128757] ------------[ cut here ]------------
[10724.128760] WARNING: at mm/mmap.c:2711 exit_mmap+0x166/0x170()
[10724.128761] Hardware name: ProLiant DL120 G7
[10724.128761] Modules linked in: binfmt_misc(F) bnep(F) hidp(F) tun(F) fuse(F) cmtp(F) kernelcapi(F) l2tp_ppp(F) l2tp_netlink(F) l2tp_core(F) rfcomm(F) nfnetlink(F) scsi_transport_iscsi(F) ipt_ULOG(F) af_802154(F) atm(F) bluetooth(F) rfkill(F) af_key(F) nfc(F) pppoe(F) pppox(F) ppp_generic(F) slhc(F) rds(F) lockd(F) sunrpc(F) nf_conntrack_netbios_ns(F) nf_conntrack_broadcast(F) ipt_MASQUERADE(F) ip6table_nat(F) nf_nat_ipv6(F) ip6table_mangle(F) ip6t_REJECT(F) nf_conntrack_ipv6(F) nf_defrag_ipv6(F) iptable_nat(F) nf_nat_ipv4(F) nf_nat(F) iptable_mangle(F) ipt_REJECT(F) nf_conntrack_ipv4(F) nf_defrag_ipv4(F) xt_conntrack(F) nf_conntrack(F) ebtable_filter(F) ebtables(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) sg(F) ixgbe(F) iTCO_wdt(F) mdio(F) iTCO_vendor_support(F) dca(F) e1000e(F) lpc_ich(F) serio_raw(F) ptp(F) mfd_core(F) hpilo(F) pcspkr(F) hpwdt(F) pps_core(F) microcode(F) xfs(F) libcrc32c(F) sd_mod(F) crc_t10dif(F) ata_generic(F) mgag200(F) i2c_algo_bit(F) pata_acpi(F) drm_kms_helper(F) ttm(F) ata_piix(F) drm(F) hpsa(F) libata(F) i2c_core(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F)
[10724.128795] Pid: 6604, comm: sleep Tainted: GF 3.9.4 #1
[10724.128796] Call Trace:
[10724.128796] [<ffffffff81060f50>] warn_slowpath_common+0x70/0xa0
[10724.128800] traps: auditd[520] general protection ip:7f9a89363be5 sp:7f9a8801f680 error:0 [<ffffffff8106103a>] warn_slowpath_null+0x1a/0x20
[10724.128802] [<ffffffff811647d6>] exit_mmap+0x166/0x170
[10724.128804] in libc-2.17.so[7f9a8931c000+1b5000] [<ffffffff8105e377>] mmput+0x67/0xf0
[10724.128806]
[10724.128806] [<ffffffff81066898>] do_exit+0x278/0xa20
[10724.128816] [<ffffffff810670bf>] do_group_exit+0x3f/0xa0
[10724.128817] [<ffffffff81075ead>] get_signal_to_deliver+0x1ad/0x5c0
[10724.128820] [<ffffffff81013438>] do_signal+0x48/0x5a0
[10724.128823] [<ffffffff81614d4c>] ? __do_page_fault+0x45c/0x510
[10724.128825] [<ffffffff810879b8>] ? hrtimer_try_to_cancel+0x48/0xe0
[10724.128827] [<ffffffff8108879d>] ? hrtimer_nanosleep+0xad/0x170
[10724.128829] [<ffffffff81087820>] ? hrtimer_get_res+0x50/0x50
[10724.128830] [<ffffffff81013a00>] do_notify_resume+0x70/0xa0
[10724.128831] [<ffffffff816112bc>] retint_signal+0x48/0x8c
[10724.128834] ---[ end trace aa0156eab31509fc ]---
[10724.128901] BUG: Bad rss-counter state mm:ffff8801fb90af40 idx:0 val:5
[10724.128901] BUG: Bad rss-counter state mm:ffff8801fb90af40 idx:1 val:3
[10724.128927] rhts-test-runne[48621]: segfault at 19cc ip 00000000004401c4 sp 00007fff27f840e0 error 6 in bash[400000+db000]
[10724.129726] =============================================================================
[10724.129727] BUG kmalloc-4096 (Tainted: GF W ): Padding overwritten. 0xffff88019152fc28-0xffff88019152ffff
[10724.129727] -----------------------------------------------------------------------------
[10724.129727]
[10724.129728] INFO: Slab 0xffffea0006454a00 objects=7 used=7 fp=0x (null) flags=0x20000000004080
[10724.129729] Pid: 43729, comm: kworker/0:0 Tainted: GF B W 3.9.4 #1
[10724.129729] Call Trace:
[10724.129730] [<ffffffff81181ed2>] slab_err+0xc2/0xf0
[10724.129733] [<ffffffff812f8007>] ? kset_init+0x27/0x40
[10724.129735] [<ffffffff81181ff5>] slab_pad_check.part.41+0xf5/0x170
[10724.129736] [<ffffffff810ce4d2>] ? cgroup_release_agent+0x42/0x180
[10724.129739] [<ffffffff811820e3>] check_slab+0x73/0x100
[10724.129740] [<ffffffff81606b50>] alloc_debug_processing+0x21/0x118
[10724.129742] [<ffffffff8160772f>] __slab_alloc+0x3b8/0x4a2
[10724.129744] [<ffffffff8109f5d8>] ? load_balance+0x108/0x7d0
[10724.129746] [<ffffffff810ce4d2>] ? cgroup_release_agent+0x42/0x180
[10724.129747] [<ffffffff8108fa7c>] ? update_rq_clock.part.67+0x1c/0x170
[10724.129749] [<ffffffff81184dd1>] kmem_cache_alloc_trace+0x1b1/0x200
[10724.129751] [<ffffffff810ce4d2>] cgroup_release_agent+0x42/0x180
[10724.129752] [<ffffffff8160eda0>] ? __schedule+0x3b0/0x7d0
[10724.129754] [<ffffffff8107d80f>] process_one_work+0x16f/0x3f0
[10724.129756] [<ffffffff8107f360>] worker_thread+0x110/0x3d0
[10724.129757] [<ffffffff8107f250>] ? __alloc_workqueue_key+0x500/0x500
[10724.129758] [<ffffffff81084240>] kthread+0xc0/0xd0
[10724.129759] [<ffffffff81084180>] ? insert_kthread_work+0x40/0x40
[10724.129760] [<ffffffff816192ac>] ret_from_fork+0x7c/0xb0
[10724.129762] [<ffffffff81084180>] ? insert_kthread_work+0x40/0x40
[10724.129765] Padding ffff88019152f8f8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129765] Padding ffff88019152f908: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129766] Padding ffff88019152f918: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129766] Padding ffff88019152f928: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129767] Padding ffff88019152f938: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129767] Padding ffff88019152f948: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129768] Padding ffff88019152f958: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129768] Padding ffff88019152f968: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129769] Padding ffff88019152f978: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129769] Padding ffff88019152f988: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129770] Padding ffff88019152f998: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129770] Padding ffff88019152f9a8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129771] Padding ffff88019152f9b8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129771] Padding ffff88019152f9c8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129772] Padding ffff88019152f9d8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129772] Padding ffff88019152f9e8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129773] Padding ffff88019152f9f8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129773] Padding ffff88019152fa08: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129774] Padding ffff88019152fa18: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129774] Padding ffff88019152fa28: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129775] Padding ffff88019152fa38: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129775] Padding ffff88019152fa48: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129776] Padding ffff88019152fa58: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129776] Padding ffff88019152fa68: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129777] Padding ffff88019152fa78: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129777] Padding ffff88019152fa88: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129778] Padding ffff88019152fa98: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129778] Padding ffff88019152faa8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129779] Padding ffff88019152fab8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129779] Padding ffff88019152fac8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129780] Padding ffff88019152fad8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129780] Padding ffff88019152fae8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129781] Padding ffff88019152faf8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129781] Padding ffff88019152fb08: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129782] Padding ffff88019152fb18: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129782] Padding ffff88019152fb28: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129783] Padding ffff88019152fb38: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129783] Padding ffff88019152fb48: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129784] Padding ffff88019152fb58: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129784] Padding ffff88019152fb68: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129785] Padding ffff88019152fb78: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129785] Padding ffff88019152fb88: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129786] Padding ffff88019152fb98: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129786] Padding ffff88019152fba8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129787] Padding ffff88019152fbb8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129787] Padding ffff88019152fbc8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129788] Padding ffff88019152fbd8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129788] Padding ffff88019152fbe8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129789] Padding ffff88019152fbf8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129789] Padding ffff88019152fc08: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129790] Padding ffff88019152fc18: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[10724.129790] Padding ffff88019152fc28: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129791] Padding ffff88019152fc38: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129792] Padding ffff88019152fc48: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129792] Padding ffff88019152fc58: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129793] Padding ffff88019152fc68: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129793] Padding ffff88019152fc78: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129794] Padding ffff88019152fc88: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129794] Padding ffff88019152fc98: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129795] Padding ffff88019152fca8: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129795] Padding ffff88019152fcb8: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129796] Padding ffff88019152fcc8: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129796] Padding ffff88019152fcd8: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129797] Padding ffff88019152fce8: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129797] Padding ffff88019152fcf8: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129798] Padding ffff88019152fd08: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129798] Padding ffff88019152fd18: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129799] Padding ffff88019152fd28: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129799] Padding ffff88019152fd38: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129800] Padding ffff88019152fd48: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129800] Padding ffff88019152fd58: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129801] Padding ffff88019152fd68: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129801] Padding ffff88019152fd78: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129802] Padding ffff88019152fd88: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129802] Padding ffff88019152fd98: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129803] Padding ffff88019152fda8: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129803] Padding ffff88019152fdb8: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129804] Padding ffff88019152fdc8: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129804] Padding ffff88019152fdd8: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129805] Padding ffff88019152fde8: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129805] Padding ffff88019152fdf8: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129806] Padding ffff88019152fe08: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129806] Padding ffff88019152fe18: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129807] Padding ffff88019152fe28: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129807] Padding ffff88019152fe38: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129808] Padding ffff88019152fe48: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129808] Padding ffff88019152fe58: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129809] Padding ffff88019152fe68: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129809] Padding ffff88019152fe78: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129810] Padding ffff88019152fe88: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129810] Padding ffff88019152fe98: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129811] Padding ffff88019152fea8: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129811] Padding ffff88019152feb8: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129812] Padding ffff88019152fec8: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129812] Padding ffff88019152fed8: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129813] Padding ffff88019152fee8: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129813] Padding ffff88019152fef8: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129814] Padding ffff88019152ff08: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129814] Padding ffff88019152ff18: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129815] Padding ffff88019152ff28: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129815] Padding ffff88019152ff38: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129816] Padding ffff88019152ff48: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129816] Padding ffff88019152ff58: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129817] Padding ffff88019152ff68: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129817] Padding ffff88019152ff78: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129818] Padding ffff88019152ff88: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129818] Padding ffff88019152ff98: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129819] Padding ffff88019152ffa8: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129819] Padding ffff88019152ffb8: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129820] Padding ffff88019152ffc8: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129820] Padding ffff88019152ffd8: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129821] Padding ffff88019152ffe8: c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 c6 ................
[10724.129821] Padding ffff88019152fff8: c6 c6 c6 c6 c6 c6 c6 c6 ........
[10724.129822] FIX kmalloc-4096: Restoring 0xffff88019152f8f8-0xffff88019152ffff=0x5a
[10724.129822]
[10724.135254] systemd[1]: segfault at 45 ip 00007f93e348c780 sp 00007fff4deb3a9e error 4 in libdbus-1.so.3.7.2[7f93e3479000+45000]
[10724.135272] traps: systemd[1] trap invalid opcode ip:7f93e498c490 sp:7fff4deb34f8 error:0 in systemd[7f93e496d000+f6000]
[10724.135281] traps: systemd[1] trap invalid opcode ip:7f93e498c490 sp:7fff4deb2f38 error:0 in systemd[7f93e496d000+f6000]
[10724.135289] traps: systemd[1] trap invalid opcode ip:7f93e498c490 sp:7fff4deb2978 error:0 in systemd[7f93e496d000+f6000]
[10724.135297] traps: systemd[1] trap invalid opcode ip:7f93e498c490 sp:7fff4deb23b8 error:0 in systemd[7f93e496d000+f6000]
[10724.135304] traps: systemd[1] trap invalid opcode ip:7f93e498c490 sp:7fff4deb1df8 error:0 in systemd[7f93e496d000+f6000]
[10724.135312] traps: systemd[1] trap invalid opcode ip:7f93e498c490 sp:7fff4deb1838 error:0 in systemd[7f93e496d000+f6000]
[10724.169488] audit: *NO* daemon at audit_pid=505
[10732.129639] i2c_algo_bit(F) pata_acpi(F) drm_kms_helper(F) ttm(F) ata_piix(F) drm(F) hpsa(F) libata(F) i2c_core(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F)
[10732.206898] CPU 3
[10732.215891] Pid: 6911, comm: attr Tainted: GF B W 3.9.4 #1 HP ProLiant DL120 G7
[10732.256496] RIP: 0010:[<ffffffff81302305>] [<ffffffff81302305>] memmove+0x55/0x1a0
[10732.296932] RSP: 0018:ffff8801bbe15ad8 EFLAGS: 00010282
[10732.325205] RAX: ffff880191529168 RBX: ffff8801e687b000 RCX: 000000000000d00b
[10732.363603] RDX: ffffffffdbf5d138 RSI: ffff8801b55cc3e0 RDI: ffff8801b55cbfe8
[10732.401581] RBP: ffff8801bbe15b30 R08: 38408bffffff4085 R09: 8b48dceb000000b8
[10732.440757] R10: 9f89498eeb000000 R11: 20befffffc65e9e2 R12: ffff880191529148
[10732.475959] R13: 00000000000003d8 R14: 0000000000001800 R15: 000000000000007b
[10732.512079] FS: 00007fc626dc9740(0000) GS:ffff88020b460000(0000) knlGS:0000000000000000
[10732.553296] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[10732.581757] CR2: ffff8801b55cc000 CR3: 00000001e169c000 CR4: 00000000000407e0
[10732.617759] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[10732.653908] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[10732.689045] Process attr (pid: 6911, threadinfo ffff8801bbe14000, task ffff8801e6b71ac0)
[10732.729564] Stack:
[10732.739413] ffffffffa01651a1 0000000000000000 0000000000000000 ffff880191529168
[10732.776436] ffff880191529148 0000000000000018 ffff880191529148 0000000000000000
[10732.813437] ffff8801de8237b0 ffff8801c491d280 ffff8801e2c776c0 ffff8801bbe15b68
[10732.852979] Call Trace:
[10732.865992] [<ffffffffa01651a1>] ? xfs_attr_leaf_moveents.isra.2+0x91/0x280 [xfs]
[10732.906399] [<ffffffffa0165467>] xfs_attr_leaf_compact+0xd7/0x130 [xfs]
[10732.942002] [<ffffffffa0166a2e>] xfs_attr_leaf_add+0xce/0x170 [xfs]
[10732.975905] [<ffffffffa0162850>] xfs_attr_leaf_addname+0xc0/0x3d0 [xfs]
[10733.009452] [<ffffffffa0170d4e>] ? xfs_bmap_one_block+0x3e/0xa0 [xfs]
[10733.042797] [<ffffffffa016378c>] xfs_attr_set_int+0x30c/0x420 [xfs]
[10733.074645] [<ffffffff811be9f4>] ? setxattr+0xa4/0x1c0
[10733.100539] [<ffffffffa0163d1f>] xfs_attr_set+0x7f/0x90 [xfs]
[10733.129870] [<ffffffffa0159a12>] xfs_xattr_set+0x42/0x70 [xfs]
[10733.159595] [<ffffffff811bdef2>] generic_setxattr+0x62/0x80
[10733.188208] [<ffffffff811be743>] __vfs_setxattr_noperm+0x63/0x1b0
[10733.218633] [<ffffffff811be945>] vfs_setxattr+0xb5/0xc0
[10733.245380] [<ffffffff811bea76>] setxattr+0x126/0x1c0
[10733.270691] [<ffffffff8118358d>] ? kmem_cache_free+0x1cd/0x1e0
[10733.300416] [<ffffffff811a72b2>] ? final_putname+0x22/0x50
[10733.327832] [<ffffffff811a74cb>] ? putname+0x2b/0x40
[10733.354383] [<ffffffff811ab96f>] ? user_path_at_empty+0x5f/0x90
[10733.385198] [<ffffffff8119e5c9>] ? __sb_start_write+0x49/0x100
[10733.416587] [<ffffffff811bedef>] sys_lsetxattr+0x8f/0xd0
[10733.445421] [<ffffffff81619359>] system_call_fastpath+0x16/0x1b
[10733.477008] Code: 40 38 fe 74 41 48 83 ea 20 48 83 ea 20 4c 8b 1e 4c 8b 56 08 4c 8b 4e 10 4c 8b 46 18 48 8d 76 20 4c 89 1f 4c 89 57 08 4c 89 4f 10 <4c> 89 47 18 48 8d 7f 20 73 d4 48 83 c2 20 e9 a8 00 00 00 0f 1f
[10733.575487] RIP [<ffffffff81302305>] memmove+0x55/0x1a0
[10733.602350] RSP <ffff8801bbe15ad8>
[10733.620195] CR2: ffff8801b55cc000
[10733.693452] ---[ end trace aa0156eab31509fd ]---
[10733.693454] general protection fault: 0000 [#2] SMP
[10733.693468] Modules linked in: binfmt_misc(F) bnep(F) hidp(F) tun(F) fuse(F) cmtp(F) kernelcapi(F) l2tp_ppp(F) l2tp_netlink(F) l2tp_core(F) rfcomm(F) nfnetlink(F) scsi_transport_iscsi(F) ipt_ULOG(F) af_802154(F) atm(F) bluetooth(F) rfkill(F) af_key(F) nfc(F) pppoe(F) pppox(F) ppp_generic(F) slhc(F) rds(F) lockd(F) sunrpc(F) nf_conntrack_netbios_ns(F) nf_conntrack_broadcast(F) ipt_MASQUERADE(F) ip6table_nat(F) nf_nat_ipv6(F) ip6table_mangle(F) ip6t_REJECT(F) nf_conntrack_ipv6(F) nf_defrag_ipv6(F) iptable_nat(F) nf_nat_ipv4(F) nf_nat(F) iptable_mangle(F) ipt_REJECT(F) nf_conntrack_ipv4(F) nf_defrag_ipv4(F) xt_conntrack(F) nf_conntrack(F) ebtable_filter(F) ebtables(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) sg(F) ixgbe(F) iTCO_wdt(F) mdio(F) iTCO_vendor_support(F) dca(F) e1000e(F) lpc_ich(F) serio_raw(F) ptp(F) mfd_core(F) hpilo(F) pcspkr(F) hpwdt(F) pps_core(F) microcode(F) xfs(F) libcrc32c(F) sd_mod(F) crc_t10dif(F) ata_generic(F) mgag200(F) i2c_algo_bit(F) pata_acpi(F) drm_kms_helper(F) ttm(F) ata_piix(F) drm(F) hpsa(F) libata(F) i2c_core(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F)
[10733.693475] CPU 1
[10733.693475] Pid: 48621, comm: rhts-test-runne Tainted: GF B D W 3.9.4 #1 HP ProLiant DL120 G7
[10733.693478] RIP: 0010:[<ffffffff8115787e>] [<ffffffff8115787e>] vma_interval_tree_remove+0x3e/0x240
[10733.693479] RSP: 0000:ffff8801fcdcfba0 EFLAGS: 00010206
[10733.693479] RAX: 0000be0f00000000 RBX: ffff8801e0abda00 RCX: ffff8801a65ca058
[10733.693480] RDX: 0000be0f00000000 RSI: ffff8801f74b0760 RDI: ffff8801e0abda00
[10733.693480] RBP: ffff8801fcdcfbb8 R08: ffff8801a65cb800 R09: 0000000000017700
[10733.693481] R10: ffff8801e0abda58 R11: 000000000000000e R12: ffff8801e0abda00
[10733.693481] R13: ffff8801f74b0778 R14: ffff8801f74b0740 R15: ffff8801e0abda00
[10733.693482] FS: 0000000000000000(0000) GS:ffff88020b420000(0000) knlGS:0000000000000000
[10733.693483] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[10733.693483] CR2: 00000000000019cc CR3: 00000001fbc0b000 CR4: 00000000000407e0
[10733.693484] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[10733.693484] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[10733.693485] Process rhts-test-runne (pid: 48621, threadinfo ffff8801fcdce000, task ffff8801e54f3580)
[10733.693485] Stack:
[10733.693487] ffffffff811613e6 ffff8801c4a68480 ffff8801e0abda00 ffff8801fcdcfbe8
[10733.693488] ffffffff811619a0 ffff8801dc1c1000 00007f9810be3000 0000000000000000
[10733.693489] ffff8801fcdcfc40 ffff8801fcdcfc28 ffffffff8115a240 0000000000000000
[10733.693489] Call Trace:
[10733.693491] [<ffffffff811613e6>] ? __remove_shared_vm_struct+0x46/0x70
[10733.693492] [<ffffffff811619a0>] unlink_file_vma+0x40/0x60
[10733.693493] [<ffffffff8115a240>] free_pgtables+0xb0/0x120
[10733.693495] [<ffffffff81164722>] exit_mmap+0xb2/0x170
[10733.693498] [<ffffffff8105e377>] mmput+0x67/0xf0
[10733.693500] [<ffffffff81066898>] do_exit+0x278/0xa20
[10733.693501] [<ffffffff810670bf>] do_group_exit+0x3f/0xa0
[10733.693503] [<ffffffff81075ead>] get_signal_to_deliver+0x1ad/0x5c0
[10733.693506] [<ffffffff81013438>] do_signal+0x48/0x5a0
[10733.693507] [<ffffffff81614d4c>] ? __do_page_fault+0x45c/0x510
[10733.693510] [<ffffffff8130aabd>] ? list_del+0xd/0x30
[10733.693511] [<ffffffff81084f51>] ? remove_wait_queue+0x31/0x40
[10733.693513] [<ffffffff8106736e>] ? sys_wait4+0x6e/0xe0
[10733.693514] [<ffffffff81013a00>] do_notify_resume+0x70/0xa0
[10733.693516] [<ffffffff816112bc>] retint_signal+0x48/0x8c
[10733.693525] Code: d2 0f 84 9f 01 00 00 48 85 c9 0f 84 bd 01 00 00 48 8b 41 10 49 89 c8 48 85 c0 75 0f e9 cb 01 00 00 0f 1f 40 00 49 89 c0 48 89 d0 <48> 8b 50 10 48 85 d2 75 f1 4c 8b 58 08 4d 89 58 10 48 89 48 08
[10733.693526] RIP [<ffffffff8115787e>] vma_interval_tree_remove+0x3e/0x240
[10733.693527] RSP <ffff8801fcdcfba0>
[10733.693529] general protection fault: 0000 [#3] SMP
[10733.693549] Modules linked in: binfmt_misc(F) bnep(F) hidp(F) tun(F) fuse(F) cmtp(F) kernelcapi(F) l2tp_ppp(F) l2tp_netlink(F) l2tp_core(F) rfcomm(F) nfnetlink(F) scsi_transport_iscsi(F) ipt_ULOG(F) af_802154(F) atm(F) bluetooth(F) rfkill(F) af_key(F) nfc(F) pppoe(F) pppox(F) ppp_generic(F) slhc(F) rds(F) lockd(F) sunrpc(F) nf_conntrack_netbios_ns(F) nf_conntrack_broadcast(F) ipt_MASQUERADE(F) ip6table_nat(F) nf_nat_ipv6(F) ip6table_mangle(F) ip6t_REJECT(F) nf_conntrack_ipv6(F) nf_defrag_ipv6(F) iptable_nat(F) nf_nat_ipv4(F) nf_nat(F) iptable_mangle(F) ipt_REJECT(F) nf_conntrack_ipv4(F) nf_defrag_ipv4(F) xt_conntrack(F) nf_conntrack(F) ebtable_filter(F) ebtables(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) sg(F) ixgbe(F) iTCO_wdt(F) mdio(F) iTCO_vendor_support(F) dca(F) e1000e(F) lpc_ich(F) serio_raw(F) ptp(F) mfd_core(F) hpilo(F) pcspkr(F) hpwdt(F) pps_core(F) microcode(F) xfs(F) libcrc32c(F) sd_mod(F) crc_t10dif(F) ata_generic(F) mgag200(F) i2c_algo_bit(F) pata_acpi(F) drm_kms_helper(F) ttm(F) ata_piix(F) drm(F) hpsa(F) libata(F) i2c_core(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F)
[10733.693557] CPU 0
[10733.693557] Pid: 43729, comm: kworker/0:0 Tainted: GF B D W 3.9.4 #1 HP ProLiant DL120 G7
[10733.693559] RIP: 0010:[<ffffffff81183889>] [<ffffffff81183889>] deactivate_slab+0x119/0x510
[10733.693562] RSP: 0000:ffff8801fdfd5bb8 EFLAGS: 00010006
[10733.693563] RAX: 0000000000001008 RBX: ffff88019152e7b0 RCX: 0000000180070006
[10733.693564] RDX: 0000000180070007 RSI: ffffea0006454a00 RDI: ffff880202c0ee40
[10733.693565] RBP: ffff8801fdfd5ca8 R08: 0000000000000000 R09: ffff880191529000
[10733.693567] R10: 0000000000000000 R11: ffff880202c00c40 R12: ffffea0006454a00
[10733.693567] R13: ffff880202c0ee40 R14: 5a5a5a5a5a5a5a5a R15: 00000000ffffffff
[10733.693568] FS: 0000000000000000(0000) GS:ffff88020b400000(0000) knlGS:0000000000000000
[10733.693569] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[10733.693569] CR2: 0000000000000000 CR3: 00000000018fa000 CR4: 00000000000407f0
[10733.693570] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[10733.693570] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[10733.693572] Process kworker/0:0 (pid: 43729, threadinfo ffff8801fdfd4000, task ffff8801b575d040)
[10733.693572] Stack:
[10733.693576] ffffffff810154cf ffff8801fdfd5bf4 0000000f00000001 ffff88020b403fc0
[10733.693578] 0000000000000000 ffff8801fdfd4000 ffff8801fdfd5c50 00000000811817db
[10733.693580] ffffea0006454a00 ffff8801fdfd5c50 ffffffff810ce4d2 ffff880191528000
[10733.693581] Call Trace:
[10733.693584] [<ffffffff810154cf>] ? dump_trace+0x16f/0x2b0
[10733.693586] [<ffffffff810ce4d2>] ? cgroup_release_agent+0x42/0x180
[10733.693589] [<ffffffff81020c2b>] ? save_stack_trace+0x2b/0x50
[10733.693590] [<ffffffff81180911>] ? set_track+0x61/0x1b0
[10733.693592] [<ffffffff8118176d>] ? init_object+0x3d/0x70
[10733.693593] ---[ end trace aa0156eab31509fe ]---
[10733.693595] [<ffffffff81607756>] __slab_alloc+0x3df/0x4a2
[10733.693595] Fixing recursive fault but reboot is needed!
[10733.693597] [<ffffffff8109f5d8>] ? load_balance+0x108/0x7d0
[10733.693599] [<ffffffff810ce4d2>] ? cgroup_release_agent+0x42/0x180
[10733.693601] [<ffffffff8108fa7c>] ? update_rq_clock.part.67+0x1c/0x170
[10733.693603] [<ffffffff81184dd1>] kmem_cache_alloc_trace+0x1b1/0x200
[10733.693605] [<ffffffff810ce4d2>] cgroup_release_agent+0x42/0x180
[10733.693607] [<ffffffff8160eda0>] ? __schedule+0x3b0/0x7d0
[10733.693610] [<ffffffff8107d80f>] process_one_work+0x16f/0x3f0
[10733.693612] [<ffffffff8107f360>] worker_thread+0x110/0x3d0
[10733.693613] [<ffffffff8107f250>] ? __alloc_workqueue_key+0x500/0x500
[10733.693615] [<ffffffff81084240>] kthread+0xc0/0xd0
[10733.693616] [<ffffffff81084180>] ? insert_kthread_work+0x40/0x40
[10733.693618] [<ffffffff816192ac>] ret_from_fork+0x7c/0xb0
[10733.693620] [<ffffffff81084180>] ? insert_kthread_work+0x40/0x40
[10733.693642] Code: 40 00 49 3b 54 24 18 75 ed 49 89 5c 24 10 49 89 4c 24 18 41 0f ba 34 24 00 66 0f 1f 84 00 00 00 00 00 4d 85 f6 74 1d 49 63 45 20 <49> 8b 14 06 48 85 d2 74 10 4c 89 f3 4d 8b 44 24 10 49 89 d6 e9
[10733.693643] RIP [<ffffffff81183889>] deactivate_slab+0x119/0x510
[10733.693644] RSP <ffff8801fdfd5bb8>
[10733.693645] ---[ end trace aa0156eab31509ff ]---
[10733.693673] audit_printk_skb: 5695 callbacks suppressed
[10733.693674] type=1701 audit(1369899862.224:50838): auid=4294967295 uid=0 gid=0 ses=4294967295 subj=system_u:system_r:syslogd_t:s0 pid=38988 comm="in:imklog" reason="memory violation" sig=11
[10733.693771] general protection fault: 0000 [#4] SMP
[10733.693783] Modules linked in: binfmt_misc(F) bnep(F) hidp(F) tun(F) fuse(F) cmtp(F) kernelcapi(F) l2tp_ppp(F) l2tp_netlink(F) l2tp_core(F) rfcomm(F) nfnetlink(F) scsi_transport_iscsi(F) ipt_ULOG(F) af_802154(F) atm(F) bluetooth(F) rfkill(F) af_key(F) nfc(F) pppoe(F) pppox(F) ppp_generic(F) slhc(F) rds(F) lockd(F) sunrpc(F) nf_conntrack_netbios_ns(F) nf_conntrack_broadcast(F) ipt_MASQUERADE(F) ip6table_nat(F) nf_nat_ipv6(F) ip6table_mangle(F) ip6t_REJECT(F) nf_conntrack_ipv6(F) nf_defrag_ipv6(F) iptable_nat(F) nf_nat_ipv4(F) nf_nat(F) iptable_mangle(F) ipt_REJECT(F) nf_conntrack_ipv4(F) nf_defrag_ipv4(F) xt_conntrack(F) nf_conntrack(F) ebtable_filter(F) ebtables(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) sg(F) ixgbe(F) iTCO_wdt(F) mdio(F) iTCO_vendor_support(F) dca(F) e1000e(F) lpc_ich(F) serio_raw(F) ptp(F) mfd_core(F) hpilo(F) pcspkr(F) hpwdt(F) pps_core(F) microcode(F) xfs(F) libcrc32c(F) sd_mod(F) crc_t10dif(F) ata_generic(F) mgag200(F) i2c_algo_bit(F) pata_acpi(F) drm_kms_helper(F) ttm(F) ata_piix(F) drm(F) hpsa(F) libata(F) i2c_core(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F)
[10733.693790] CPU 2
[10733.693790] Pid: 38988, comm: in:imklog Tainted: GF B D W 3.9.4 #1 HP ProLiant DL120 G7
[10733.693792] RIP: 0010:[<ffffffff810c8b00>] [<ffffffff810c8b00>] acct_collect+0x60/0x1b0
[10733.693793] RSP: 0000:ffff8801e7977cd0 EFLAGS: 00010206
[10733.693793] RAX: 0000be0d00000000 RBX: ffff8801deb93c80 RCX: ffff8801e7977fd8
[10733.693794] RDX: 0000000000021000 RSI: 0000000000000001 RDI: ffff8801de900de8
[10733.693794] RBP: ffff8801e7977cf0 R08: ffff8801ff629ac0 R09: 0000000000000000
[10733.693795] R10: 0000000000000000 R11: 0000000000000001 R12: ffff8801ff629ac0
[10733.693795] R13: 000000000000000b R14: 0000000000000000 R15: ffff8801ff629ac0
[10733.693796] FS: 00007fc4aade0700(0000) GS:ffff88020b440000(0000) knlGS:0000000000000000
[10733.693797] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[10733.693797] CR2: 00007fff7fbabb10 CR3: 000000017e40c000 CR4: 00000000000407e0
[10733.693798] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[10733.693799] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[10733.693799] Process in:imklog (pid: 38988, threadinfo ffff8801e7976000, task ffff8801ff629ac0)
[10733.693800] Stack:
[10733.693801] 000000000000000b 000000000000984c ffff8801ff629ac0 ffff8801ff629ac0
[10733.693802] ffff8801e7977d78 ffffffff81066e01 00000001de900de8 ffff8801f9ee1128
[10733.693803] ffff8801e193ee50 000000800000000a ffff8801e7977e78 ffff8801e7977f58
[10733.693803] Call Trace:
[10733.693805] [<ffffffff81066e01>] do_exit+0x7e1/0xa20
[10733.693807] [<ffffffff810670bf>] do_group_exit+0x3f/0xa0
[10733.693808] [<ffffffff81075ead>] get_signal_to_deliver+0x1ad/0x5c0
[10733.693810] [<ffffffff81013438>] do_signal+0x48/0x5a0
[10733.693812] [<ffffffff81614d4c>] ? __do_page_fault+0x45c/0x510
[10733.693813] [<ffffffff810850e0>] ? wake_up_bit+0x30/0x30
[10733.693814] [<ffffffff81013a00>] do_notify_resume+0x70/0xa0
[10733.693816] [<ffffffff816112bc>] retint_signal+0x48/0x8c
[10733.693826] Code: 00 00 00 74 56 49 8b 84 24 80 04 00 00 48 8d 78 68 e8 25 5b 54 00 49 8b 84 24 80 04 00 00 48 8b 00 48 85 c0 74 1c 31 d2 0f 1f 00 <48> 03 50 08 48 2b 10 48 8b 40 10 48 85 c0 75 f0 48 c1 ea 0a 49
[10733.693827] RIP [<ffffffff810c8b00>] acct_collect+0x60/0x1b0
[10733.693827] RSP <ffff8801e7977cd0>
[10733.693828] ---[ end trace aa0156eab3150a00 ]---
[10733.693828] Fixing recursive fault but reboot is needed!
[10733.694251] NetworkManager[568]: segfault at 75 ip 00007eff6bc52bcb sp 00007fff5e0d5db0 error 4 in libglib-2.0.so.0.3600.1[7eff6bc0b000+12a000]
[10733.694275] type=1701 audit(1369899862.225:50839): auid=4294967295 uid=0 gid=0 ses=4294967295 subj=system_u:system_r:NetworkManager_t:s0 pid=568 comm="NetworkManager" reason="memory violation" sig=11
[10733.695784] BUG: unable to handle kernel paging request at ffffffffffffffd8
[10733.695786] IP: [<ffffffff810847e0>] kthread_data+0x10/0x20
[10733.695787] PGD 18fd067 PUD 18ff067 PMD 0
[10733.695788] Oops: 0000 [#5] SMP
[10733.695799] Modules linked in: binfmt_misc(F) bnep(F) hidp(F) tun(F) fuse(F) cmtp(F) kernelcapi(F) l2tp_ppp(F) l2tp_netlink(F) l2tp_core(F) rfcomm(F) nfnetlink(F) scsi_transport_iscsi(F) ipt_ULOG(F) af_802154(F) atm(F) bluetooth(F) rfkill(F) af_key(F) nfc(F) pppoe(F) pppox(F) ppp_generic(F) slhc(F) rds(F) lockd(F) sunrpc(F) nf_conntrack_netbios_ns(F) nf_conntrack_broadcast(F) ipt_MASQUERADE(F) ip6table_nat(F) nf_nat_ipv6(F) ip6table_mangle(F) ip6t_REJECT(F) nf_conntrack_ipv6(F) nf_defrag_ipv6(F) iptable_nat(F) nf_nat_ipv4(F) nf_nat(F) iptable_mangle(F) ipt_REJECT(F) nf_conntrack_ipv4(F) nf_defrag_ipv4(F) xt_conntrack(F) nf_conntrack(F) ebtable_filter(F) ebtables(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) sg(F) ixgbe(F) iTCO_wdt(F) mdio(F) iTCO_vendor_support(F) dca(F) e1000e(F) lpc_ich(F) serio_raw(F) ptp(F) mfd_core(F) hpilo(F) pcspkr(F) hpwdt(F) pps_core(F) microcode(F) xfs(F) libcrc32c(F) sd_mod(F) crc_t10dif(F) ata_generic(F) mgag200(F) i2c_algo_bit(F) pata_acpi(F) drm_kms_helper(F) ttm(F) ata_piix(F) drm(F) hpsa(F) libata(F) i2c_core(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F)
[10733.695806] CPU 0
[10733.695806] Pid: 43729, comm: kworker/0:0 Tainted: GF B D W 3.9.4 #1 HP ProLiant DL120 G7
[10733.695807] RIP: 0010:[<ffffffff810847e0>] [<ffffffff810847e0>] kthread_data+0x10/0x20
[10733.695808] RSP: 0018:ffff8801fdfd5968 EFLAGS: 00010002
[10733.695808] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 000000000000000f
[10733.695809] RDX: 0000000000000000 RSI: 0000000000000000 RDI: ffff8801b575d040
[10733.695809] RBP: ffff8801fdfd5968 R08: ffff8801b575d0b0 R09: 000009c3221f4808
[10733.695810] R10: 0000000000000000 R11: ffffea0007952900 R12: ffff88020b414400
[10733.695810] R13: 0000000000000000 R14: ffff8801b575d030 R15: ffff8801b575d040
[10733.695811] FS: 0000000000000000(0000) GS:ffff88020b400000(0000) knlGS:0000000000000000
[10733.695812] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[10733.695812] CR2: ffffffffffffffd8 CR3: 00000001e784d000 CR4: 00000000000407f0
[10733.695813] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[10733.695813] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[10733.695814] Process kworker/0:0 (pid: 43729, threadinfo ffff8801fdfd4000, task ffff8801b575d040)
[10733.695814] Stack:
[10733.695815] ffff8801fdfd5980 ffffffff8107f9a1 ffff8801b575d040 ffff8801fdfd59e0
[10733.695816] ffffffff8160efa2 ffff8801fdfd5fd8 0000000000014400 ffff8801fdfd5fd8
[10733.695817] 0000000000014400 ffff8801b575d040 ffff8801b575d828 ffff8801fdfd5550
[10733.695818] Call Trace:
[10733.695819] [<ffffffff8107f9a1>] wq_worker_sleeping+0x11/0x70
[10733.695821] [<ffffffff8160efa2>] __schedule+0x5b2/0x7d0
[10733.695822] [<ffffffff8160f1e9>] schedule+0x29/0x70
[10733.695823] [<ffffffff81066ce7>] do_exit+0x6c7/0xa20
[10733.695825] [<ffffffff81612034>] oops_end+0xa4/0xe0
[10733.695827] [<ffffffff810168db>] die+0x4b/0x70
[10733.695828] [<ffffffff81611ada>] do_general_protection+0xda/0x160
[10733.695830] [<ffffffff81611468>] general_protection+0x28/0x30
[10733.695832] [<ffffffff81183889>] ? deactivate_slab+0x119/0x510
[10733.695833] [<ffffffff810154cf>] ? dump_trace+0x16f/0x2b0
[10733.695835] [<ffffffff810ce4d2>] ? cgroup_release_agent+0x42/0x180
[10733.695837] [<ffffffff81020c2b>] ? save_stack_trace+0x2b/0x50
[10733.695838] [<ffffffff81180911>] ? set_track+0x61/0x1b0
[10733.695839] [<ffffffff8118176d>] ? init_object+0x3d/0x70
[10733.695840] [<ffffffff81607756>] __slab_alloc+0x3df/0x4a2
[10733.695842] [<ffffffff8109f5d8>] ? load_balance+0x108/0x7d0
[10733.695843] [<ffffffff810ce4d2>] ? cgroup_release_agent+0x42/0x180
[10733.695844] [<ffffffff8108fa7c>] ? update_rq_clock.part.67+0x1c/0x170
[10733.695846] [<ffffffff81184dd1>] kmem_cache_alloc_trace+0x1b1/0x200
[10733.695847] [<ffffffff810ce4d2>] cgroup_release_agent+0x42/0x180
[10733.695848] [<ffffffff8160eda0>] ? __schedule+0x3b0/0x7d0
[10733.695849] [<ffffffff8107d80f>] process_one_work+0x16f/0x3f0
[10733.695851] [<ffffffff8107f360>] worker_thread+0x110/0x3d0
[10733.695852] [<ffffffff8107f250>] ? __alloc_workqueue_key+0x500/0x500
[10733.695853] [<ffffffff81084240>] kthread+0xc0/0xd0
[10733.695854] [<ffffffff81084180>] ? insert_kthread_work+0x40/0x40
[10733.695855] [<ffffffff816192ac>] ret_from_fork+0x7c/0xb0
[10733.695856] [<ffffffff81084180>] ? insert_kthread_work+0x40/0x40
[10733.695865] Code: 00 48 89 e5 5d 48 8b 40 c8 48 c1 e8 02 83 e0 01 c3 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 48 8b 87 80 05 00 00 55 48 89 e5 <48> 8b 40 d8 5d c3 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90
[10733.695866] RIP [<ffffffff810847e0>] kthread_data+0x10/0x20
[10733.695867] RSP <ffff8801fdfd5968>
[10733.695867] CR2: ffffffffffffffd8
[10733.695868] ---[ end trace aa0156eab3150a01 ]---
[10733.695868] Fixing recursive fault but reboot is needed!
[10741.654996] BUG: unable to handle kernel paging request at 00007f18c4071018
[10741.691831] IP: [<ffffffff8115798b>] vma_interval_tree_remove+0x14b/0x240
[10741.727684] PGD 0
[10741.738338] Oops: 0000 [#6] SMP
[10741.755746] Modules linked in: binfmt_misc(F) bnep(F) hidp(F) tun(F) fuse(F) cmtp(F) kernelcapi(F) l2tp_ppp(F) l2tp_netlink(F) l2tp_core(F) rfcomm(F) nfnetlink(F) scsi_transport_iscsi(F) ipt_ULOG(F) af_802154(F) atm(F) bluetooth(F) rfkill(F) af_key(F) nfc(F) pppoe(F) pppox(F) ppp_generic(F) slhc(F) rds(F) lockd(F) sunrpc(F) nf_conntrack_netbios_ns(F) nf_conntrack_broadcast(F) ipt_MASQUERADE(F) ip6table_nat(F) nf_nat_ipv6(F) ip6table_mangle(F) ip6t_REJECT(F) nf_conntrack_ipv6(F) nf_defrag_ipv6(F) iptable_nat(F) nf_nat_ipv4(F) nf_nat(F) iptable_mangle(F) ipt_REJECT(F) nf_conntrack_ipv4(F) nf_defrag_ipv4(F) xt_conntrack(F) nf_conntrack(F) ebtable_filter(F) ebtables(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) sg(F) ixgbe(F) iTCO_wdt(F) mdio(F) iTCO_vendor_support(F) dca(F) e1000e(F) lpc_ich(F) serio_raw(F) ptp(F) mfd_core(F) hpilo(F) pcspkr(F) hpwdt(F) pps_core(F) microcode(F) xfs(F) libcrc32c(F) sd_mod(F) crc_t10dif(F) ata_generic(F) mgag200(F) i2c_algo_bit(F) pata_acpi(F) drm_kms_helper(F) ttm(F) ata_piix(F) drm(F) hpsa(F) libata(F) i2c_core(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F)
[10742.257869] CPU 3
[10742.267698] Pid: 6911, comm: attr Tainted: GF B D W 3.9.4 #1 HP ProLiant DL120 G7
[10742.311646] RIP: 0010:[<ffffffff8115798b>] [<ffffffff8115798b>] vma_interval_tree_remove+0x14b/0x240
[10742.358495] RSP: 0018:ffff8801bbe15670 EFLAGS: 00010206
[10742.384615] RAX: ffff8801a65cae20 RBX: ffff8801fb849200 RCX: 00007f18c4071000
[10742.420770] RDX: ffffffffffffffff RSI: ffff8801feb63220 RDI: ffff8801fb849200
[10742.456593] RBP: ffff8801bbe15688 R08: ffff8801fba5dc58 R09: ffff8801fb849a58
[10742.494108] R10: ffff8801fb849258 R11: 0000000000000000 R12: ffff8801fb849200
[10742.530422] R13: ffff8801feb63238 R14: ffff8801feb63200 R15: ffff8801fb849200
[10742.565618] FS: 0000000000000000(0000) GS:ffff88020b460000(0000) knlGS:0000000000000000
[10742.606623] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[10742.635672] CR2: 00007f18c4071018 CR3: 00000001e169c000 CR4: 00000000000407e0
[10742.670958] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[10742.706707] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[10742.744778] Process attr (pid: 6911, threadinfo ffff8801bbe14000, task ffff8801e6b71ac0)
[10742.788295] Stack:
[10742.798965] ffffffff811613e6 ffff8801c46e7740 ffff8801fb849200 ffff8801bbe156b8
[10742.839037] ffffffff811619a0 ffff8801fb849a00 00007fc6265f1000 0000000000000000
[10742.876283] ffff8801bbe15710 ffff8801bbe156f8 ffffffff8115a240 0000000000000000
[10742.913418] Call Trace:
[10742.925426] [<ffffffff811613e6>] ? __remove_shared_vm_struct+0x46/0x70
[10742.958588] [<ffffffff811619a0>] unlink_file_vma+0x40/0x60
[10742.985945] [<ffffffff8115a240>] free_pgtables+0xb0/0x120
[10743.013519] [<ffffffff81164722>] exit_mmap+0xb2/0x170
[10743.038820] [<ffffffff8105e377>] mmput+0x67/0xf0
[10743.062990] [<ffffffff81066898>] do_exit+0x278/0xa20
[10743.087864] [<ffffffff81604989>] ? printk+0x67/0x69
[10743.112837] [<ffffffff81063088>] ? console_unlock+0x1e8/0x3f0
[10743.141614] [<ffffffff81612034>] oops_end+0xa4/0xe0
[10743.166575] [<ffffffff81604212>] no_context+0x263/0x270
[10743.192871] [<ffffffff81604292>] __bad_area_nosemaphore+0x73/0x1cc
[10743.224652] [<ffffffff816043fe>] bad_area_nosemaphore+0x13/0x15
[10743.255236] [<ffffffff81614c0e>] __do_page_fault+0x31e/0x510
[10743.286559] [<ffffffffa015d497>] ? kmem_alloc+0x67/0xf0 [xfs]
[10743.317700] [<ffffffff81020c2b>] ? save_stack_trace+0x2b/0x50
[10743.349363] [<ffffffff81180911>] ? set_track+0x61/0x1b0
[10743.378060] [<ffffffff8118176d>] ? init_object+0x3d/0x70
[10743.407085] [<ffffffff81614e0e>] do_page_fault+0xe/0x10
[10743.436241] [<ffffffff81611498>] page_fault+0x28/0x30
[10743.462402] [<ffffffff81302305>] ? memmove+0x55/0x1a0
[10743.487998] [<ffffffffa01651a1>] ? xfs_attr_leaf_moveents.isra.2+0x91/0x280 [xfs]
[10743.526426] [<ffffffffa0165467>] xfs_attr_leaf_compact+0xd7/0x130 [xfs]
[10743.559879] [<ffffffffa0166a2e>] xfs_attr_leaf_add+0xce/0x170 [xfs]
[10743.591996] [<ffffffffa0162850>] xfs_attr_leaf_addname+0xc0/0x3d0 [xfs]
[10743.625676] [<ffffffffa0170d4e>] ? xfs_bmap_one_block+0x3e/0xa0 [xfs]
[10743.658136] [<ffffffffa016378c>] xfs_attr_set_int+0x30c/0x420 [xfs]
[10743.689678] [<ffffffff811be9f4>] ? setxattr+0xa4/0x1c0
[10743.716002] [<ffffffffa0163d1f>] xfs_attr_set+0x7f/0x90 [xfs]
[10743.744715] [<ffffffffa0159a12>] xfs_xattr_set+0x42/0x70 [xfs]
[10743.774512] [<ffffffff811bdef2>] generic_setxattr+0x62/0x80
[10743.802374] [<ffffffff811be743>] __vfs_setxattr_noperm+0x63/0x1b0
[10743.833373] [<ffffffff811be945>] vfs_setxattr+0xb5/0xc0
[10743.859695] [<ffffffff811bea76>] setxattr+0x126/0x1c0
[10743.887070] [<ffffffff8118358d>] ? kmem_cache_free+0x1cd/0x1e0
[10743.919013] [<ffffffff811a72b2>] ? final_putname+0x22/0x50
[10743.948596] [<ffffffff811a74cb>] ? putname+0x2b/0x40
[10743.976310] [<ffffffff811ab96f>] ? user_path_at_empty+0x5f/0x90
[10744.007107] [<ffffffff8119e5c9>] ? __sb_start_write+0x49/0x100
[10744.036839] [<ffffffff811bedef>] sys_lsetxattr+0x8f/0xd0
[10744.063840] [<ffffffff81619359>] system_call_fastpath+0x16/0x1b
[10744.094150] Code: 48 89 50 18 48 8b 00 48 83 e0 fc 74 43 48 8b 50 b0 48 2b 50 a8 48 8b 48 40 48 c1 ea 0c 48 8d 54 11 ff 48 8b 48 10 48 85 c9 74 0b <48> 8b 49 18 48 39 ca 48 0f 42 d1 48 8b 48 08 48 85 c9 74 0b 48
[10744.188945] RIP [<ffffffff8115798b>] vma_interval_tree_remove+0x14b/0x240
[10744.223099] RSP <ffff8801bbe15670>
[10744.240096] CR2: 00007f18c4071018
[10744.257118] ---[ end trace aa0156eab3150a02 ]---
[10744.280164] Fixing recursive fault but reboot is needed!
[10748.689674] general protection fault: 0000 [#7] SMP
[10748.716550] Modules linked in: binfmt_misc(F) bnep(F) hidp(F) tun(F) fuse(F) cmtp(F) kernelcapi(F) l2tp_ppp(F) l2tp_netlink(F) l2tp_core(F) rfcomm(F) nfnetlink(F) scsi_transport_iscsi(F) ipt_ULOG(F) af_802154(F) atm(F) bluetooth(F) rfkill(F) af_key(F) nfc(F) pppoe(F) pppox(F) ppp_generic(F) slhc(F) rds(F) lockd(F) sunrpc(F) nf_conntrack_netbios_ns(F) nf_conntrack_broadcast(F) ipt_MASQUERADE(F) ip6table_nat(F) nf_nat_ipv6(F) ip6table_mangle(F) ip6t_REJECT(F) nf_conntrack_ipv6(F) nf_defrag_ipv6(F) iptable_nat(F) nf_nat_ipv4(F) nf_nat(F) iptable_mangle(F) ipt_REJECT(F) nf_conntrack_ipv4(F) nf_defrag_ipv4(F) xt_conntrack(F) nf_conntrack(F) ebtable_filter(F) ebtables(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) sg(F) ixgbe(F) iTCO_wdt(F) mdio(F) iTCO_vendor_support(F) dca(F) e1000e(F) lpc_ich(F) serio_raw(F) ptp(F) mfd_core(F) hpilo(F) pcspkr(F) hpwdt(F) pps_core(F) microcode(F) xfs(F) libcrc32c(F) sd_mod(F) crc_t10dif(F) ata_generic(F) mgag200(F) i2c_algo_bit(F) pata_acpi(F) drm_kms_helper(F) ttm(F) ata_piix(F) drm(F) hpsa(F) libata(F) i2c_core(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F)
[10749.217082] CPU 2
[10749.227162] Pid: 464, comm: flush-253:1 Tainted: GF B D W 3.9.4 #1 HP ProLiant DL120 G7
[10749.274289] RIP: 0010:[<ffffffff81157609>] [<ffffffff81157609>] vma_interval_tree_subtree_search+0x19/0x80
[10749.326848] RSP: 0000:ffff8801e6937a30 EFLAGS: 00010202
[10749.355371] RAX: 6b6b6b6b6b6b6b6b RBX: ffff8801f74a6fc0 RCX: 0000000000000276
[10749.394358] RDX: 00000000000006af RSI: 00000000000006af RDI: ffff8801a65c8400
[10749.429501] RBP: ffff8801e6937a30 R08: ffff8801e6937b01 R09: 0000000000001000
[10749.465183] R10: 0000000000000100 R11: 0000000000000210 R12: 00000000000006af
[10749.500930] R13: ffff8801e6937cc8 R14: ffffea00047e1340 R15: ffffea00047e1340
[10749.536294] FS: 0000000000000000(0000) GS:ffff88020b440000(0000) knlGS:0000000000000000
[10749.577185] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[10749.605877] CR2: 0000000000000000 CR3: 00000000018fa000 CR4: 00000000000407e0
[10749.641518] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[10749.677263] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[10749.712490] Process flush-253:1 (pid: 464, threadinfo ffff8801e6936000, task ffff8801fcea3580)
[10749.755984] Stack:
[10749.765953] ffff8801e6937a40 ffffffff81157aa2 ffff8801e6937a98 ffffffff8116714e
[10749.803931] ffff8801f74a6ff8 00000100638e3f00 00000000000006b1 00000000000006c0
[10749.843909] ffffea00047e1340 ffff8801f74a6fc0 ffff8801e6937cc8 0000000000000000
[10749.882823] Call Trace:
[10749.895832] [<ffffffff81157aa2>] vma_interval_tree_iter_first+0x22/0x30
[10749.932506] [<ffffffff8116714e>] page_mkclean+0x6e/0x1d0
[10749.960173] [<ffffffff8113dbb8>] clear_page_dirty_for_io+0x48/0xc0
[10749.991604] [<ffffffff8113de4a>] write_cache_pages+0x21a/0x4b0
[10750.020918] [<ffffffff8113d4b0>] ? global_dirtyable_memory+0x50/0x50
[10750.053649] [<ffffffff8113e120>] generic_writepages+0x40/0x60
[10750.083106] [<ffffffffa01449e5>] xfs_vm_writepages+0x35/0x40 [xfs]
[10750.113998] [<ffffffff8113f02e>] do_writepages+0x1e/0x40
[10750.141300] [<ffffffff811c2ec0>] __writeback_single_inode+0x40/0x210
[10750.173209] [<ffffffff811c4b77>] writeback_sb_inodes+0x197/0x400
[10750.204613] [<ffffffff811c4e7f>] __writeback_inodes_wb+0x9f/0xd0
[10750.235080] [<ffffffff811c5543>] wb_writeback+0x233/0x2b0
[10750.263128] [<ffffffff811c6f65>] wb_do_writeback+0x1e5/0x1f0
[10750.291413] [<ffffffff811c6ffb>] bdi_writeback_thread+0x8b/0x210
[10750.321825] [<ffffffff811c6f70>] ? wb_do_writeback+0x1f0/0x1f0
[10750.35 80>] ? insert_kthread_work+0x40/0x40
[10750.409867] [<ffffffff816192ac>] ret_from_fork+0x7c/0xb0
[10750.439257] [<ffffffff81084180>] ? insert_kthread_work+0x40/0x40
[10750.471505] Code: c3 0f 0b 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 66 66 66 66 90 55 48 89 e5 0f 1f 80 00 00 00 00 48 8b 47 68 48 85 c0 74 17 <48> 39 70 18 72 11 48 8d 78 a8 48 8b 47 68 48 85 c0 75 ed 0f 1f
[10750.568513] RIP [<ffffffff81157609>] vma_interval_tree_subtree_search+0x19/0x80
[10750.605629] RSP <ffff8801e6937a30>
[10750.624263] ---[ end trace aa0156eab3150a03 ]---
[10761.391486] general protection fault: 0000 [#8] SMP
[10761.418069] Modules linked in: binfmt_misc(F) bnep(F) hidp(F) tun(F) fuse(F) cmtp(F) kernelcapi(F) l2tp_ppp(F) l2tp_netlink(F) l2tp_core(F) rfcomm(F) nfnetlink(F) scsi_transport_iscsi(F) ipt_ULOG(F) af_802154(F) atm(F) bluetooth(F) rfkill(F) af_key(F) nfc(F) pppoe(F) pppox(F) ppp_generic(F) slhc(F) rds(F) lockd(F) sunrpc(F) nf_conntrack_netbios_ns(F) nf_conntrack_broadcast(F) ipt_MASQUERADE(F) ip6table_nat(F) nf_nat_ipv6(F) ip6table_mangle(F) ip6t_REJECT(F) nf_conntrack_ipv6(F) nf_defrag_ipv6(F) iptable_nat(F) nf_nat_ipv4(F) nf_nat(F) iptable_mangle(F) ipt_REJECT(F) nf_conntrack_ipv4(F) nf_defrag_ipv4(F) xt_conntrack(F) nf_conntrack(F) ebtable_filter(F) ebtables(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) sg(F) ixgbe(F) iTCO_wdt(F) mdio(F) iTCO_vendor_support(F) dca(F) e1000e(F) lpc_ich(F) serio_raw(F) ptp(F) mfd_core(F) hpilo(F) pcspkr(F) hpwdt(F) pps_core(F) microcode(F) xfs(F) libcrc32c(F) sd_mod(F) crc_t10dif(F) ata_generic(F) mgag200(F) i2c_algo_bit(F) pata_acpi(F) drm_kms_helper(F) ttm(F) ata_piix(F) drm(F) hpsa(F) libata(F) i2c_core(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F)
[10761.921495] CPU 2
[10761.930503] Pid: 63, comm: khugepaged Tainted: GF B D W 3.9.4 #1 HP ProLiant DL120 G7
[10761.975282] RIP: 0010:[<ffffffff8118b7fa>] [<ffffffff8118b7fa>] hugepage_vma_check+0xa/0x70
[10762.020574] RSP: 0000:ffff8801fceedd60 EFLAGS: 00010206
[10762.049366] RAX: 0000000000000003 RBX: 0000000002e00000 RCX: ffff8801fceedfd8
[10762.087874] RDX: 0000000000000000 RSI: 0000000002f08000 RDI: 0000be0300000001
[10762.127078] RBP: ffff8801fceedd68 R08: ffff8801e0a15c00 R09: 0000000000000000
[10762.162283] R10: 0000000000000000 R11: ffff8801d9101000 R12: 0000000003000000
[10762.197992] R13: 0000be0300000001 R14: ffff8801d9101000 R15: 0000000000000202
[10762.234176] FS: 0000000000000000(0000) GS:ffff88020b440000(0000) knlGS:0000000000000000
[10762.274068] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[10762.302770] CR2: 0000000000000000 CR3: 00000000018fa000 CR4: 00000000000407e0
[10762.338146] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[10762.373845] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[10762.409883] Process khugepaged (pid: 63, threadinfo ffff8801fceec000, task ffff8801fceb3580)
[10762.452060] Stack:
[10762.462025] 0000000002e00000 ffff8801fceede40 ffffffff8118bb0c ffff8801fceedd90
[10762.498485] ffffffff810712d2 ffff8801ff640000 ffff8801fceede40 ffff8801f9a28a10
[10762.536010] ffff880000000000 ffff8801e56f28e8 ffff8801fceede70 ffffea0007644000
[10762.575870] Call Trace:
[10762.589029] [<ffffffff8118bb0c>] khugepaged_scan_mm_slot+0xcc/0xd80
[10762.623447] [<ffffffff810712d2>] ? del_timer_sync+0x52/0x60
[10762.654387] [<ffffffff8118c8e3>] khugepaged+0x123/0x600
[10762.682370] [<ffffffff810850e0>] ? wake_up_bit+0x30/0x30
[10762.709898] [<ffffffff8118c7c0>] ? khugepaged_scan_mm_slot+0xd80/0xd80
[10762.742505] [<ffffffff81084240>] kthread+0xc0/0xd0
[10762.767903] [<ffffffff81084180>] ? insert_kthread_work+0x40/0x40
[10762.799013] [<ffffffff816192ac>] ret_from_fork+0x7c/0xb0
[10762.826476] [<ffffffff81084180>] ? insert_kthread_work+0x40/0x40
[10762.856488] Code: 89 c4 e8 2a 25 48 00 49 63 c4 45 85 e4 48 0f 45 d8 48 89 d8 5b 41 5c 5d c3 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 53 <48> 8b 47 50 48 89 fb a9 00 00 00 20 75 09 f6 05 b1 04 87 00 01
[10762.951263] RIP [<ffffffff8118b7fa>] hugepage_vma_check+0xa/0x70
[10762.982007] RSP <ffff8801fceedd60>
[10762.999155] ---[ end trace aa0156eab3150a04 ]---
[-- MARK -- Thu May 30 07:45:00 2013]
[10779.411271] hpsa 0000:04:00.0: Abort request on C0:B0:T0:L0
[10783.500306] INFO: rcu_sched detected stalls on CPUs/tasks: {} (detected by 3, t=60031 jiffies, g=677612, c=677611, q=125)
[10783.559039] INFO: Stall ended before state dump start
[10801.881812] Kernel panic - not syncing: Watchdog detected hard LOCKUP on cpu 1
[10802.953682] Shutting down cpus with NMI
[10802.974055] drm_kms_helper: panic occurred, switching back to text console
----- Original Message -----
> From: "CAI Qian" <[email protected]>
> To: "Dave Chinner" <[email protected]>
> Cc: [email protected], [email protected]
> Sent: Thursday, May 30, 2013 1:40:32 PM
> Subject: Re: 3.9.3: Oops running xfstests
>
>
>
> ----- Original Message -----
> > From: "Dave Chinner" <[email protected]>
> > To: "CAI Qian" <[email protected]>
> > Cc: [email protected], [email protected]
> > Sent: Thursday, May 30, 2013 1:20:49 PM
> > Subject: Re: 3.9.3: Oops running xfstests
> >
> > On Wed, May 29, 2013 at 11:45:28PM -0400, CAI Qian wrote:
> > > Also reproduced this on 3.9.4 on several different machines.
> >
> > How? what workload are you running that causes it to occur?
> I have been running the following workloads, trinity, ltp (including
> oom tests and some filesystem tests), some hugetlbfs tests, selinux
> test suite, tracepoint tests, connectathon tests and then xfstests.
> I am still trying to narrow down the minimal workload that could
> reproduce this.
> >
> > <snip stack trace showing page overrun in
> > xfs_attr_leaf_compact/xfs_attr_leaf_moveents>
> >
> > > System information is below:
> > >
> > > + cat /proc/mounts
> > ....
> > > /dev/mapper/rhel_hp--xw9400--02-root / xfs
> > > rw,seclabel,relatime,attr2,inode64,noquota 0 0
> > > /dev/mapper/rhel_hp--xw9400--02-home /home xfs
> > > rw,seclabel,relatime,attr2,inode64,noquota 0 0
> > > /dev/sda1 /boot xfs rw,seclabel,relatime,attr2,inode64,noquota 0 0
> >
> > Which of these filesystems did it happen on? Or some other
> > filesystem not mounted when you took this snapshot?
> + export TEST_DEV=/dev/mapper/rhel_hp--bl495cg5--02-swap
> + TEST_DEV=/dev/mapper/rhel_hp--bl495cg5--02-swap
> + export TEST_DIR=/mnt/testarea/test
> + TEST_DIR=/mnt/testarea/test
> + export SCRATCH_DEV=/dev/mapper/rhel_hp--bl495cg5--02-home
> + SCRATCH_DEV=/dev/mapper/rhel_hp--bl495cg5--02-home
> + export SCRATCH_MNT=/mnt/testarea/scratch
> + SCRATCH_MNT=/mnt/testarea/scratch
> + mkdir -p /mnt/testarea/test
> home = /dev/mapper/rhel_hp--bl495cg5--02-home
> + mkdir -p /mnt/testarea/scratch
> + mkfs.xfs -f /dev/mapper/rhel_hp--bl495cg5--02-swap
> meta-data=/dev/mapper/rhel_hp--bl495cg5--02-swap isize=256 agcount=4,
> agsize=510976 blks
> = sectsz=512 attr=2, projid32bit=0
> data = bsize=4096 blocks=2043904, imaxpct=25
> = sunit=0 swidth=0 blks
> naming =version 2 bsize=4096 ascii-ci=0
> log =internal log bsize=4096 blocks=2560, version=2
> = sectsz=512 sunit=0 blks, lazy-count=1
> realtime =none extsz=4096 blocks=0, rtextents=0
> + mkfs.xfs -f /dev/mapper/rhel_hp--bl495cg5--02-home
> meta-data=/dev/mapper/rhel_hp--bl495cg5--02-home isize=256 agcount=4,
> agsize=1103872 blks
> = sectsz=512 attr=2, projid32bit=0
> data = bsize=4096 blocks=4415488, imaxpct=25
> = sunit=0 swidth=0 blks
> naming =version 2 bsize=4096 ascii-ci=0
> log =internal log bsize=4096 blocks=2560, version=2
> = sectsz=512 sunit=0 blks, lazy-count=1
> realtime =none extsz=4096 blocks=0, rtextents=0
> >
> > And can you post the xfs_info output of the filesystems under test,
> > too?
> xfs_info: /dev/mapper/rhel_hp--bl495cg5--02-swap is not a mounted XFS
> filesystem
> + echo
>
> + xfs_info /dev/mapper/rhel_hp--bl495cg5--02-home
> meta-data=/dev/mapper/rhel_hp--bl495cg5--02-home isize=256 agcount=4,
> agsize=1103872 blks
> = sectsz=512 attr=2
> data = bsize=4096 blocks=4415488, imaxpct=25
> = sunit=0 swidth=0 blks
> naming =version 2 bsize=4096 ascii-ci=0
> log =internal bsize=4096 blocks=2560, version=2
> = sectsz=512 sunit=0 blks, lazy-count=1
> realtime =none extsz=4096 blocks=0, rtextents=0
> CAI Qian
> >
> > Cheers,
> >
> > Dave.
> > --
> > Dave Chinner
> > [email protected]
> >
>
On 5/30/13 10:03 PM, CAI Qian wrote:
> OK, so the minimal workload to trigger this I found so far was to
> run trinity, ltp and then xfstests. I have been able to easily
> reproduced on 3 servers so far, and I'll post full logs here for
> LKML and linux-mm as this may unrelated to XFS only. As far as
> I can tell from the previous testing results, this has never been
> reproduced back in 3.9 GA time. This seems also been reproduced
> on 3.10-rc3 mostly on s390x so far.
> CAI Qian
>
Can you hit it w/o trinity? I ask because trinity's stated
goal is to fuzz and corrupt, right - so it's quite possible
that blowing up later in xfs is a side effect?
-Eric
On Thu, May 30, 2013 at 11:03:35PM -0400, CAI Qian wrote:
> OK, so the minimal workload to trigger this I found so far was to
> run trinity, ltp and then xfstests. I have been able to easily
> reproduced on 3 servers so far, and I'll post full logs here for
> LKML and linux-mm as this may unrelated to XFS only. As far as
> I can tell from the previous testing results, this has never been
> reproduced back in 3.9 GA time. This seems also been reproduced
> on 3.10-rc3 mostly on s390x so far.
> CAI Qian
>
> 1)
> [10948.104344] XFS (dm-2): Mounting Filesystem
> [10948.257335] XFS (dm-2): Ending clean mount
> [10949.753951] XFS (dm-0): Mounting Filesystem
> [10950.012824] XFS (dm-0): Ending clean mount
> [10951.115722] BUG: unable to handle kernel paging request at ffff88003a9ca478
> [10951.122727] IP: [<ffffffff810f7f47>] __call_rcu.constprop.47+0x77/0x230
Note the timestamp there -I'll come back to it.
This has blown up in rcu processing on a idle CPU....
> [10951.527453] BUG: unable to handle kernel paging request at ffff88003db52000
> [10951.527459] IP: [<ffffffff81302301>] memmove+0x51/0x1a0
.. aproximately 400ms before XFS has tripped over in
xfs_attr_leaf_compact().
And for the second crash:
> 2)
> [20914.575680] XFS (dm-2): Mounting Filesystem
> [20914.653966] XFS (dm-2): Ending clean mount
> [20915.812263] XFS (dm-0): Mounting Filesystem
> [20916.628209] XFS (dm-0): Ending clean mount
> [20917.997069] divide error: 0000 [#1] SMP
A divide by zero error in the scheduler find_busiest_group()
function during CPU idle work, with no XFS traces at all.
> 3)
> [10721.161888] XFS (dm-0): Ending clean mount
> [10722.090855] XFS (dm-2): Mounting Filesystem
> [10722.143682] XFS (dm-2): Ending clean mount
> [10722.584327] XFS (dm-0): Mounting Filesystem
> [10722.949519] XFS (dm-0): Ending clean mount
> [10723.579704] BUG: unable to handle kernel paging request at ffff8801b55cc000
> [10723.619185] IP: [<ffffffff81302305>] memmove+0x55/0x1a0
XFS trips over, almost simultaneously with:
> [10724.128760] WARNING: at mm/mmap.c:2711 exit_mmap+0x166/0x170()
A warning in exit_mmap() in a singal handler..
> [10724.128901] BUG: Bad rss-counter state mm:ffff8801fb90af40 idx:0 val:5
> [10724.128901] BUG: Bad rss-counter state mm:ffff8801fb90af40 idx:1 val:3
A bunch of obviously bad mm state warnings, and
> [10724.128927] rhts-test-runne[48621]: segfault at 19cc ip 00000000004401c4 sp 00007fff27f840e0 error 6 in bash[400000+db000]
> [10724.129726] =============================================================================
> [10724.129727] BUG kmalloc-4096 (Tainted: GF W ): Padding overwritten. 0xffff88019152fc28-0xffff88019152ffff
> [10724.129727] -----------------------------------------------------------------------------
> [10724.129727]
> [10724.129728] INFO: Slab 0xffffea0006454a00 objects=7 used=7 fp=0x (null) flags=0x20000000004080
> [10724.129729] Pid: 43729, comm: kworker/0:0 Tainted: GF B W 3.9.4 #1
> [10724.129729] Call Trace:
> [10724.129730] [<ffffffff81181ed2>] slab_err+0xc2/0xf0
> [10724.129733] [<ffffffff812f8007>] ? kset_init+0x27/0x40
> [10724.129735] [<ffffffff81181ff5>] slab_pad_check.part.41+0xf5/0x170
> [10724.129736] [<ffffffff810ce4d2>] ? cgroup_release_agent+0x42/0x180
> [10724.129739] [<ffffffff811820e3>] check_slab+0x73/0x100
> [10724.129740] [<ffffffff81606b50>] alloc_debug_processing+0x21/0x118
> [10724.129742] [<ffffffff8160772f>] __slab_alloc+0x3b8/0x4a2
> [10724.129744] [<ffffffff8109f5d8>] ? load_balance+0x108/0x7d0
> [10724.129746] [<ffffffff810ce4d2>] ? cgroup_release_agent+0x42/0x180
> [10724.129747] [<ffffffff8108fa7c>] ? update_rq_clock.part.67+0x1c/0x170
> [10724.129749] [<ffffffff81184dd1>] kmem_cache_alloc_trace+0x1b1/0x200
> [10724.129751] [<ffffffff810ce4d2>] cgroup_release_agent+0x42/0x180
cgroup slab memory poison overwrite warning.
> [10724.135254] systemd[1]: segfault at 45 ip 00007f93e348c780 sp 00007fff4deb3a9e error 4 in libdbus-1.so.3.7.2[7f93e3479000+45000]
> [10724.135272] traps: systemd[1] trap invalid opcode ip:7f93e498c490 sp:7fff4deb34f8 error:0 in systemd[7f93e496d000+f6000]
> [10724.135281] traps: systemd[1] trap invalid opcode ip:7f93e498c490 sp:7fff4deb2f38 error:0 in systemd[7f93e496d000+f6000]
> [10724.135289] traps: systemd[1] trap invalid opcode ip:7f93e498c490 sp:7fff4deb2978 error:0 in systemd[7f93e496d000+f6000]
> [10724.135297] traps: systemd[1] trap invalid opcode ip:7f93e498c490 sp:7fff4deb23b8 error:0 in systemd[7f93e496d000+f6000]
> [10724.135304] traps: systemd[1] trap invalid opcode ip:7f93e498c490 sp:7fff4deb1df8 error:0 in systemd[7f93e496d000+f6000]
> [10724.135312] traps: systemd[1] trap invalid opcode ip:7f93e498c490 sp:7fff4deb1838 error:0 in systemd[7f93e496d000+f6000]
A bunch of nasty systemd trap warnings for illegal operations.
FWIW:
> [10748.689674] general protection fault: 0000 [#7] SMP
> [10749.217082] CPU 2
> [10749.227162] Pid: 464, comm: flush-253:1 Tainted: GF B D W 3.9.4 #1 HP ProLiant DL120 G7
...
> [10749.882823] Call Trace:
> [10749.895832] [<ffffffff81157aa2>] vma_interval_tree_iter_first+0x22/0x30
> [10749.932506] [<ffffffff8116714e>] page_mkclean+0x6e/0x1d0
> [10749.960173] [<ffffffff8113dbb8>] clear_page_dirty_for_io+0x48/0xc0
> [10749.991604] [<ffffffff8113de4a>] write_cache_pages+0x21a/0x4b0
> [10750.020918] [<ffffffff8113d4b0>] ? global_dirtyable_memory+0x50/0x50
> [10750.053649] [<ffffffff8113e120>] generic_writepages+0x40/0x60
> [10750.083106] [<ffffffffa01449e5>] xfs_vm_writepages+0x35/0x40 [xfs]
> [10750.113998] [<ffffffff8113f02e>] do_writepages+0x1e/0x40
> [10750.141300] [<ffffffff811c2ec0>] __writeback_single_inode+0x40/0x210
> [10750.173209] [<ffffffff811c4b77>] writeback_sb_inodes+0x197/0x400
> [10750.204613] [<ffffffff811c4e7f>] __writeback_inodes_wb+0x9f/0xd0
> [10750.235080] [<ffffffff811c5543>] wb_writeback+0x233/0x2b0
> [10750.263128] [<ffffffff811c6f65>] wb_do_writeback+0x1e5/0x1f0
> [10750.291413] [<ffffffff811c6ffb>] bdi_writeback_thread+0x8b/0x210
> [10750.321825] [<ffffffff811c6f70>] ? wb_do_writeback+0x1f0/0x1f0
There's a clear indication that VM system is completely screwed up -
The flusher thread crashed trying to clear the dirty page bit during
data writeback.
There's memory corruption all over the place. It is most likely
that trinity is causing this - it's purpose is to trigger corruption
issues, but they aren't always immediately seen. If you can trigger
this xfs trace without trinity having been run and without all the
RCU/idle/scheduler/cgroup issues occuring at the same time, then
it's likely to be caused by XFS. But right now, I'd say XFS is just
an innocent bystander caught in the crossfire. There's nothing I can
do from an XFS persepctive to track this down...
Cheers,
Dave.
--
Dave Chinner
[email protected]
> There's memory corruption all over the place. It is most likely
> that trinity is causing this - it's purpose is to trigger corruption
> issues, but they aren't always immediately seen. If you can trigger
> this xfs trace without trinity having been run and without all the
> RCU/idle/scheduler/cgroup issues occuring at the same time, then
> it's likely to be caused by XFS. But right now, I'd say XFS is just
> an innocent bystander caught in the crossfire. There's nothing I can
> do from an XFS persepctive to track this down...
OK, this can be reproduced by just running LTP and then xfstests without
trinity at all...
[ 302.311213] XFS (dm-0): Mounting Filesystem
[ 302.608320] XFS (dm-0): Ending clean mount
[ 303.625760] XFS (dm-2): Mounting Filesystem
[ 303.674648] XFS (dm-2): Ending clean mount
[ 304.247740] XFS (dm-0): Mounting Filesystem
[ 304.563899] XFS (dm-0): Ending clean mount
[ 305.118268] BUG: unable to handle kernel paging request at ffff8801f7067000
[ 305.156637] IP: [<ffffffff813022fa>] memmove+0x4a/0x1a0
[ 305.185560] PGD 1ddf067 PUD 20bdf9067 PMD 1f9b56063 PTE 80000001f7067161
[ 305.222852] Oops: 0003 [#1] SMP
[ 305.238742] Modules linked in: lockd(F) sunrpc(F) nf_conntrack_netbios_ns(F) nf_conntrack_broadcast(F) ipt_MASQUERADE(F) ip6table_nat(F) nf_nat_ipv6(F) ip6table_mangle(F) ip6t_REJECT(F) nf_conntrack_ipv6(F) nf_defrag_ipv6(F) iptable_nat(F) nf_nat_ipv4(F) nf_nat(F) iptable_mangle(F) ipt_REJECT(F) nf_conntrack_ipv4(F) nf_defrag_ipv4(F) xt_conntrack(F) nf_conntrack(F) ebtable_filter(F) ebtables(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) sg(F) iTCO_wdt(F) e1000e(F) ixgbe(F) iTCO_vendor_support(F) ptp(F) mdio(F) dca(F) serio_raw(F) hpwdt(F) pcspkr(F) pps_core(F) hpilo(F) lpc_ich(F) mfd_core(F) microcode(F) xfs(F) libcrc32c(F) sd_mod(F) mgag200(F) ata_generic(F) crc_t10dif(F) i2c_algo_bit(F) pata_acpi(F) drm_kms_helper(F) ttm(F) ata_piix(F) drm(F) hpsa(F) libata(F) i2c_core(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F)
[ 305.615528] CPU 3
[ 305.624597] Pid: 19138, comm: attr Tainted: GF 3.9.4 #1 HP ProLiant DL120 G7
[ 305.667290] RIP: 0010:[<ffffffff813022fa>] [<ffffffff813022fa>] memmove+0x4a/0x1a0
[ 305.707799] RSP: 0018:ffff8801da2a7ad8 EFLAGS: 00010282
[ 305.736413] RAX: ffff8801f7034540 RBX: ffff8801d3bba000 RCX: 000000000000d00b
[ 305.774576] RDX: fffffffffffcd4f8 RSI: ffff8801f70673f8 RDI: ffff8801f7067000
[ 305.811143] RBP: ffff8801da2a7b30 R08: 3ee6e8057eed854d R09: 9066aaebe17ba23c
[ 305.847038] R10: 0d8b3a7401f8834f R11: 74c08510ffdf89f6 R12: ffff8801f7034520
[ 305.883189] R13: 00000000000003d8 R14: 0000000000001800 R15: 000000000000007b
[ 305.921160] FS: 00007fbdabf4e740(0000) GS:ffff88020b460000(0000) knlGS:0000000000000000
[ 305.961370] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 305.990403] CR2: ffff8801f7067000 CR3: 00000001d3d90000 CR4: 00000000000407e0
[ 306.025585] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 306.061610] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[ 306.097768] Process attr (pid: 19138, threadinfo ffff8801da2a6000, task ffff8801dfdb3580)
[ 306.138073] Stack:
[ 306.148523] ffffffffa01691a1 0000000000000000 0000000000000000 ffff8801f7034540
[ 306.187280] ffff8801f7034520 0000000000000018 ffff8801f7034520 0000000000000000
[ 306.227244] ffff8801e2425cd0 ffff8801d207e880 ffff8801d3e6ad20 ffff8801da2a7b68
[ 306.267164] Call Trace:
[ 306.280421] [<ffffffffa01691a1>] ? xfs_attr_leaf_moveents.isra.2+0x91/0x280 [xfs]
[ 306.321077] [<ffffffffa0169467>] xfs_attr_leaf_compact+0xd7/0x130 [xfs]
[ 306.355110] [<ffffffffa016aa2e>] xfs_attr_leaf_add+0xce/0x170 [xfs]
[ 306.386559] [<ffffffffa0166850>] xfs_attr_leaf_addname+0xc0/0x3d0 [xfs]
[ 306.420459] [<ffffffffa0174d4e>] ? xfs_bmap_one_block+0x3e/0xa0 [xfs]
[ 306.453034] [<ffffffffa016778c>] xfs_attr_set_int+0x30c/0x420 [xfs]
[ 306.484498] [<ffffffff811be9f4>] ? setxattr+0xa4/0x1c0
[ 306.511064] [<ffffffffa0167d1f>] xfs_attr_set+0x7f/0x90 [xfs]
[ 306.539972] [<ffffffffa015da12>] xfs_xattr_set+0x42/0x70 [xfs]
[ 306.570403] [<ffffffff811bdef2>] generic_setxattr+0x62/0x80
[ 306.598333] [<ffffffff811be743>] __vfs_setxattr_noperm+0x63/0x1b0
[ 306.629324] [<ffffffff811be945>] vfs_setxattr+0xb5/0xc0
[ 306.655470] [<ffffffff811bea76>] setxattr+0x126/0x1c0
[ 306.681365] [<ffffffff8118358d>] ? kmem_cache_free+0x1cd/0x1e0
[ 306.710684] [<ffffffff811a72b2>] ? final_putname+0x22/0x50
[ 306.740728] [<ffffffff811a74cb>] ? putname+0x2b/0x40
[ 306.767873] [<ffffffff811ab96f>] ? user_path_at_empty+0x5f/0x90
[ 306.799781] [<ffffffff8119e5c9>] ? __sb_start_write+0x49/0x100
[ 306.831920] [<ffffffff811bedef>] sys_lsetxattr+0x8f/0xd0
[ 306.861316] [<ffffffff81619359>] system_call_fastpath+0x16/0x1b
[ 306.894531] Code: 00 00 48 81 fa a8 02 00 00 72 05 40 38 fe 74 41 48 83 ea 20 48 83 ea 20 4c 8b 1e 4c 8b 56 08 4c 8b 4e 10 4c 8b 46 18 48 8d 76 20 <4c> 89 1f 4c 89 57 08 4c 89 4f 10 4c 89 47 18 48 8d 7f 20 73 d4
[ 306.989301] RIP [<ffffffff813022fa>] memmove+0x4a/0x1a0
[ 307.016021] RSP <ffff8801da2a7ad8>
[ 307.033212] CR2: ffff8801f7067000
[ 307.049503] ---[ end trace 94a32a2cfce117d8 ]---
[ 307.073111] BUG: unable to handle kernel paging request at 0000428200000008
[ 307.107376] IP: [<ffffffff810c8b00>] acct_collect+0x60/0x1b0
[ 307.135985] PGD 0
[ 307.145833] Oops: 0000 [#2] SMP
[ 307.161695] Modules linked in: lockd(F) sunrpc(F) nf_conntrack_netbios_ns(F) nf_conntrack_broadcast(F) ipt_MASQUERADE(F) ip6table_nat(F) nf_nat_ipv6(F) ip6table_mangle(F) ip6t_REJECT(F) nf_conntrack_ipv6(F) nf_defrag_ipv6(F) iptable_nat(F) nf_nat_ipv4(F) nf_nat(F) iptable_mangle(F) ipt_REJECT(F) nf_conntrack_ipv4(F) nf_defrag_ipv4(F) xt_conntrack(F) nf_conntrack(F) ebtable_filter(F) ebtables(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) sg(F) iTCO_wdt(F) e1000e(F) ixgbe(F) iTCO_vendor_support(F) ptp(F) mdio(F) dca(F) serio_raw(F) hpwdt(F) pcspkr(F) pps_core(F) hpilo(F) lpc_ich(F) mfd_core(F) microcode(F) xfs(F) libcrc32c(F) sd_mod(F) mgag200(F) ata_generic(F) crc_t10dif(F) i2c_algo_bit(F) pata_acpi(F) drm_kms_helper(F) ttm(F) ata_piix(F) drm(F) hpsa(F) libata(F) i2c_core(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F)
[ 307.547533] CPU 3
[ 307.556537] Pid: 18958, comm: sleep Tainted: GF D 3.9.4 #1 HP ProLiant DL120 G7
[ 307.598058] RIP: 0010:[<ffffffff810c8b00>] [<ffffffff810c8b00>] acct_collect+0x60/0x1b0
[ 307.638608] RSP: 0018:ffff8801fd1e3e90 EFLAGS: 00010206
[ 307.664796] RAX: 0000428200000000 RBX: ffff8801ff0a5d80 RCX: ffff8801fd1e3fd8
[ 307.700780] RDX: 0000000006936000 RSI: 0000000000000001 RDI: ffff8801d3aef328
[ 307.735921] RBP: ffff8801fd1e3eb0 R08: ffff8801dff5d040 R09: 0000000000000000
[ 307.771698] R10: ffffffffffffff80 R11: 0000000000000246 R12: ffff8801dff5d040
[ 307.808645] R13: 0000000000000000 R14: 0000000000000000 R15: ffff8801dff5d040
[ 307.847144] FS: 00007f3a74a7b740(0000) GS:ffff88020b460000(0000) knlGS:0000000000000000
[ 307.891651] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 307.922666] CR2: 0000428200000008 CR3: 00000001d21f7000 CR4: 00000000000407e0
[ 307.961110] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 307.999904] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[ 308.035206] Process sleep (pid: 18958, threadinfo ffff8801fd1e2000, task ffff8801dff5d040)
[ 308.076654] Stack:
[ 308.086497] 0000000000000000 0000000000004a0e ffff8801dff5d040 00007fffa372b608
[ 308.123532] ffff8801fd1e3f38 ffffffff81066e01 0000000100000000 0000000000000000
[ 308.160362] 00000046b1319ffe 00000046b130dcae ffffffff81087820 ffff88020b46e700
[ 308.197286] Call Trace:
[ 308.209307] [<ffffffff81066e01>] do_exit+0x7e1/0xa20
[ 308.234897] [<ffffffff81087820>] ? hrtimer_get_res+0x50/0x50
[ 308.263462] [<ffffffff8160e341>] ? do_nanosleep+0x61/0xe0
[ 308.291687] [<ffffffff810670bf>] do_group_exit+0x3f/0xa0
[ 308.318371] [<ffffffff81067134>] sys_exit_group+0x14/0x20
[ 308.346294] [<ffffffff81619359>] system_call_fastpath+0x16/0x1b
[ 308.376928] Code: 00 00 00 74 56 49 8b 84 24 80 04 00 00 48 8d 78 68 e8 25 5b 54 00 49 8b 84 24 80 04 00 00 48 8b 00 48 85 c0 74 1c 31 d2 0f 1f 00 <48> 03 50 08 48 2b 10 48 8b 40 10 48 85 c0 75 f0 48 c1 ea 0a 49
[ 308.480424] RIP [<ffffffff810c8b00>] acct_collect+0x60/0x1b0
[ 308.509509] RSP <ffff8801fd1e3e90>
[ 308.527362] CR2: 0000428200000008
[ 308.543697] general protection fault: 0000 [#3] [ 308.543760] ---[ end trace 94a32a2cfce117d9 ]---
[ 308.543761] Fixing recursive fault but reboot is needed!
SMP
[ 308.618050] Modules linked in: lockd(F) sunrpc(F) nf_conntrack_netbios_ns(F) nf_conntrack_broadcast(F) ipt_MASQUERADE(F) ip6table_nat(F) nf_nat_ipv6(F) ip6table_mangle(F) ip6t_REJECT(F) nf_conntrack_ipv6(F) nf_defrag_ipv6(F) iptable_nat(F) nf_nat_ipv4(F) nf_nat(F) iptable_mangle(F) ipt_REJECT(F) nf_conntrack_ipv4(F) nf_defrag_ipv4(F) xt_conntrack(F) nf_conntrack(F) ebtable_filter(F) ebtables(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) sg(F) iTCO_wdt(F) e1000e(F) ixgbe(F) iTCO_vendor_support(F) ptp(F) mdio(F) dca(F) serio_raw(F) hpwdt(F) pcspkr(F) pps_core(F) hpilo(F) lpc_ich(F) mfd_core(F) microcode(F) xfs(F) libcrc32c(F) sd_mod(F) mgag200(F) ata_generic(F) crc_t10dif(F) i2c_algo_bit(F) pata_acpi(F) drm_kms_helper(F) ttm(F) ata_piix(F) drm(F) hpsa(F) libata(F) i2c_core(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F)
[ 309.002756] CPU 0
[ 309.012338] Pid: 19138, comm: attr Tainted: GF D 3.9.4 #1 HP ProLiant DL120 G7
[ 309.056022] RIP: 0010:[<ffffffff8115798b>] [<ffffffff8115798b>] vma_interval_tree_remove+0x14b/0x240
[ 309.101815] RSP: 0018:ffff8801da2a7670 EFLAGS: 00010202
[ 309.128508] RAX: ffff8801f703d058 RBX: ffff8801fb9f0e00 RCX: 6b6b6b6b6b6b6b6b
[ 309.163784] RDX: 6b6b6b6b6b6b6b6a RSI: ffff8801febdb220 RDI: ffff8801fb9f0e00
[ 309.199476] RBP: ffff8801da2a7688 R08: 0000000000000000 R09: 0000000000017700
[ 309.235490] R10: ffff8801fb9f0e58 R11: 000000000000000e R12: ffff8801fb9f0e00
[ 309.270754] R13: ffff8801febdb238 R14: ffff8801febdb200 R15: ffff8801fb9f2a00
[ 309.307101] FS: 00007fbdabf4e740(0000) GS:ffff88020b400000(0000) knlGS:0000000000000000
[ 309.348288] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 309.376730] CR2: 0000000002077840 CR3: 00000001d3d90000 CR4: 00000000000407f0
[ 309.413051] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 309.449081] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[ 309.487646] Process attr (pid: 19138, threadinfo ffff8801da2a6000, task ffff8801dfdb3580)
[ 309.532131] Stack:
[ 309.543173] ffffffff811613e6 ffff8801d21718c0 ffff8801fb9f0e00 ffff8801da2a76b8
[ 309.582822] ffffffff811619a0 ffff8801fb9f0e00 00007fbdab776000 0000000000000000
[ 309.621914] ffff8801da2a7710 ffff8801da2a76f8 ffffffff8115a1d8 0000000000000000
[ 309.659092] Call Trace:
[ 309.671075] [<ffffffff811613e6>] ? __remove_shared_vm_struct+0x46/0x70
[ 309.704907] [<ffffffff811619a0>] unlink_file_vma+0x40/0x60
[ 309.732536] [<ffffffff8115a1d8>] free_pgtables+0x48/0x120
[ 309.760022] [<ffffffff81164722>] exit_mmap+0xb2/0x170
[ 309.785585] [<ffffffff8160eda5>] ? __schedule+0x3b5/0x7d0
[ 309.813041] [<ffffffff8105e377>] mmput+0x67/0xf0
[ 309.837444] [<ffffffff81066898>] do_exit+0x278/0xa20
[ 309.862503] [<ffffffff81604989>] ? printk+0x67/0x69
[ 309.888016] [<ffffffff81063088>] ? console_unlock+0x1e8/0x3f0
[ 309.916848] [<ffffffff81612034>] oops_end+0xa4/0xe0
[ 309.942105] [<ffffffff81604212>] no_context+0x263/0x270
[ 309.968280] [<ffffffff81604292>] __bad_area_nosemaphore+0x73/0x1cc
[ 310.000888] [<ffffffff816043fe>] bad_area_nosemaphore+0x13/0x15
[ 310.033080] [<ffffffff81614c0e>] __do_page_fault+0x31e/0x510
[ 310.064304] [<ffffffffa0161497>] ? kmem_alloc+0x67/0xf0 [xfs]
[ 310.095564] [<ffffffff81020c2b>] ? save_stack_trace+0x2b/0x50
[ 310.125723] [<ffffffff81180911>] ? set_track+0x61/0x1b0
[ 310.153458] [<ffffffff8118176d>] ? init_object+0x3d/0x70
[ 310.182482] [<ffffffff81614e0e>] do_page_fault+0xe/0x10
[ 310.209285] [<ffffffff81611498>] page_fault+0x28/0x30
[ 310.234674] [<ffffffff813022fa>] ? memmove+0x4a/0x1a0
[ 310.260888] [<ffffffffa01691a1>] ? xfs_attr_leaf_moveents.isra.2+0x91/0x280 [xfs]
[ 310.298854] [<ffffffffa0169467>] xfs_attr_leaf_compact+0xd7/0x130 [xfs]
[ 310.332735] [<ffffffffa016aa2e>] xfs_attr_leaf_add+0xce/0x170 [xfs]
[ 310.364814] [<ffffffffa0166850>] xfs_attr_leaf_addname+0xc0/0x3d0 [xfs]
[ 310.398089] [<ffffffffa0174d4e>] ? xfs_bmap_one_block+0x3e/0xa0 [xfs]
[ 310.431514] [<ffffffffa016778c>] xfs_attr_set_int+0x30c/0x420 [xfs]
[ 310.462923] [<ffffffff811be9f4>] ? setxattr+0xa4/0x1c0
[ 310.489423] [<ffffffffa0167d1f>] xfs_attr_set+0x7f/0x90 [xfs]
[ 310.518114] [<ffffffffa015da12>] xfs_xattr_set+0x42/0x70 [xfs]
[ 310.548571] [<ffffffff811bdef2>] generic_setxattr+0x62/0x80
[ 310.579571] [<ffffffff811be743>] __vfs_setxattr_noperm+0x63/0x1b0
[ 310.613506] [<ffffffff811be945>] vfs_setxattr+0xb5/0xc0
[ 310.640943] [<ffffffff811bea76>] setxattr+0x126/0x1c0
[ 310.668401] [<ffffffff8118358d>] ? kmem_cache_free+0x1cd/0x1e0
[ 310.700284] [<ffffffff811a72b2>] ? final_putname+0x22/0x50
[ 310.727708] [<ffffffff811a74cb>] ? putname+0x2b/0x40
[ 310.753288] [<ffffffff811ab96f>] ? user_path_at_empty+0x5f/0x90
[ 310.782891] [<ffffffff8119e5c9>] ? __sb_start_write+0x49/0x100
[ 310.812647] [<ffffffff811bedef>] sys_lsetxattr+0x8f/0xd0
[ 310.839684] [<ffffffff81619359>] system_call_fastpath+0x16/0x1b
[ 310.870818] Code: 48 89 50 18 48 8b 00 48 83 e0 fc 74 43 48 8b 50 b0 48 2b 50 a8 48 8b 48 40 48 c1 ea 0c 48 8d 54 11 ff 48 8b 48 10 48 85 c9 74 0b <48> 8b 49 18 48 39 ca 48 0f 42 d1 48 8b 48 08 48 85 c9 74 0b 48
[ 310.966164] RIP [<ffffffff8115798b>] vma_interval_tree_remove+0x14b/0x240
[ 311.000221] RSP <ffff8801da2a7670>
[ 311.017819] BUG: unable to handle kernel [ 311.017973] ---[ end trace 94a32a2cfce117da ]---
[ 311.017974] Fixing recursive fault but reboot is needed!
paging request at 00007f7f3985cfe8
[ 311.105330] IP: [<ffffffff81160ff3>] find_vma+0x43/0xa0
[ 311.133081] PGD 0
[ 311.144699] Oops: 0000 [#4] SMP
[ 311.161948] Modules linked in: lockd(F) sunrpc(F) nf_conntrack_netbios_ns(F) nf_conntrack_broadcast(F) ipt_MASQUERADE(F) ip6table_nat(F) nf_nat_ipv6(F) ip6table_mangle(F) ip6t_REJECT(F) nf_conntrack_ipv6(F) nf_defrag_ipv6(F) iptable_nat(F) nf_nat_ipv4(F) nf_nat(F) iptable_mangle(F) ipt_REJECT(F) nf_conntrack_ipv4(F) nf_defrag_ipv4(F) xt_conntrack(F) nf_conntrack(F) ebtable_filter(F) ebtables(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) sg(F) iTCO_wdt(F) e1000e(F) ixgbe(F) iTCO_vendor_support(F) ptp(F) mdio(F) dca(F) serio_raw(F) hpwdt(F) pcspkr(F) pps_core(F) hpilo(F) lpc_ich(F) mfd_core(F) microcode(F) xfs(F) libcrc32c(F) sd_mod(F) mgag200(F) ata_generic(F) crc_t10dif(F) i2c_algo_bit(F) pata_acpi(F) drm_kms_helper(F) ttm(F) ata_piix(F) drm(F) hpsa(F) libata(F) i2c_core(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F)
[ 311.542611] CPU 1
[ 311.552199] Pid: 17036, comm: systemd-udevd Tainted: GF D 3.9.4 #1 HP ProLiant DL120 G7
[ 311.596657] RIP: 0010:[<ffffffff81160ff3>] [<ffffffff81160ff3>] find_vma+0x43/0xa0
[ 311.636364] RSP: 0000:ffff8801d16ede48 EFLAGS: 00010206
[ 311.665124] RAX: 0000000000000000 RBX: 0000000000000014 RCX: ffff8801d16edfd8
[ 311.703530] RDX: 00007f7f3985d000 RSI: 00007fb5f3715a90 RDI: ffff8801d3aee540
[ 311.741784] RBP: ffff8801d16edf38 R08: 0000000000000001 R09: 00007fb5f28317d8
[ 311.777051] R10: 00007fb5f28317d8 R11: 0000000000000246 R12: 0000000000000028
[ 311.813094] R13: 00007fb5f3715a90 R14: ffff8801d16edf58 R15: ffff8801d3aee540
[ 311.848908] FS: 00007fb5f36e0880(0000) GS:ffff88020b420000(0000) knlGS:0000000000000000
[ 311.890296] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 311.919321] CR2: 00007f7f3985cfe8 CR3: 00000001fbef8000 CR4: 00000000000407e0
[ 311.954562] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 311.990646] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[ 312.026415] Process systemd-udevd (pid: 17036, threadinfo ffff8801d16ec000, task ffff8801f70d8000)
[ 312.071166] Stack:
[ 312.081624] ffffffff816149f3 ffff8801d12620dc ffff8801f70d8000 ffff8801d3aee5a8
[ 312.119621] ffff8801d16ede90 ffffffff811b9a9e ffff8801d16418c0 0000000000000008
[ 312.159495] ffff8801feb0d340 ffff8801d16edea0 ffffffff811b9ba6 ffff8801d16edee8
[ 312.199195] Call Trace:
[ 312.212470] [<ffffffff816149f3>] ? __do_page_fault+0x103/0x510
[ 312.244953] [<ffffffff811b9a9e>] ? mntput_no_expire+0x3e/0x120
[ 312.277378] [<ffffffff811b9ba6>] ? mntput+0x26/0x40
[ 312.302467] [<ffffffff8119d9e3>] ? __fput+0x183/0x270
[ 312.327922] [<ffffffff8119db8e>] ? ____fput+0xe/0x10
[ 312.353480] [<ffffffff81080f2c>] ? task_work_run+0xac/0xe0
[ 312.381167] [<ffffffff81614e0e>] do_page_fault+0xe/0x10
[ 312.408212] [<ffffffff81611498>] page_fault+0x28/0x30
[ 312.433289] Code: 47 48 8b 57 08 31 c0 48 85 d2 75 1f eb 44 66 2e 0f 1f 84 00 00 00 00 00 48 3b 72 e0 48 8d 42 e0 73 1d 48 8b 52 10 48 85 d2 74 0f <48> 3b 72 e8 72 e7 48 8b 52 08 48 85 d2 75 f1 48 85 c0 74 10 48
[ 312.528198] RIP [<ffffffff81160ff3>] find_vma+0x43/0xa0
[ 312.555203] RSP <ffff8801d16ede48>
[ 312.572391] CR2: 00007f7f3985cfe8
[ 312.588724] ---[ end trace 94a32a2cfce117db ]---
[ 312.612043] BUG: unable to handle kernel paging request at 0000428200000008
[ 312.646416] IP: [<ffffffff810c8b00>] acct_collect+0x60/0x1b0
[ 312.675123] PGD 0
[ 312.685662] Oops: 0000 [#5] SMP
[ 312.702907] Modules linked in: lockd(F) sunrpc(F) nf_conntrack_netbios_ns(F) nf_conntrack_broadcast(F) ipt_MASQUERADE(F) ip6table_nat(F) nf_nat_ipv6(F) ip6table_mangle(F) ip6t_REJECT(F) nf_conntrack_ipv6(F) nf_defrag_ipv6(F) iptable_nat(F) nf_nat_ipv4(F) nf_nat(F) iptable_mangle(F) ipt_REJECT(F) nf_conntrack_ipv4(F) nf_defrag_ipv4(F) xt_conntrack(F) nf_conntrack(F) ebtable_filter(F) ebtables(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) sg(F) iTCO_wdt(F) e1000e(F) ixgbe(F) iTCO_vendor_support(F) ptp(F) mdio(F) dca(F) serio_raw(F) hpwdt(F) pcspkr(F) pps_core(F) hpilo(F) lpc_ich(F) mfd_core(F) microcode(F) xfs(F) libcrc32c(F) sd_mod(F) mgag200(F) ata_generic(F) crc_t10dif(F) i2c_algo_bit(F) pata_acpi(F) drm_kms_helper(F) ttm(F) ata_piix(F) drm(F) hpsa(F) libata(F) i2c_core(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F)
[ 313.090194] CPU 2
[ 313.099192] Pid: 17036, comm: systemd-udevd Tainted: GF D 3.9.4 #1 HP ProLiant DL120 G7
[ 313.143933] RIP: 0010:[<ffffffff810c8b00>] [<ffffffff810c8b00>] acct_collect+0x60/0x1b0
[ 313.183945] RSP: 0018:ffff8801d16edaf8 EFLAGS: 00010206
[ 313.210509] RAX: 0000428200000000 RBX: ffff8801dfefac00 RCX: ffff8801d16edfd8
[ 313.246545] RDX: 00000000029b7000 RSI: 0000000000000001 RDI: ffff8801d3aee5a8
[ 313.285230] RBP: ffff8801d16edb18 R08: ffff8801f70d8000 R09: 0000000000000000
[ 313.323961] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8801f70d8000
[ 313.362768] R13: 0000000000000009 R14: 0000000000000000 R15: ffff8801f70d8000
[ 313.402688] FS: 00007fb5f36e0880(0000) GS:ffff88020b440000(0000) knlGS:0000000000000000
[ 313.445042] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 313.474147] CR2: 0000428200000008 CR3: 00000001fbef8000 CR4: 00000000000407e0
[ 313.510209] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 313.545396] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[ 313.581255] Process systemd-udevd (pid: 17036, threadinfo ffff8801d16ec000, task ffff8801f70d8000)
[ 313.626232] Stack:
[ 313.636139] 0000000000000009 000000000000428c 0000000000000246 0000000000000000
[ 313.672927] ffff8801d16edba0 ffffffff81066e01 00000001d16edb80 ffffffff81604989
[ 313.709663] 0000000000000010 ffff8801d16edb90 ffff8801d16edb50 ffffffff819f6f88
[ 313.746821] Call Trace:
[ 313.759043] [<ffffffff81066e01>] do_exit+0x7e1/0xa20
[ 313.784649] [<ffffffff81604989>] ? printk+0x67/0x69
[ 313.809568] [<ffffffff81612034>] oops_end+0xa4/0xe0
[ 313.834180] [<ffffffff81604212>] no_context+0x263/0x270
[ 313.863369] [<ffffffff81604292>] __bad_area_nosemaphore+0x73/0x1cc
[ 313.896335] [<ffffffff8160460f>] bad_area+0x43/0x4a
[ 313.924110] [<ffffffff81614d4c>] __do_page_fault+0x45c/0x510
[ 313.955197] [<ffffffff81020c2b>] ? save_stack_trace+0x2b/0x50
[ 313.986579] [<ffffffff81180911>] ? set_track+0x61/0x1b0
[ 314.012728] [<ffffffff8118176d>] ? init_object+0x3d/0x70
[ 314.039782] [<ffffffff81606e25>] ? free_debug_processing+0x1de/0x222
[ 314.071858] [<ffffffff81614e0e>] do_page_fault+0xe/0x10
[ 314.098809] [<ffffffff81611498>] page_fault+0x28/0x30
[ 314.125269] [<ffffffff81160ff3>] ? find_vma+0x43/0xa0
[ 314.152148] [<ffffffff816149f3>] ? __do_page_fault+0x103/0x510
[ 314.181402] [<ffffffff811b9a9e>] ? mntput_no_expire+0x3e/0x120
[ 314.211469] [<ffffffff811b9ba6>] ? mntput+0x26/0x40
[ 314.236258] [<ffffffff8119d9e3>] ? __fput+0x183/0x270
[ 314.262654] [<ffffffff8119db8e>] ? ____fput+0xe/0x10
[ 314.287521] [<ffffffff81080f2c>] ? task_work_run+0xac/0xe0
[ 314.315854] [<ffffffff81614e0e>] do_page_fault+0xe/0x10
[ 314.342280] [<ffffffff81611498>] page_fault+0x28/0x30
[ 314.368412] Code: 00 00 00 74 56 49 8b 84 24 80 04 00 00 48 8d 78 68 e8 25 5b 54 00 49 8b 84 24 80 04 00 00 48 8b 00 48 85 c0 74 1c 31 d2 0f 1f 00 <48> 03 50 08 48 2b 10 48 8b 40 10 48 85 c0 75 f0 48 c1 ea 0a 49
[ 314.472334] RIP [<ffffffff810c8b00>] acct_collect+0x60/0x1b0
[ 314.502845] RSP <ffff8801d16edaf8>
[ 314.520964] CR2: 0000428200000008
[ 314.537469] ---[ end trace 94a32a2cfce117dc ]---
[ 314.560276] Fixing recursive fault but reboot is needed!
[ 314.598307] =============================================================================
[ 314.639471] BUG kmalloc-4096 (Tainted: GF D ): Padding overwritten. 0xffff8801f7037c28-0xffff8801f7037fff
[ 314.691639] -----------------------------------------------------------------------------
[ 314.691639]
[ 314.739818] INFO: Slab 0xffffea0007dc0c00 objects=7 used=4 fp=0xffff8801f7031148 flags=0x20000000004081
[ 314.786319] Pid: 302, comm: kworker/0:1H Tainted: GF B D 3.9.4 #1
[ 314.820301] Call Trace:
[ 314.832314] [<ffffffff81181ed2>] slab_err+0xc2/0xf0
[ 314.857597] [<ffffffff810c0007>] ? on_each_cpu_mask+0x47/0x60
[ 314.886444] [<ffffffff810154cf>] ? dump_trace+0x16f/0x2b0
[ 314.915497] [<ffffffff81181ff5>] slab_pad_check.part.41+0xf5/0x170
[ 314.949702] [<ffffffff811820e3>] check_slab+0x73/0x100
[ 314.978505] [<ffffffff81180007>] ? page_referenced_ksm+0x77/0x1a0
[ 315.012162] [<ffffffff81606c9e>] free_debug_processing+0x57/0x222
[ 315.045042] [<ffffffffa0161595>] ? kmem_free+0x35/0x40 [xfs]
[ 315.076862] [<ffffffffa0161595>] ? kmem_free+0x35/0x40 [xfs]
[ 315.105209] [<ffffffff81606ea3>] __slab_free+0x3a/0x318
[ 315.132235] [<ffffffffa0161595>] ? kmem_free+0x35/0x40 [xfs]
[ 315.160550] [<ffffffffa014e532>] ? xfs_extent_busy_clear_one+0x42/0xd0 [xfs]
[ 315.196283] [<ffffffffa0161595>] ? kmem_free+0x35/0x40 [xfs]
[ 315.224568] [<ffffffff8118373f>] kfree+0xff/0x130
[ 315.248848] [<ffffffffa0161595>] kmem_free+0x35/0x40 [xfs]
[ 315.276362] [<ffffffffa01a0adf>] xlog_cil_committed+0x9f/0xe0 [xfs]
[ 315.308507] [<ffffffffa019def0>] xlog_state_do_callback+0x170/0x2c0 [xfs]
[ 315.342483] [<ffffffffa019e0ae>] xlog_state_done_syncing+0x6e/0x90 [xfs]
[ 315.376540] [<ffffffffa019e11b>] xlog_iodone+0x4b/0xa0 [xfs]
[ 315.405695] [<ffffffffa014b866>] xfs_buf_iodone_work+0x46/0xd0 [xfs]
[ 315.437442] [<ffffffff8107d80f>] process_one_work+0x16f/0x3f0
[ 315.467401] [<ffffffff8107f360>] worker_thread+0x110/0x3d0
[ 315.497398] [<ffffffff8107f250>] ? __alloc_workqueue_key+0x500/0x500
[ 315.532523] [<ffffffff81084240>] kthread+0xc0/0xd0
[ 315.559039] [<ffffffff81084180>] ? insert_kthread_work+0x40/0x40
[ 315.592030] [<ffffffff816192ac>] ret_from_fork+0x7c/0xb0
[ 315.621438] [<ffffffff81084180>] ? insert_kthread_work+0x40/0x40
[ 315.652053] Padding ffff8801f70378f8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 315.699076] Padding ffff8801f7037908: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 315.745979] Padding ffff8801f7037918: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 315.792851] Padding ffff8801f7037928: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 315.840050] Padding ffff8801f7037938: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 315.887178] Padding ffff8801f7037948: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 315.934795] Padding ffff8801f7037958: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 315.982248] Padding ffff8801f7037968: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 316.030041] Padding ffff8801f7037978: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 316.080829] Padding ffff8801f7037988: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 316.131932] Padding ffff8801f7037998: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 316.183194] Padding ffff8801f70379a8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 316.233916] Padding ffff8801f70379b8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 316.281215] Padding ffff8801f70379c8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 316.328966] Padding ffff8801f70379d8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 316.376138] Padding ffff8801f70379e8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 316.422472] Padding ffff8801f70379f8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 316.469480] Padding ffff8801f7037a08: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 316.516904] Padding ffff8801f7037a18: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 316.564165] Padding ffff8801f7037a28: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 316.612961] Padding ffff8801f7037a38: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 316.664073] Padding ffff8801f7037a48: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 316.715212] Padding ffff8801f7037a58: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 316.764149] Padding ffff8801f7037a68: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 316.811182] Padding ffff8801f7037a78: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 316.858494] Padding ffff8801f7037a88: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 316.906132] Padding ffff8801f7037a98: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 316.954730] Padding ffff8801f7037aa8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 317.001916] Padding ffff8801f7037ab8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 317.049322] Padding ffff8801f7037ac8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 317.097121] Padding ffff8801f7037ad8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 317.145051] Padding ffff8801f7037ae8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 317.196764] Padding ffff8801f7037af8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 317.247078] Padding ffff8801f7037b08: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 317.297532] Padding ffff8801f7037b18: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 317.345002] Padding ffff8801f7037b28: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 317.392045] Padding ffff8801f7037b38: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 317.439071] Padding ffff8801f7037b48: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 317.485932] Padding ffff8801f7037b58: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 317.532876] Padding ffff8801f7037b68: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 317.580171] Padding ffff8801f7037b78: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 317.627661] Padding ffff8801f7037b88: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 317.674527] Padding ffff8801f7037b98: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 317.725010] Padding ffff8801f7037ba8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 317.775448] Padding ffff8801f7037bb8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 317.826466] Padding ffff8801f7037bc8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 317.873530] Padding ffff8801f7037bd8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 317.920442] Padding ffff8801f7037be8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 317.968769] Padding ffff8801f7037bf8: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 318.015333] Padding ffff8801f7037c08: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 318.062740] Padding ffff8801f7037c18: 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
[ 318.109415] Padding ffff8801f7037c28: e0 bd 03 f7 01 88 ff ff ff ff ff ff ff ff ff ff ................
[ 318.156333] Padding ffff8801f7037c38: ff ff ff ff ff ff ff ff 80 01 00 00 00 00 00 00 ................
[ 318.203365] Padding ffff8801f7037c48: cc cc cc cc cc cc cc cc 68 81 03 f7 01 88 ff ff ........h.......
[ 318.252859] Padding ffff8801f7037c58: 27 1e 2a 81 ff ff ff ff 2f 77 60 81 ff ff ff ff '.*...../w`.....
[ 318.303185] Padding ffff8801f7037c68: d1 4d 18 81 ff ff ff ff 27 1e 2a 81 ff ff ff ff .M......'.*.....
[ 318.354505] Padding ffff8801f7037c78: 55 2c 2a 81 ff ff ff ff 0d 30 2a 81 ff ff ff ff U,*......0*.....
[ 318.404617] Padding ffff8801f7037c88: 77 bf 2a 81 ff ff ff ff a3 07 2a 81 ff ff ff ff w.*.......*.....
[ 318.451210] Padding ffff8801f7037c98: 08 ec 29 81 ff ff ff ff df c6 19 81 ff ff ff ff ..).............
[ 318.497880] Padding ffff8801f7037ca8: d9 c9 19 81 ff ff ff ff 59 93 61 81 ff ff ff ff ........Y.a.....
[ 318.544873] Padding ffff8801f7037cb8: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 318.592075] Padding ffff8801f7037cc8: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 318.639232] Padding ffff8801f7037cd8: 00 00 00 00 00 00 00 00 00 00 00 00 d7 21 00 00 .............!..
[ 318.686150] Padding ffff8801f7037ce8: 14 b3 fc ff 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 318.733295] Padding ffff8801f7037cf8: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 318.780243] Padding ffff8801f7037d08: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 318.829838] Padding ffff8801f7037d18: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 318.881128] Padding ffff8801f7037d28: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 318.931308] Padding ffff8801f7037d38: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 318.979077] Padding ffff8801f7037d48: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 319.026194] Padding ffff8801f7037d58: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 319.073630] Padding ffff8801f7037d68: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 319.120712] Padding ffff8801f7037d78: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 319.167629] Padding ffff8801f7037d88: 5a 5a 5a 5a 5a 5a 5a 5a 78 bc 03 f7 01 88 ff ff ZZZZZZZZx.......
[ 319.214535] Padding ffff8801f7037d98: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ................
[ 319.261416] Padding ffff8801f7037da8: 80 02 00 00 00 00 00 00 cc cc cc cc cc cc cc cc ................
[ 319.307982] Padding ffff8801f7037db8: d0 82 03 f7 01 88 ff ff 27 1e 2a 81 ff ff ff ff ........'.*.....
[ 319.358389] Padding ffff8801f7037dc8: 2f 77 60 81 ff ff ff ff d1 4d 18 81 ff ff ff ff /w`......M......
[ 319.409888] Padding ffff8801f7037dd8: 27 1e 2a 81 ff ff ff ff 55 2c 2a 81 ff ff ff ff '.*.....U,*.....
[ 319.460846] Padding ffff8801f7037de8: 0d 30 2a 81 ff ff ff ff 77 bf 2a 81 ff ff ff ff .0*.....w.*.....
[ 319.508176] Padding ffff8801f7037df8: a3 07 2a 81 ff ff ff ff 08 ec 29 81 ff ff ff ff ..*.......).....
[ 319.555362] Padding ffff8801f7037e08: df c6 19 81 ff ff ff ff d9 c9 19 81 ff ff ff ff ................
[ 319.602255] Padding ffff8801f7037e18: 59 93 61 81 ff ff ff ff 00 00 00 00 00 00 00 00 Y.a.............
[ 319.649541] Padding ffff8801f7037e28: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 319.697595] Padding ffff8801f7037e38: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 319.744772] Padding ffff8801f7037e48: 00 00 00 00 d7 21 00 00 14 b3 fc ff 00 00 00 00 .....!..........
[ 319.791652] Padding ffff8801f7037e58: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 319.838095] Padding ffff8801f7037e68: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 319.888739] Padding ffff8801f7037e78: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 319.939962] Padding ffff8801f7037e88: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 319.991256] Padding ffff8801f7037e98: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 320.039515] Padding ffff8801f7037ea8: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 320.086594] Padding ffff8801f7037eb8: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 320.133629] Padding ffff8801f7037ec8: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 320.180672] Padding ffff8801f7037ed8: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 320.227713] Padding ffff8801f7037ee8: 00 00 00 00 00 00 00 00 5a 5a 5a 5a 5a 5a 5a 5a ........ZZZZZZZZ
[ 320.274136] Padding ffff8801f7037ef8: 00 00 00 00 00 00 00 00 ff ff ff ff ff ff ff ff ................
[ 320.321932] Padding ffff8801f7037f08: ff ff ff ff ff ff ff ff 80 03 00 00 00 00 00 00 ................
[ 320.370826] Padding ffff8801f7037f18: cc cc cc cc cc cc cc cc 38 84 03 f7 01 88 ff ff ........8.......
[ 320.420767] Padding ffff8801f7037f28: 27 1e 2a 81 ff ff ff ff 2f 77 60 81 ff ff ff ff '.*...../w`.....
[ 320.471678] Padding ffff8801f7037f38: d1 4d 18 81 ff ff ff ff 27 1e 2a 81 ff ff ff ff .M......'.*.....
[ 320.522943] Padding ffff8801f7037f48: 55 2c 2a 81 ff ff ff ff 0d 30 2a 81 ff ff ff ff U,*......0*.....
[ 320.571368] Padding ffff8801f7037f58: 77 bf 2a 81 ff ff ff ff a3 07 2a 81 ff ff ff ff w.*.......*.....
[ 320.618584] Padding ffff8801f7037f68: 08 ec 29 81 ff ff ff ff df c6 19 81 ff ff ff ff ..).............
[ 320.665595] Padding ffff8801f7037f78: d9 c9 19 81 ff ff ff ff 59 93 61 81 ff ff ff ff ........Y.a.....
[ 320.713267] Padding ffff8801f7037f88: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 320.759666] Padding ffff8801f7037f98: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 320.806540] Padding ffff8801f7037fa8: 00 00 00 00 00 00 00 00 00 00 00 00 d7 21 00 00 .............!..
[ 320.854175] Padding ffff8801f7037fb8: 14 b3 fc ff 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 320.901359] Padding ffff8801f7037fc8: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 320.950409] Padding ffff8801f7037fd8: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 321.001549] Padding ffff8801f7037fe8: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 321.052274] Padding ffff8801f7037ff8: 00 00 00 00 00 00 00 00 ........
[ 321.099932] FIX kmalloc-4096: Restoring 0xffff8801f70378f8-0xffff8801f7037fff=0x5a
[ 321.099932]
[ 343.457847] general protection fault: 0000 [#6] SMP
[ 343.485571] Modules linked in: lockd(F) sunrpc(F) nf_conntrack_netbios_ns(F) nf_conntrack_broadcast(F) ipt_MASQUERADE(F) ip6table_nat(F) nf_nat_ipv6(F) ip6table_mangle(F) ip6t_REJECT(F) nf_conntrack_ipv6(F) nf_defrag_ipv6(F) iptable_nat(F) nf_nat_ipv4(F) nf_nat(F) iptable_mangle(F) ipt_REJECT(F) nf_conntrack_ipv4(F) nf_defrag_ipv4(F) xt_conntrack(F) nf_conntrack(F) ebtable_filter(F) ebtables(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) sg(F) iTCO_wdt(F) e1000e(F) ixgbe(F) iTCO_vendor_support(F) ptp(F) mdio(F) dca(F) serio_raw(F) hpwdt(F) pcspkr(F) pps_core(F) hpilo(F) lpc_ich(F) mfd_core(F) microcode(F) xfs(F) libcrc32c(F) sd_mod(F) mgag200(F) ata_generic(F) crc_t10dif(F) i2c_algo_bit(F) pata_acpi(F) drm_kms_helper(F) ttm(F) ata_piix(F) drm(F) hpsa(F) libata(F) i2c_core(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F)
[ 343.867974] CPU 3
[ 343.876992] Pid: 1119, comm: master Tainted: GF B D 3.9.4 #1 HP ProLiant DL120 G7
[ 343.918848] RIP: 0010:[<ffffffff811e1cb5>] [<ffffffff811e1cb5>] sys_epoll_ctl+0x135/0x7c0
[ 343.960461] RSP: 0018:ffff8801e2f65ee8 EFLAGS: 00010206
[ 343.986626] RAX: ffff8801e262b580 RBX: ffff8801dff6b840 RCX: ffff8801e2f65fd8
[ 344.025028] RDX: 0000000000000000 RSI: ffffffff8196ff30 RDI: ffff8801e21ed350
[ 344.063326] RBP: ffff8801e2f65f78 R08: 5a5a5a5a5a5a5a5a R09: 0101010101010101
[ 344.103131] R10: 00007fffc305ba30 R11: 0000000000000297 R12: 0000000000000001
[ 344.140479] R13: 0000000000000015 R14: ffff8801d2177bc0 R15: ffff8801e21ed348
[ 344.175959] FS: 00007facb165d840(0000) GS:ffff88020b460000(0000) knlGS:0000000000000000
[ 344.217871] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 344.246145] CR2: 00007fbb90d88e70 CR3: 00000001e62c3000 CR4: 00000000000407e0
[ 344.282166] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 344.317848] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[ 344.353523] Process master (pid: 1119, threadinfo ffff8801e2f64000, task ffff8801e262b580)
[ 344.395286] Stack:
[ 344.405276] 0000000000000001 0000000000000002 ffff8801e18a0f30 00007facb1696821
[ 344.442554] ffff8801e21ed350 00000001e3390790 ffff8801e21ed348 00000001e3390790
[ 344.479873] 0000000100000015 ffff8801e3390790 ffff8801e3390790 ffff8801e2f65f78
[ 344.516638] Call Trace:
[ 344.529475] [<ffffffff810e0ef6>] ? __audit_syscall_exit+0x1f6/0x2a0
[ 344.564393] [<ffffffff81619359>] system_call_fastpath+0x16/0x1b
[ 344.597127] Code: f8 01 0f 86 45 01 00 00 4c 8b 7c 24 30 4c 89 f8 48 83 c0 08 48 89 c7 48 89 44 24 20 e8 75 c0 42 00 4d 8b 47 68 90 4d 85 c0 74 1d <4d> 3b 70 30 77 0e 72 63 44 89 e8 41 2b 40 38 83 f8 00 7e 4d 4d
[ 344.700053] RIP [<ffffffff811e1cb5>] sys_epoll_ctl+0x135/0x7c0
[ 344.729172] RSP <ffff8801e2f65ee8>
[ 344.747188] ---[ end trace 94a32a2cfce117dd ]---
[ 373.803073] Kernel panic - not syncing: Watchdog detected hard LOCKUP on cpu 1
[ 373.844067] drm_kms_helper: panic occurred, switching back to text console
[ 373.882178] ------------[ cut here ]------------
[ 373.908207] WARNING: at arch/x86/kernel/smp.c:123 native_smp_send_reschedule+0x5a/0x60()
[ 373.948563] Hardware name: ProLiant DL120 G7
[ 373.969969] Modules linked in: lockd(F) sunrpc(F) nf_conntrack_netbios_ns(F) nf_conntrack_broadcast(F) ipt_MASQUERADE(F) ip6table_nat(F) nf_nat_ipv6(F) ip6table_mangle(F) ip6t_REJECT(F) nf_conntrack_ipv6(F) nf_defrag_ipv6(F) iptable_nat(F) nf_nat_ipv4(F) nf_nat(F) iptable_mangle(F) ipt_REJECT(F) nf_conntrack_ipv4(F) nf_defrag_ipv4(F) xt_conntrack(F) nf_conntrack(F) ebtable_filter(F) ebtables(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) sg(F) iTCO_wdt(F) e1000e(F) ixgbe(F) iTCO_vendor_support(F) ptp(F) mdio(F) dca(F) serio_raw(F) hpwdt(F) pcspkr(F) pps_core(F) hpilo(F) lpc_ich(F) mfd_core(F) microcode(F) xfs(F) libcrc32c(F) sd_mod(F) mgag200(F) ata_generic(F) crc_t10dif(F) i2c_algo_bit(F) pata_acpi(F) drm_kms_helper(F) ttm(F) ata_piix(F) drm(F) hpsa(F) libata(F) i2c_core(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F)
[ 374.346510] Pid: 8673, comm: ntpd Tainted: GF B D 3.9.4 #1
[ 374.379076] Call Trace:
[ 374.392291] <IRQ> [<ffffffff81060f50>] warn_slowpath_common+0x70/0xa0
[ 374.428962] [<ffffffff8106103a>] warn_slowpath_null+0x1a/0x20
[ 374.460338] [<ffffffff8103697a>] native_smp_send_reschedule+0x5a/0x60
[ 374.496389] [<ffffffff810a0545>] trigger_load_balance+0x175/0x210
[ 374.527311] [<ffffffff810943af>] scheduler_tick+0xff/0x140
[ 374.555719] [<ffffffff81071f46>] update_process_times+0x66/0x80
[ 374.585366] [<ffffffff810ba1c5>] tick_sched_handle.isra.13+0x25/0x60
[ 374.617683] [<ffffffff810ba241>] tick_sched_timer+0x41/0x60
[ 374.645722] [<ffffffff81087af4>] __run_hrtimer+0x74/0x1d0
[ 374.673706] [<ffffffff810ba200>] ? tick_sched_handle.isra.13+0x60/0x60
[ 374.707178] [<ffffffff810882d7>] hrtimer_interrupt+0xe7/0x220
[ 374.736837] [<ffffffff8161b0e9>] smp_apic_timer_interrupt+0x69/0x9c
[ 374.768883] [<ffffffff81619fdd>] apic_timer_interrupt+0x6d/0x80
[ 374.799400] <EOI> <NMI> [<ffffffff816048a8>] ? panic+0x193/0x1d7
[ 374.831195] [<ffffffff81604812>] ? panic+0xfd/0x1d7
[ 374.855966] [<ffffffff810f0120>] ? watchdog_enable+0x1e0/0x1e0
[ 374.886109] [<ffffffff810f01e2>] watchdog_overflow_callback+0xc2/0xd0
[ 374.921097] [<ffffffff8112a89e>] __perf_event_overflow+0x8e/0x220
[ 374.955486] [<ffffffff81129729>] ? perf_event_update_userpage+0x19/0x100
[ 374.991471] [<ffffffff8112b484>] perf_event_overflow+0x14/0x20
[ 375.024211] [<ffffffff81028c92>] intel_pmu_handle_irq+0x1b2/0x370
[ 375.057297] [<ffffffff816129ed>] perf_event_nmi_handler+0x1d/0x20
[ 375.089114] [<ffffffff816121b9>] nmi_handle.isra.0+0x59/0x90
[ 375.117439] [<ffffffff816122c0>] do_nmi+0xd0/0x310
[ 375.142167] [<ffffffff81611801>] end_repeat_nmi+0x1e/0x2e
[ 375.169249] [<ffffffff81610de0>] ? _raw_spin_lock_irqsave+0x30/0x40
[ 375.201372] [<ffffffff81610de0>] ? _raw_spin_lock_irqsave+0x30/0x40
[ 375.232709] [<ffffffff81610de0>] ? _raw_spin_lock_irqsave+0x30/0x40
[ 375.264788] <<EOE>> [<ffffffff81084f39>] remove_wait_queue+0x19/0x40
[ 375.298191] [<ffffffff811ae9c1>] poll_freewait+0x71/0xa0
[ 375.324866] [<ffffffff811af15a>] do_select+0x5ca/0x650
[ 375.351625] [<ffffffff811ae760>] ? poll_schedule_timeout+0x70/0x70
[ 375.382475] [<ffffffff811ae760>] ? poll_schedule_timeout+0x70/0x70
[ 375.414047] [<ffffffff811ae760>] ? poll_schedule_timeout+0x70/0x70
[ 375.445344] [<ffffffff811ae760>] ? poll_schedule_timeout+0x70/0x70
[ 375.479068] [<ffffffff811ae760>] ? poll_schedule_timeout+0x70/0x70
[ 375.512964] [<ffffffff811ae760>] ? poll_schedule_timeout+0x70/0x70
[ 375.547448] [<ffffffff811ae760>] ? poll_schedule_timeout+0x70/0x70
[ 375.580977] [<ffffffff811ae760>] ? poll_schedule_timeout+0x70/0x70
[ 375.613662] [<ffffffff811ae760>] ? poll_schedule_timeout+0x70/0x70
[ 375.644540] [<ffffffff811af3d7>] core_sys_select+0x1f7/0x310
[ 375.673468] [<ffffffff814ec7f6>] ? sys_sendto+0x126/0x1c0
[ 375.700596] [<ffffffff8101a309>] ? read_tsc+0x9/0x20
[ 375.726347] [<ffffffff810b2a28>] ? ktime_get_ts+0x48/0xe0
[ 375.753522] [<ffffffff8101a309>] ? read_tsc+0x9/0x20
[ 375.779227] [<ffffffff810b2a28>] ? ktime_get_ts+0x48/0xe0
[ 375.806136] [<ffffffff811af59e>] sys_select+0xae/0x100
[ 375.833004] [<ffffffff81619359>] system_call_fastpath+0x16/0x1b
[ 375.863012] ---[ end trace 94a32a2cfce117de ]-
or this,
[ 7263.377379] XFS (dm-2): Mounting Filesystem
[ 7263.491740] XFS (dm-2): Ending clean mount
[ 7265.014356] XFS (dm-0): Mounting Filesystem
[ 7265.671630] XFS (dm-0): Ending clean mount
[ 7266.738912] BUG: unable to handle kernel paging request at ffff8800764e0000
[ 7266.745904] IP: [<ffffffff813022fa>] memmove+0x4a/0x1a0
[ 7266.751141] PGD 1ddf067 PUD 1de2067 PMD 76520063 PTE 80000000764e0161
[ 7266.757637] Oops: 0003 [#1] SMP
[ 7266.760892] Modules linked in: binfmt_misc(F) snd_seq_dummy(F) tun(F) lockd(F) sunrpc(F) nf_conntrack_netbios_ns(F) nf_conntrack_broadcast(F) ipt_MASQUERADE(F) ip6table_nat(F) nf_nat_ipv6(F) ip6table_mangle(F) ip6t_REJECT(F) nf_conntrack_ipv6(F) nf_defrag_ipv6(F) iptable_nat(F) nf_nat_ipv4(F) nf_nat(F) iptable_mangle(F) ipt_REJECT(F) nf_conntrack_ipv4(F) nf_defrag_ipv4(F) xt_conntrack(F) nf_conntrack(F) ebtable_filter(F) ebtables(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) sg(F) snd_hda_codec_hdmi(F) snd_hda_codec_realtek(F) snd_hda_intel(F) snd_hda_codec(F) snd_hwdep(F) snd_seq(F) e1000e(F) snd_seq_device(F) snd_pcm(F) hp_wmi(F) iTCO_wdt(F) sparse_keymap(F) iTCO_vendor_support(F) snd_page_alloc(F) rfkill(F) ptp(F) snd_timer(F) mei(F) lpc_ich(F) pps_core(F) snd(F) pcspkr(F) i2c_i801(F) mfd_core(F) microcode(F) soundcore(F) xfs(F) libcrc32c(F) sr_mod(F) sd_mod(F) cdrom(F) crc_t10dif(F) nouveau(F) video(F) mxm_wmi(F) i2c_algo_bit(F) drm_kms_helper(F) ata_generic(F) ttm(F) ahci(F) pata_acpi(F) drm(F) libahci(F) i2c_core(F) wmi(F) libata(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F)
[ 7266.860252] CPU 3
[ 7266.862092] Pid: 24364, comm: attr Tainted: GF 3.9.4 #1 Hewlett-Packard HP Z210 Workstation/1587h
[ 7266.872091] RIP: 0010:[<ffffffff813022fa>] [<ffffffff813022fa>] memmove+0x4a/0x1a0
[ 7266.879756] RSP: 0018:ffff880063573ad8 EFLAGS: 00010282
[ 7266.885057] RAX: ffff88007649c540 RBX: ffff880057665000 RCX: 000000000000d00b
[ 7266.892182] RDX: fffffffffffbc4f8 RSI: ffff8800764e03f8 RDI: ffff8800764e0000
[ 7266.899302] RBP: ffff880063573b30 R08: b7fffffbfeffff47 R09: 530000008efffffc
[ 7266.906422] R10: 0affff42d0000000 R11: 8ffffffc16ffff42 R12: ffff88007649c520
[ 7266.913549] R13: 00000000000003d8 R14: 0000000000001800 R15: 000000000000007b
[ 7266.920668] FS: 00007f1b4a778740(0000) GS:ffff88007d180000(0000) knlGS:0000000000000000
[ 7266.928750] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 7266.934484] CR2: ffff8800764e0000 CR3: 000000006478c000 CR4: 00000000000407e0
[ 7266.941607] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 7266.948734] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[ 7266.955853] Process attr (pid: 24364, threadinfo ffff880063572000, task ffff880062621ac0)
[ 7266.964020] Stack:
[ 7266.966029] ffffffffa028b1a1 0000000000000000 0000000000000000 ffff88007649c540
[ 7266.973478] ffff88007649c520 0000000000000018 ffff88007649c520 0000000000000000
[ 7266.980928] ffff88005f1fb7b0 ffff88003e8a8840 ffff88005f9476a0 ffff880063573b68
[ 7266.988376] Call Trace:
[ 7266.990846] [<ffffffffa028b1a1>] ? xfs_attr_leaf_moveents.isra.2+0x91/0x280 [xfs]
[ 7266.998425] [<ffffffffa028b467>] xfs_attr_leaf_compact+0xd7/0x130 [xfs]
[ 7267.005138] [<ffffffffa028ca2e>] xfs_attr_leaf_add+0xce/0x170 [xfs]
[ 7267.011502] [<ffffffffa0288850>] xfs_attr_leaf_addname+0xc0/0x3d0 [xfs]
[ 7267.018207] [<ffffffffa0296d4e>] ? xfs_bmap_one_block+0x3e/0xa0 [xfs]
[ 7267.024745] [<ffffffffa028978c>] xfs_attr_set_int+0x30c/0x420 [xfs]
[ 7267.031091] [<ffffffff811be9f4>] ? setxattr+0xa4/0x1c0
[ 7267.036326] [<ffffffffa0289d1f>] xfs_attr_set+0x7f/0x90 [xfs]
[ 7267.042163] [<ffffffffa027fa12>] xfs_xattr_set+0x42/0x70 [xfs]
[ 7267.048073] [<ffffffff811bdef2>] generic_setxattr+0x62/0x80
[ 7267.053728] [<ffffffff811be743>] __vfs_setxattr_noperm+0x63/0x1b0
[ 7267.059903] [<ffffffff811be945>] vfs_setxattr+0xb5/0xc0
[ 7267.065210] [<ffffffff811bea76>] setxattr+0x126/0x1c0
[ 7267.070348] [<ffffffff8118358d>] ? kmem_cache_free+0x1cd/0x1e0
[ 7267.076263] [<ffffffff811a72b2>] ? final_putname+0x22/0x50
[ 7267.081833] [<ffffffff811a74cb>] ? putname+0x2b/0x40
[ 7267.086885] [<ffffffff811ab96f>] ? user_path_at_empty+0x5f/0x90
[ 7267.092890] [<ffffffff8119e5c9>] ? __sb_start_write+0x49/0x100
[ 7267.098807] [<ffffffff811bedef>] sys_lsetxattr+0x8f/0xd0
[ 7267.104206] [<ffffffff81619359>] system_call_fastpath+0x16/0x1b
[ 7267.110205] Code: 00 00 48 81 fa a8 02 00 00 72 05 40 38 fe 74 41 48 83 ea 20 48 83 ea 20 4c 8b 1e 4c 8b 56 08 4c 8b 4e 10 4c 8b 46 18 48 8d 76 20 <4c> 89 1f 4c 89 57 08 4c 89 4f 10 4c 89 47 18 48 8d 7f 20 73 d4
[ 7267.130148] RIP [<ffffffff813022fa>] memmove+0x4a/0x1a0
[ 7267.135476] RSP <ffff880063573ad8>
[ 7267.138959] CR2: ffff8800764e0000
[ 7267.142290] general protection fault: 0000 [#2] SMP
[ 7267.147285] Modules linked in: binfmt_misc(F) snd_seq_dummy(F) tun(F) lockd(F) sunrpc(F) nf_conntrack_netbios_ns(F) nf_conntrack_broadcast(F) ipt_MASQUERADE(F) ip6table_nat(F) nf_nat_ipv6(F) ip6table_mangle(F) ip6t_REJECT(F) nf_conntrack_ipv6(F) nf_defrag_ipv6(F) iptable_nat(F) nf_nat_ipv4(F) nf_nat(F) iptable_mangle(F) ipt_REJECT(F) nf_conntrack_ipv4(F) nf_defrag_ipv4(F) xt_conntrack(F) nf_conntrack(F) ebtable_filter(F) ebtables(F) ip6table_filter(F) ip6_tables(F) iptable_filter(F) ip_tables(F) sg(F) snd_hda_codec_hdmi(F) snd_hda_codec_realtek(F) snd_hda_intel(F) snd_hda_codec(F) snd_hwdep(F) snd_seq(F) e1000e(F) snd_seq_device(F) snd_pcm(F) hp_wmi(F) iTCO_wdt(F) sparse_keymap(F) iTCO_vendor_support(F) snd_page_alloc(F) rfkill(F) ptp(F) snd_timer(F) mei(F) lpc_ich(F) pps_core(F) snd(F) pcspkr(F) i2c_i801(F) mfd_core(F) microcode(F) soundcore(F) xfs(F) libcrc32c(F) sr_mod(F) sd_mod(F) cdrom(F) crc_t10dif(F) nouveau(F) video(F) mxm_wmi(F) i2c_algo_bit(F) drm_kms_helper(F) ata_generic(F) ttm(F) ahci(F) pata_acpi(F) drm(F) libahci(F) i2c_core(F) wmi(F) libata(F) dm_mirror(F) dm_region_hash(F) dm_log(F) dm_mod(F)
[ 7267.246679] CPU 3
[ 7267.248519] Pid: 24364, comm: attr Tainted: GF 3.9.4 #1 Hewlett-Packard HP Z210 Workstation/1587h
[ 7267.258522] RIP: 0010:[<ffffffffa01c4da8>] [<ffffffffa01c4da8>] nv50_crtc_lut_load+0x98/0x110 [nouveau]
[ 7267.268031] RSP: 0018:ffff8800635734a0 EFLAGS: 00010046
[ 7267.273330] RAX: ffff880036e14908 RBX: ffffffff810a3297 RCX: 0000000000000000
[ 7267.280450] RDX: 0000000000000000 RSI: 0000000000000000 RDI: ffffc90010414000
[ 7267.287570] RBP: ffff8800635734b8 R08: ffffc90010414000 R09: 0000000000000000
[ 7267.294697] R10: 8b4cc28948c08548 R11: 000000000000ffff R12: ffffc90010414000
[ 7267.301817] R13: ffff880036e14520 R14: ffff880076abfb10 R15: ffff880076abfbb0
[ 7267.308938] FS: 00007f1b4a778740(0000) GS:ffff88007d180000(0000) knlGS:0000000000000000
[ 7267.317017] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 7267.322750] CR2: ffff8800764e0000 CR3: 000000006478c000 CR4: 00000000000407e0
[ 7267.329869] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 7267.336987] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[ 7267.344115] Process attr (pid: 24364, threadinfo ffff880063572000, task ffff880062621ac0)
[ 7267.352280] Stack:
[ 7267.354289] ffff880075c68000 000000000000000f 0000000000000010 ffff880063573570
[ 7267.361737] ffffffffa00e62fb ffff8800635735f8 ffffffff8133fa06 0000000000000010
[ 7267.369186] ffff880075c70320 ffff880075c68000 ffffffffa01de540 ffff880076abfb10
[ 7267.376641] Call Trace:
[ 7267.379089] [<ffffffffa00e62fb>] drm_fb_helper_setcmap+0x2fb/0x3c0 [drm_kms_helper]
[ 7267.386822] [<ffffffff8133fa06>] ? fb_set_var+0x276/0x430
[ 7267.392305] [<ffffffff8160dd32>] ? mutex_lock+0x12/0x30
[ 7267.397613] [<ffffffff8134268a>] fb_set_cmap+0x6a/0x140
[ 7267.402923] [<ffffffff8134ad9c>] fbcon_set_palette+0x13c/0x170
[ 7267.408838] [<ffffffff8134dd85>] fbcon_switch+0x3b5/0x550
[ 7267.414325] [<ffffffff813becc9>] redraw_screen+0x189/0x260
[ 7267.419891] [<ffffffff8133f70e>] ? fb_blank+0x9e/0xc0
[ 7267.425027] [<ffffffff8134c4ca>] fbcon_blank+0x20a/0x2d0
[ 7267.430425] [<ffffffff810898c2>] ? up+0x32/0x50
[ 7267.435039] [<ffffffff81062e84>] ? wake_up_klogd+0x34/0x50
[ 7267.440609] [<ffffffff810701f3>] ? __internal_add_timer+0x113/0x130
[ 7267.446953] [<ffffffff81070847>] ? internal_add_timer+0x17/0x40
[ 7267.452956] [<ffffffff81071b4f>] ? mod_timer+0x10f/0x220
[ 7267.458354] [<ffffffff813bf374>] do_unblank_screen+0xb4/0x1e0
[ 7267.464181] [<ffffffff813bf4b0>] unblank_screen+0x10/0x20
[ 7267.469660] [<ffffffff813034f9>] bust_spinlocks+0x19/0x40
[ 7267.475144] [<ffffffff81611fc4>] oops_end+0x34/0xe0
[ 7267.480103] [<ffffffff81604212>] no_context+0x263/0x270
[ 7267.485410] [<ffffffff81604292>] __bad_area_nosemaphore+0x73/0x1cc
[ 7267.491671] [<ffffffff816043fe>] bad_area_nosemaphore+0x13/0x15
[ 7267.497674] [<ffffffff81614c0e>] __do_page_fault+0x31e/0x510
[ 7267.503434] [<ffffffffa0283497>] ? kmem_alloc+0x67/0xf0 [xfs]
[ 7267.509266] [<ffffffff81020c2b>] ? save_stack_trace+0x2b/0x50
[ 7267.515092] [<ffffffff81180911>] ? set_track+0x61/0x1b0
[ 7267.520402] [<ffffffff8118176d>] ? init_object+0x3d/0x70
[ 7267.525799] [<ffffffff81614e0e>] do_page_fault+0xe/0x10
[ 7267.531109] [<ffffffff81611498>] page_fault+0x28/0x30
[ 7267.536246] [<ffffffff813022fa>] ? memmove+0x4a/0x1a0
[ 7267.541398] [<ffffffffa028b1a1>] ? xfs_attr_leaf_moveents.isra.2+0x91/0x280 [xfs]
[ 7267.548972] [<ffffffffa028b467>] xfs_attr_leaf_compact+0xd7/0x130 [xfs]
[ 7267.555676] [<ffffffffa028ca2e>] xfs_attr_leaf_add+0xce/0x170 [xfs]
[ 7267.562030] [<ffffffffa0288850>] xfs_attr_leaf_addname+0xc0/0x3d0 [xfs]
[ 7267.568734] [<ffffffffa0296d4e>] ? xfs_bmap_one_block+0x3e/0xa0 [xfs]
[ 7267.575264] [<ffffffffa028978c>] xfs_attr_set_int+0x30c/0x420 [xfs]
[ 7267.581610] [<ffffffff811be9f4>] ? setxattr+0xa4/0x1c0
[ 7267.586845] [<ffffffffa0289d1f>] xfs_attr_set+0x7f/0x90 [xfs]
[ 7267.592685] [<ffffffffa027fa12>] xfs_xattr_set+0x42/0x70 [xfs]
[ 7267.598595] [<ffffffff811bdef2>] generic_setxattr+0x62/0x80
[ 7267.604252] [<ffffffff811be743>] __vfs_setxattr_noperm+0x63/0x1b0
[ 7267.610429] [<ffffffff811be945>] vfs_setxattr+0xb5/0xc0
[ 7267.615738] [<ffffffff811bea76>] setxattr+0x126/0x1c0
[ 7267.620875] [<ffffffff8118358d>] ? kmem_cache_free+0x1cd/0x1e0
[ 7267.626791] [<ffffffff811a72b2>] ? final_putname+0x22/0x50
[ 7267.632361] [<ffffffff811a74cb>] ? putname+0x2b/0x40
[ 7267.637412] [<ffffffff811ab96f>] ? user_path_at_empty+0x5f/0x90
[ 7267.643415] [<ffffffff8119e5c9>] ? __sb_start_write+0x49/0x100
[ 7267.649331] [<ffffffff811bedef>] sys_lsetxattr+0x8f/0xd0
[ 7267.654727] [<ffffffff81619359>] system_call_fastpath+0x16/0x1b
[ 7267.660726] Code: c1 01 48 83 c0 02 48 83 c7 08 49 83 c0 20 41 81 f9 00 01 00 00 74 4c 4c 8b 13 0f b7 30 0f b7 88 00 02 00 00 0f b7 90 00 04 00 00 <4d> 8b 12 66 c1 ee 02 66 c1 e9 02 66 c1 ea 02 66 41 81 3a 6f 90
[ 7267.680665] RIP [<ffffffffa01c4da8>] nv50_crtc_lut_load+0x98/0x110 [nouveau]
[ 7267.687835] RSP <ffff8800635734a0>
[ 7267.691324] ---[ end trace 0ac6265371f9a5bf ]---
[ 7287.146356] Kernel panic - not syncing: Watchdog detected hard LOCKUP on cpu 3
[ 7288.185689] Shutting down cpus with NMI
[ 7288.189526] drm_kms_helper: panic occurred, switching back to text console
CAI Qian
On Sun, Jun 02, 2013 at 11:04:11PM -0400, CAI Qian wrote:
>
> > There's memory corruption all over the place. It is most likely
> > that trinity is causing this - it's purpose is to trigger corruption
> > issues, but they aren't always immediately seen. If you can trigger
> > this xfs trace without trinity having been run and without all the
> > RCU/idle/scheduler/cgroup issues occuring at the same time, then
> > it's likely to be caused by XFS. But right now, I'd say XFS is just
> > an innocent bystander caught in the crossfire. There's nothing I can
> > do from an XFS persepctive to track this down...
> OK, this can be reproduced by just running LTP and then xfstests without
> trinity at all...
Cai, can you be more precise about what is triggering it? LTP and
xfstests do a large amount of stuff, and stack traces do not do not
help narrow down the cause at all. Can you provide the follwoing
information and perform the follwoing steps:
1. What xfstest is tripping over it?
2. Can you reproduce it just by running that one specific test
on a pristine system (i.e. freshly mkfs'd filesystems,
immediately after boot)
3. if you can't reproduce it like that, does it reproduce on
an xfstest run on a pristine system? If so, what command
line are you running, and what are the filesystem
configurations?
4. if you cannot reproduce it just with xfstests and you need
to run LTP first, then can you just run the xfstest that
is failing after running LTP and see if that triggers the
problem. If it does, please take a metadump of the
filesystems after LTP has run, save them, and if the
single test then fails send me the metadumps and your
xfstests command line.
5. If all else fails, bisect the kernel to identify the
commit that introduces the problem....
Cheers,
Dave.
--
Dave Chinner
[email protected]
----- Original Message -----
> From: "Dave Chinner" <[email protected]>
> To: "CAI Qian" <[email protected]>
> Cc: [email protected], [email protected], "LKML" <[email protected]>, "linux-mm" <[email protected]>
> Sent: Monday, June 3, 2013 12:00:38 PM
> Subject: Re: 3.9.4 Oops running xfstests (WAS Re: 3.9.3: Oops running xfstests)
>
> On Sun, Jun 02, 2013 at 11:04:11PM -0400, CAI Qian wrote:
> >
> > > There's memory corruption all over the place. It is most likely
> > > that trinity is causing this - it's purpose is to trigger corruption
> > > issues, but they aren't always immediately seen. If you can trigger
> > > this xfs trace without trinity having been run and without all the
> > > RCU/idle/scheduler/cgroup issues occuring at the same time, then
> > > it's likely to be caused by XFS. But right now, I'd say XFS is just
> > > an innocent bystander caught in the crossfire. There's nothing I can
> > > do from an XFS persepctive to track this down...
> > OK, this can be reproduced by just running LTP and then xfstests without
> > trinity at all...
>
> Cai, can you be more precise about what is triggering it? LTP and
> xfstests do a large amount of stuff, and stack traces do not do not
> help narrow down the cause at all. Can you provide the follwoing
> information and perform the follwoing steps:
>
> 1. What xfstest is tripping over it?
Test #20.
> 2. Can you reproduce it just by running that one specific test
> on a pristine system (i.e. freshly mkfs'd filesystems,
> immediately after boot)
Yes, it was reproduced without LTP at all.
[ 98.534402] XFS (dm-0): Mounting Filesystem
[ 98.586673] XFS (dm-0): Ending clean mount
[ 99.741704] XFS (dm-2): Mounting Filesystem
[ 100.117248] XFS (dm-2): Ending clean mount
[ 100.723228] XFS (dm-0): Mounting Filesystem
[ 100.775965] XFS (dm-0): Ending clean mount
[ 101.980250] BUG: unable to handle kernel NULL pointer dereference at 0000000000000098
[ 101.988136] IP: [<ffffffff81098cac>] tg_load_down+0x4c/0x80
[ 101.993737] PGD 0
[ 101.995769] Oops: 0002 [#1] SMP
[ 101.999038] Modules linked in: lockd sunrpc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_MASQUERADE ip6table_nat nf_nat_ipv6 ip6table_mangle ip6t_REJECT nf_conntrack_ipv6 nf_defrag_ipv6 iptable_nat nf_nat_ipv4 nf_nat iptable_mangle ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter ip_tables sg snd_hda_codec_hdmi snd_hda_codec_realtek snd_hda_intel snd_hda_codec snd_hwdep snd_seq snd_seq_device snd_pcm hp_wmi sparse_keymap rfkill iTCO_wdt e1000e pcspkr snd_page_alloc iTCO_vendor_support mei ptp pps_core lpc_ich i2c_i801 snd_timer mfd_core microcode(+) snd soundcore xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif nouveau video mxm_wmi i2c_algo_bit drm_kms_helper ahci ata_generic ttm libahci pata_acpi drm i2c_core libata wmi dm_mirror dm_region_hash dm_log dm_mod
[ 102.075355] CPU 2
[ 102.077197] Pid: 356, comm: kworker/2:2 Not tainted 3.9.4 #1 Hewlett-Packard HP Z210 Workstation/1587h
[ 102.086691] RIP: 0010:[<ffffffff81098cac>] [<ffffffff81098cac>] tg_load_down+0x4c/0x80
[ 102.094705] RSP: 0018:ffff880078307c78 EFLAGS: 00010002
[ 102.100020] RAX: 0001f2b5a618ed0f RBX: 0000000000000001 RCX: 000000000000068a
[ 102.107157] RDX: 0000000000000000 RSI: 0000000000000001 RDI: ffff8800772ceee8
[ 102.114293] RBP: ffff880078307c78 R08: 0000000000000008 R09: ffff88007d094400
[ 102.121422] R10: 0000000000000344 R11: 0000000000000001 R12: ffffffff81c78560
[ 102.128552] R13: ffffffff8108c460 R14: 0000000000000000 R15: ffff8800772ceee8
[ 102.135682] FS: 0000000000000000(0000) GS:ffff88007d100000(0000) knlGS:0000000000000000
[ 102.143776] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 102.149524] CR2: 0000000000000098 CR3: 00000000018fa000 CR4: 00000000000407e0
[ 102.156654] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 102.163785] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[ 102.170915] Process kworker/2:2 (pid: 356, threadinfo ffff880078306000, task ffff88007736b580)
[ 102.179527] Stack:
[ 102.181545] ffff880078307cc0 ffffffff810926b2 ffffffff81098c60 ffff8800772cf008
[ 102.189005] ffff880079b96f00 000000000000069c ffff880079b96ee8 0000000000014400
[ 102.196464] ffff88007d094400 ffff880078307db0 ffffffff8109f773 ffff88007cc10480
[ 102.203920] Call Trace:
[ 102.206372] [<ffffffff810926b2>] walk_tg_tree_from+0x32/0xe0
[ 102.212118] [<ffffffff81098c60>] ? task_waking_fair+0x20/0x20
[ 102.217955] [<ffffffff8109f773>] load_balance+0x2a3/0x7d0
[ 102.223444] [<ffffffff8108fa7c>] ? update_rq_clock.part.67+0x1c/0x170
[ 102.229977] [<ffffffff810a0142>] idle_balance+0x182/0x2f0
[ 102.235468] [<ffffffff8160f1ac>] __schedule+0x7bc/0x7d0
[ 102.240786] [<ffffffff8160f1e9>] schedule+0x29/0x70
[ 102.245756] [<ffffffff8107f404>] worker_thread+0x1b4/0x3d0
[ 102.251332] [<ffffffff8107f250>] ? __alloc_workqueue_key+0x500/0x500
[ 102.257777] [<ffffffff81084240>] kthread+0xc0/0xd0
[ 102.262662] [<ffffffff81084180>] ? insert_kthread_work+0x40/0x40
[ 102.268761] [<ffffffff816192ac>] ret_from_fork+0x7c/0xb0
[ 102.274164] [<ffffffff81084180>] ? insert_kthread_work+0x40/0x40
[ 102.280262] Code: 00 00 00 00 48 8b 14 f0 48 8b 0c f1 48 8b 82 98 00 00 00 48 0f af 01 48 8b 0a 31 d2 48 83 c1 01 48 f7 f1 48 8b 57 48 4a 8b 14 02 <48> 89 82 98 00 00 00 31 c0 5d c3 66 0f 1f 84 00 00 00 00 00 48
[ 102.300226] RIP [<ffffffff81098cac>] tg_load_down+0x4c/0x80
[ 102.305906] RSP <ffff880078307c78>
[ 102.309394] CR2: 0000000000000098
[ 102.312710] ---[ end trace ba964230a74993fe ]---
[ 102.312714] BUG: unable to handle kernel NULL pointer dereference at 0000000000000098
[ 102.312719] IP: [<ffffffff81098cac>] tg_load_down+0x4c/0x80
[ 102.312720] PGD 0
[ 102.312723] Oops: 0002 [#2] SMP
[ 102.312752] Modules linked in: lockd sunrpc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_MASQUERADE ip6table_nat nf_nat_ipv6 ip6table_mangle ip6t_REJECT nf_conntrack_ipv6 nf_defrag_ipv6 iptable_nat nf_nat_ipv4 nf_nat iptable_mangle ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter ip_tables sg snd_hda_codec_hdmi snd_hda_codec_realtek snd_hda_intel snd_hda_codec snd_hwdep snd_seq snd_seq_device snd_pcm hp_wmi sparse_keymap rfkill iTCO_wdt e1000e pcspkr snd_page_alloc iTCO_vendor_support mei ptp pps_core lpc_ich i2c_i801 snd_timer mfd_core microcode(+) snd soundcore xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif nouveau video mxm_wmi i2c_algo_bit drm_kms_helper ahci ata_generic ttm libahci pata_acpi drm i2c_core libata wmi dm_mirror dm_region_hash dm_log dm_mod
[ 102.312758] CPU 0
[ 102.312758] Pid: 78, comm: kworker/0:2 Tainted: G D 3.9.4 #1 Hewlett-Packard HP Z210 Workstation/1587h
[ 102.312762] RIP: 0010:[<ffffffff81098cac>] [<ffffffff81098cac>] tg_load_down+0x4c/0x80
[ 102.312763] RSP: 0018:ffff880036c25c78 EFLAGS: 00010002
[ 102.312764] RAX: 0001f2b5a618ed0f RBX: 0000000000000001 RCX: 000000000000068a
[ 102.312765] RDX: 0000000000000000 RSI: 0000000000000001 RDI: ffff8800772ceee8
[ 102.312766] RBP: ffff880036c25c78 R08: 0000000000000008 R09: ffff88007d094400
[ 102.312767] R10: 0000000000000344 R11: 0000000000000001 R12: ffffffff81c78560
[ 102.312768] R13: ffffffff8108c460 R14: 0000000000000000 R15: ffff8800772ceee8
[ 102.312770] FS: 0000000000000000(0000) GS:ffff88007d000000(0000) knlGS:0000000000000000
[ 102.312771] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 102.312772] CR2: 0000000000000098 CR3: 00000000018fa000 CR4: 00000000000407f0
[ 102.312773] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 102.312774] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[ 102.312776] Process kworker/0:2 (pid: 78, threadinfo ffff880036c24000, task ffff8800773c9ac0)
[ 102.312776] Stack:
[ 102.312779] ffff880036c25cc0 ffffffff810926b2 ffffffff81098c60 ffff8800772cf008
[ 102.312781] ffff880079b96f00 000000000000069c ffff880079b96ee8 0000000000014400
[ 102.312783] ffff88007d094400 ffff880036c25db0 ffffffff8109f773 ffff88007cc10080
[ 102.312783] Call Trace:
[ 102.312787] [<ffffffff810926b2>] walk_tg_tree_from+0x32/0xe0
[ 102.312789] [<ffffffff81098c60>] ? task_waking_fair+0x20/0x20
[ 102.312792] [<ffffffff8109f773>] load_balance+0x2a3/0x7d0
[ 102.312795] [<ffffffff8108fa7c>] ? update_rq_clock.part.67+0x1c/0x170
[ 102.312798] [<ffffffff810a0142>] idle_balance+0x182/0x2f0
[ 102.312801] [<ffffffff8160f1ac>] __schedule+0x7bc/0x7d0
[ 102.312803] [<ffffffff8160f1e9>] schedule+0x29/0x70
[ 102.312806] [<ffffffff8107f404>] worker_thread+0x1b4/0x3d0
[ 102.312809] [<ffffffff8107f250>] ? __alloc_workqueue_key+0x500/0x500
[ 102.312811] [<ffffffff81084240>] kthread+0xc0/0xd0
[ 102.312813] [<ffffffff81084180>] ? insert_kthread_work+0x40/0x40
[ 102.312816] [<ffffffff816192ac>] ret_from_fork+0x7c/0xb0
[ 102.312818] [<ffffffff81084180>] ? insert_kthread_work+0x40/0x40
[ 102.312837] Code: 00 00 00 00 48 8b 14 f0 48 8b 0c f1 48 8b 82 98 00 00 00 48 0f af 01 48 8b 0a 31 d2 48 83 c1 01 48 f7 f1 48 8b 57 48 4a 8b 14 02 <48> 89 82 98 00 00 00 31 c0 5d c3 66 0f 1f 84 00 00 00 00 00 48
[ 102.312839] RIP [<ffffffff81098cac>] tg_load_down+0x4c/0x80
[ 102.312840] RSP <ffff880036c25c78>
[ 102.312840] CR2: 0000000000000098
[ 102.312842] ---[ end trace ba964230a74993ff ]---
[ 102.312866] general protection fault: 0000 [#3] SMP
[ 102.312896] Modules linked in: lockd sunrpc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_MASQUERADE ip6table_nat nf_nat_ipv6 ip6table_mangle ip6t_REJECT nf_conntrack_ipv6 nf_defrag_ipv6 iptable_nat nf_nat_ipv4 nf_nat iptable_mangle ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter ip_tables sg snd_hda_codec_hdmi snd_hda_codec_realtek snd_hda_intel snd_hda_codec snd_hwdep snd_seq snd_seq_device snd_pcm hp_wmi sparse_keymap rfkill iTCO_wdt e1000e pcspkr snd_page_alloc iTCO_vendor_support mei ptp pps_core lpc_ich i2c_i801 snd_timer mfd_core microcode(+) snd soundcore xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif nouveau video mxm_wmi i2c_algo_bit drm_kms_helper ahci ata_generic ttm libahci pata_acpi drm i2c_core libata wmi dm_mirror dm_region_hash dm_log dm_mod
[ 102.312903] CPU 1
[ 102.312903] Pid: 1999, comm: attr Tainted: G D 3.9.4 #1 Hewlett-Packard HP Z210 Workstation/1587h
[ 102.312908] RIP: 0010:[<ffffffff810983a9>] [<ffffffff810983a9>] irqtime_account_process_tick.isra.2+0x239/0x3c0
[ 102.312909] =============================================================================
[ 102.312910] RSP: 0018:ffff88007d083e08 EFLAGS: 00010003
[ 102.312912] BUG kmalloc-1024 (Tainted: G D ): Padding overwritten. 0xffff88005b4e7ec0-0xffff88005b4e7fff
[ 102.312913] RAX: ffff88005b656288 RBX: ffff880079b43c80 RCX: 00000000000000a7
[ 102.312914] -----------------------------------------------------------------------------
[ 102.312914]
[ 102.312915] RDX: 6b6b6b6b6b6b6b6b RSI: 0000000000000000 RDI: 0000000000000086
[ 102.312917] INFO: Slab 0xffffea00016d3800 objects=24 used=24 fp=0x (null) flags=0x10000000004080
[ 102.312918] RBP: ffff88007d083e40 R08: 0000000000000000 R09: 0000000225c17d03
[ 102.312921] Pid: 518, comm: in:imklog Tainted: G B D 3.9.4 #1
[ 102.312922] R10: 0000000000000000 R11: 0000000000000001 R12: ffff88007d08e800
[ 102.312922] Call Trace:
[ 102.312923] R13: ffff880058063580 R14: 0000000000000000 R15: ffff88007d094c88
[ 102.312926] FS: 00007f237404a740(0000) GS:ffff88007d080000(0000) knlGS:0000000000000000
[ 102.312931] [<ffffffff81181ed2>] slab_err+0xc2/0xf0
[ 102.312932] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 102.312934] CR2: 00007f2373c352a0 CR3: 0000000063ea9000 CR4: 00000000000407e0
[ 102.312937] [<ffffffff81018018>] ? write_ok_or_segv+0x88/0x90
[ 102.312938] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 102.312939] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[ 102.312943] [<ffffffff81020c2b>] ? save_stack_trace+0x2b/0x50
[ 102.312944] Process attr (pid: 1999, threadinfo ffff88005753e000, task ffff880058063580)
[ 102.312945] Stack:
[ 102.312947] [<ffffffff81180911>] ? set_track+0x61/0x1b0
[ 102.312949] ffff88007d08e800
[ 102.312952] [<ffffffff8118176d>] ? init_object+0x3d/0x70
[ 102.312953] 0000000000000086 ffff88007d094400
[ 102.312956] [<ffffffff81181ff5>] slab_pad_check.part.41+0xf5/0x170
[ 102.312957] 0000000000014400
[ 102.312967] [<ffffffff81063bdc>] ? do_syslog+0x23c/0x5c0
[ 102.312968] ffff880058063580 0000000000000000
[ 102.312969] [<ffffffff811820e3>] check_slab+0x73/0x100
[ 102.312970] 00000017979e99b3 ffff88007d083e78
[ 102.312970]
[ 102.312972] [<ffffffff81606b50>] alloc_debug_processing+0x21/0x118
[ 102.312973] ffffffff8109874c
[ 102.312974] [<ffffffff8160772f>] __slab_alloc+0x3b8/0x4a2
[ 102.312974] ffff880058063580 0000000000000000
[ 102.312976] [<ffffffff81063bdc>] ? do_syslog+0x23c/0x5c0
[ 102.312976] 0000000000000001
[ 102.312977] Call Trace:
[ 102.312978] [<ffffffff810915aa>] ? finish_task_switch+0xba/0xe0
[ 102.312979] <IRQ>
[ 102.312980] [<ffffffff81184dd1>] kmem_cache_alloc_trace+0x1b1/0x200
[ 102.312981] [<ffffffff8109874c>] account_process_tick+0x11c/0x1d0
[ 102.312983] [<ffffffff81063bdc>] do_syslog+0x23c/0x5c0
[ 102.312985] [<ffffffff81071f0d>] update_process_times+0x2d/0x80
[ 102.312986] [<ffffffff810850e0>] ? wake_up_bit+0x30/0x30
[ 102.312988] [<ffffffff810ba1c5>] tick_sched_handle.isra.13+0x25/0x60
[ 102.312991] [<ffffffff8120ca94>] kmsg_read+0x44/0x60
[ 102.312992] [<ffffffff810ba241>] tick_sched_timer+0x41/0x60
[ 102.312994] [<ffffffff811ffa9a>] proc_reg_read+0x6a/0xa0
[ 102.312996] [<ffffffff81087af4>] __run_hrtimer+0x74/0x1d0
[ 102.312997] [<ffffffff8119c56c>] vfs_read+0x9c/0x170
[ 102.312998] [<ffffffff810ba200>] ? tick_sched_handle.isra.13+0x60/0x60
[ 102.313000] [<ffffffff8119c939>] sys_read+0x49/0xa0
[ 102.313001] [<ffffffff810882d7>] hrtimer_interrupt+0xe7/0x220
[ 102.313004] [<ffffffff810e0ef6>] ? __audit_syscall_exit+0x1f6/0x2a0
[ 102.313005] [<ffffffff8161b0e9>] smp_apic_timer_interrupt+0x69/0x9c
[ 102.313007] [<ffffffff81619359>] system_call_fastpath+0x16/0x1b
[ 102.313008] [<ffffffff81619fdd>] apic_timer_interrupt+0x6d/0x80
[ 102.313009] Padding ffff88005b4e7ec0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 102.313010] Padding ffff88005b4e7ed0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 102.313010] <EOI>
[ 102.313011] Padding ffff88005b4e7ee0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 102.313013] [<ffffffff813022f6>] ? memmove+0x46/0x1a0
[ 102.313013] Padding ffff88005b4e7ef0: 00 00 00 00 00 00 00 00 00 00 00 00 00 29 01 00 .............)..
[ 102.313014] Padding ffff88005b4e7f00: 07 1b 04 73 65 6c 69 6e 75 78 73 79 73 74 65 6d ...selinuxsystem
[ 102.313015] Padding ffff88005b4e7f10: 5f 75 3a 6f 62 6a 65 63 74 5f 72 3a 75 73 72 5f _u:object_r:usr_
[ 102.313032] [<ffffffffa02801a1>] ? xfs_attr_leaf_moveents.isra.2+0x91/0x280 [xfs]
[ 102.313032] Padding ffff88005b4e7f20: 74 3a 73 30 00 00 00 00 49 4e 81 a4 02 02 00 00 t:s0....IN......
[ 102.313033] Padding ffff88005b4e7f30: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................
[ 102.313033] Padding ffff88005b4e7f40: 00 00 00 00 00 00 00 02 51 47 09 00 00 00 00 00 ........QG......
[ 102.313042] [<ffffffffa0280467>] xfs_attr_leaf_compact+0xd7/0x130 [xfs]
[ 102.313043] Padding ffff88005b4e7f50: 51 47 09 00 00 00 00 00 51 ac 1e 27 21 f1 4e ad QG......Q..'!.N.
[ 102.313043] Padding ffff88005b4e7f60: 00 00 00 00 00 00 00 f2 00 00 00 00 00 00 00 01 ................
[ 102.313044] Padding ffff88005b4e7f70: 00 00 00 00 00 00 00 01 00 00 0e 01 00 00 00 00 ................
[ 102.313053] [<ffffffffa0281a2e>] xfs_attr_leaf_add+0xce/0x170 [xfs]
[ 102.313053] Padding ffff88005b4e7f80: 00 00 00 00 c1 6d 78 44 ff ff ff ff 00 00 00 00 .....mxD........
[ 102.313054] Padding ffff88005b4e7f90: 00 00 00 00 00 00 08 10 36 a0 00 01 00 00 00 00 ........6.......
[ 102.313062] [<ffffffffa027d850>] xfs_attr_leaf_addname+0xc0/0x3d0 [xfs]
[ 102.313062] Padding ffff88005b4e7fa0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 102.313063] Padding ffff88005b4e7fb0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 102.313072] [<ffffffffa028bd4e>] ? xfs_bmap_one_block+0x3e/0xa0 [xfs]
[ 102.313072] Padding ffff88005b4e7fc0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 102.313073] Padding ffff88005b4e7fd0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 102.313074] Padding ffff88005b4e7fe0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
[ 102.313081] [<ffffffffa027e78c>] xfs_attr_set_int+0x30c/0x420 [xfs]
[ 102.313082] Padding ffff88005b4e7ff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 29 01 00 .............)..
[ 102.313085] [<ffffffff811be9f4>] ? setxattr+0xa4/0x1c0
[ 102.313085] FIX kmalloc-1024: Restoring 0xffff88005b4e7ec0-0xffff88005b4e7fff=0x5a
[ 102.313085]
[ 102.313086] ==========
CAI Qian
> 3. if you can't reproduce it like that, does it reproduce on
> an xfstest run on a pristine system? If so, what command
> line are you running, and what are the filesystem
> configurations?
> 4. if you cannot reproduce it just with xfstests and you need
> to run LTP first, then can you just run the xfstest that
> is failing after running LTP and see if that triggers the
> problem. If it does, please take a metadump of the
> filesystems after LTP has run, save them, and if the
> single test then fails send me the metadumps and your
> xfstests command line.
> 5. If all else fails, bisect the kernel to identify the
> commit that introduces the problem....
>
> Cheers,
>
> Dave.
>
> --
> Dave Chinner
> [email protected]
>
> --
> To unsubscribe, send a message with 'unsubscribe linux-mm' in
> the body to [email protected]. For more info on Linux MM,
> see: http://www.linux-mm.org/ .
> Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
>
On Mon, Jun 03, 2013 at 11:14:56PM -0400, CAI Qian wrote:
>
>
> ----- Original Message -----
> > From: "Dave Chinner" <[email protected]>
> > To: "CAI Qian" <[email protected]>
> > Cc: [email protected], [email protected], "LKML" <[email protected]>, "linux-mm" <[email protected]>
> > Sent: Monday, June 3, 2013 12:00:38 PM
> > Subject: Re: 3.9.4 Oops running xfstests (WAS Re: 3.9.3: Oops running xfstests)
> >
> > On Sun, Jun 02, 2013 at 11:04:11PM -0400, CAI Qian wrote:
> > >
> > > > There's memory corruption all over the place. It is most likely
> > > > that trinity is causing this - it's purpose is to trigger corruption
> > > > issues, but they aren't always immediately seen. If you can trigger
> > > > this xfs trace without trinity having been run and without all the
> > > > RCU/idle/scheduler/cgroup issues occuring at the same time, then
> > > > it's likely to be caused by XFS. But right now, I'd say XFS is just
> > > > an innocent bystander caught in the crossfire. There's nothing I can
> > > > do from an XFS persepctive to track this down...
> > > OK, this can be reproduced by just running LTP and then xfstests without
> > > trinity at all...
> >
> > Cai, can you be more precise about what is triggering it? LTP and
> > xfstests do a large amount of stuff, and stack traces do not do not
> > help narrow down the cause at all. Can you provide the follwoing
> > information and perform the follwoing steps:
> >
> > 1. What xfstest is tripping over it?
> Test #20.
> > 2. Can you reproduce it just by running that one specific test
> > on a pristine system (i.e. freshly mkfs'd filesystems,
> > immediately after boot)
> Yes, it was reproduced without LTP at all.
> [ 98.534402] XFS (dm-0): Mounting Filesystem
> [ 98.586673] XFS (dm-0): Ending clean mount
> [ 99.741704] XFS (dm-2): Mounting Filesystem
> [ 100.117248] XFS (dm-2): Ending clean mount
> [ 100.723228] XFS (dm-0): Mounting Filesystem
> [ 100.775965] XFS (dm-0): Ending clean mount
> [ 101.980250] BUG: unable to handle kernel NULL pointer dereference at 0000000000000098
> [ 101.988136] IP: [<ffffffff81098cac>] tg_load_down+0x4c/0x80
first bug, in scheduler
> [ 102.312714] BUG: unable to handle kernel NULL pointer dereference at 0000000000000098
> [ 102.312719] IP: [<ffffffff81098cac>] tg_load_down+0x4c/0x80
second bug, in scheduler
> [ 102.312842] ---[ end trace ba964230a74993ff ]---
> [ 102.312866] general protection fault: 0000 [#3] SMP
Third bug
> [ 102.312903] Pid: 1999, comm: attr Tainted: G D 3.9.4 #1 Hewlett-Packard HP Z210 Workstation/1587h
> [ 102.312908] RIP: 0010:[<ffffffff810983a9>] [<ffffffff810983a9>] irqtime_account_process_tick.isra.2+0x239/0x3c0
In the timer tick code.
> [ 102.312909] =============================================================================
> [ 102.312910] RSP: 0018:ffff88007d083e08 EFLAGS: 00010003
> [ 102.312912] BUG kmalloc-1024 (Tainted: G D ): Padding overwritten. 0xffff88005b4e7ec0-0xffff88005b4e7fff
> [ 102.312913] RAX: ffff88005b656288 RBX: ffff880079b43c80 RCX: 00000000000000a7
> [ 102.312914] -----------------------------------------------------------------------------
And a memory overwrite.
> [ 102.313009] Padding ffff88005b4e7ec0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
> [ 102.313010] Padding ffff88005b4e7ed0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
> [ 102.313011] Padding ffff88005b4e7ee0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
> [ 102.313013] Padding ffff88005b4e7ef0: 00 00 00 00 00 00 00 00 00 00 00 00 00 29 01 00 .............)..
> [ 102.313014] Padding ffff88005b4e7f00: 07 1b 04 73 65 6c 69 6e 75 78 73 79 73 74 65 6d ...selinuxsystem
> [ 102.313015] Padding ffff88005b4e7f10: 5f 75 3a 6f 62 6a 65 63 74 5f 72 3a 75 73 72 5f _u:object_r:usr_
> [ 102.313032] Padding ffff88005b4e7f20: 74 3a 73 30 00 00 00 00 49 4e 81 a4 02 02 00 00 t:s0....IN......
> [ 102.313033] Padding ffff88005b4e7f30: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................
> [ 102.313033] Padding ffff88005b4e7f40: 00 00 00 00 00 00 00 02 51 47 09 00 00 00 00 00 ........QG......
> [ 102.313043] Padding ffff88005b4e7f50: 51 47 09 00 00 00 00 00 51 ac 1e 27 21 f1 4e ad QG......Q..'!.N.
> [ 102.313043] Padding ffff88005b4e7f60: 00 00 00 00 00 00 00 f2 00 00 00 00 00 00 00 01 ................
> [ 102.313044] Padding ffff88005b4e7f70: 00 00 00 00 00 00 00 01 00 00 0e 01 00 00 00 00 ................
> [ 102.313053] Padding ffff88005b4e7f80: 00 00 00 00 c1 6d 78 44 ff ff ff ff 00 00 00 00 .....mxD........
> [ 102.313054] Padding ffff88005b4e7f90: 00 00 00 00 00 00 08 10 36 a0 00 01 00 00 00 00 ........6.......
> [ 102.313062] Padding ffff88005b4e7fa0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
> [ 102.313063] Padding ffff88005b4e7fb0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
> [ 102.313072] Padding ffff88005b4e7fc0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
> [ 102.313073] Padding ffff88005b4e7fd0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
> [ 102.313074] Padding ffff88005b4e7fe0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
> [ 102.313082] Padding ffff88005b4e7ff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 29 01 00 .............)..
Oh, look, that contains attributes, and being at the top of a page,
that tallies with the attribute code copying data from the top of
the block down....
> [ 102.313013] [<ffffffff813022f6>] ? memmove+0x46/0x1a0
> [ 102.313032] [<ffffffffa02801a1>] ? xfs_attr_leaf_moveents.isra.2+0x91/0x280 [xfs]
> [ 102.313042] [<ffffffffa0280467>] xfs_attr_leaf_compact+0xd7/0x130 [xfs]
And the attribute code is writing into a block sized buffer that is
allocated in xfs_attr_leaf_compact().
So, whatever page got handed to xfs_attr_leaf_compact() is actually
referenced by the 1024 byte and supposed to be free. So, did XFS ask
for a 1k block, or something else?
Cai, I did ask you for the information that would have answered this
question:
> > 3. if you can't reproduce it like that, does it reproduce on
> > an xfstest run on a pristine system? If so, what command
> > line are you running, and what are the filesystem
> > configurations?
So, I need xfstests command line and the xfs_info output from the
filesystems in use at the time this problem occurs..
That said, this still looks like something has corrupted the heap
and XFS is just tripping over it....
Cheers,
Dave.
--
Dave Chinner
[email protected]
On Tue, Jun 04, 2013 at 02:16:18PM +1000, Dave Chinner wrote:
> On Mon, Jun 03, 2013 at 11:14:56PM -0400, CAI Qian wrote:
> > [ 102.312909] =============================================================================
> > [ 102.312910] RSP: 0018:ffff88007d083e08 EFLAGS: 00010003
> > [ 102.312912] BUG kmalloc-1024 (Tainted: G D ): Padding overwritten. 0xffff88005b4e7ec0-0xffff88005b4e7fff
> > [ 102.312913] RAX: ffff88005b656288 RBX: ffff880079b43c80 RCX: 00000000000000a7
> > [ 102.312914] -----------------------------------------------------------------------------
>
> And a memory overwrite.
>
> > [ 102.313009] Padding ffff88005b4e7ec0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
> > [ 102.313010] Padding ffff88005b4e7ed0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
> > [ 102.313011] Padding ffff88005b4e7ee0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
> > [ 102.313013] Padding ffff88005b4e7ef0: 00 00 00 00 00 00 00 00 00 00 00 00 00 29 01 00 .............)..
> > [ 102.313014] Padding ffff88005b4e7f00: 07 1b 04 73 65 6c 69 6e 75 78 73 79 73 74 65 6d ...selinuxsystem
> > [ 102.313015] Padding ffff88005b4e7f10: 5f 75 3a 6f 62 6a 65 63 74 5f 72 3a 75 73 72 5f _u:object_r:usr_
> > [ 102.313032] Padding ffff88005b4e7f20: 74 3a 73 30 00 00 00 00 49 4e 81 a4 02 02 00 00 t:s0....IN......
> > [ 102.313033] Padding ffff88005b4e7f30: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................
> > [ 102.313033] Padding ffff88005b4e7f40: 00 00 00 00 00 00 00 02 51 47 09 00 00 00 00 00 ........QG......
> > [ 102.313043] Padding ffff88005b4e7f50: 51 47 09 00 00 00 00 00 51 ac 1e 27 21 f1 4e ad QG......Q..'!.N.
> > [ 102.313043] Padding ffff88005b4e7f60: 00 00 00 00 00 00 00 f2 00 00 00 00 00 00 00 01 ................
> > [ 102.313044] Padding ffff88005b4e7f70: 00 00 00 00 00 00 00 01 00 00 0e 01 00 00 00 00 ................
> > [ 102.313053] Padding ffff88005b4e7f80: 00 00 00 00 c1 6d 78 44 ff ff ff ff 00 00 00 00 .....mxD........
> > [ 102.313054] Padding ffff88005b4e7f90: 00 00 00 00 00 00 08 10 36 a0 00 01 00 00 00 00 ........6.......
> > [ 102.313062] Padding ffff88005b4e7fa0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
> > [ 102.313063] Padding ffff88005b4e7fb0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
> > [ 102.313072] Padding ffff88005b4e7fc0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
> > [ 102.313073] Padding ffff88005b4e7fd0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
> > [ 102.313074] Padding ffff88005b4e7fe0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
> > [ 102.313082] Padding ffff88005b4e7ff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 29 01 00 .............)..
>
> Oh, look, that contains attributes, and being at the top of a page,
> that tallies with the attribute code copying data from the top of
> the block down....
On second thoughts, I'm not so sure of this now. That actually has
an inode core in it (the bit starting from "IN"), so it can't be a
piece of code from the attribute compaction. So this piece of memory
has been used several times by different things before the overwrite
has triggered by the look of it.
Cheers,
Dave.
--
Dave Chinner
[email protected]
> Cai, I did ask you for the information that would have answered this
> question:
>
> > > 3. if you can't reproduce it like that, does it reproduce on
> > > an xfstest run on a pristine system? If so, what command
> > > line are you running, and what are the filesystem
> > > configurations?
>
> So, I need xfstests command line and the xfs_info output from the
> filesystems in use at the time this problem occurs..
Here you are.
[root@hp-z210-01 xfstests-dev]# a=`grep ' swap' /etc/fstab | cut -f 1 -d ' '`
[root@hp-z210-01 xfstests-dev]# b=`grep ' /home' /etc/fstab | cut -f 1 -d ' '`
[root@hp-z210-01 xfstests-dev]# swapoff -a
[root@hp-z210-01 xfstests-dev]# umount /home
[root@hp-z210-01 xfstests-dev]# echo "swap = $a"
swap = /dev/mapper/rhel_hp--z210--01-swap
[root@hp-z210-01 xfstests-dev]# echo "home = $b"
home = /dev/mapper/rhel_hp--z210--01-home
[root@hp-z210-01 xfstests-dev]# export TEST_DEV=$a
[root@hp-z210-01 xfstests-dev]# export TEST_DIR=/mnt/testarea/test
[root@hp-z210-01 xfstests-dev]# export SCRATCH_DEV=$b
[root@hp-z210-01 xfstests-dev]# export SCRATCH_MNT=/mnt/testarea/scratch
[root@hp-z210-01 xfstests-dev]# mkdir -p /mnt/testarea/test
[root@hp-z210-01 xfstests-dev]# mkdir -p /mnt/testarea/scratch
[root@hp-z210-01 xfstests-dev]#
[root@hp-z210-01 xfstests-dev]# mkfs.xfs -f $a
meta-data=/dev/mapper/rhel_hp--z210--01-swap isize=256 agcount=4, agsize=251904 blks
= sectsz=512 attr=2, projid32bit=0
data = bsize=4096 blocks=1007616, imaxpct=25
= sunit=0 swidth=0 blks
naming =version 2 bsize=4096 ascii-ci=0
log =internal log bsize=4096 blocks=2560, version=2
= sectsz=512 sunit=0 blks, lazy-count=1
realtime =none extsz=4096 blocks=0, rtextents=0
[root@hp-z210-01 xfstests-dev]# mkfs.xfs -f $b
meta-data=/dev/mapper/rhel_hp--z210--01-home isize=256 agcount=4, agsize=11701504 blks
= sectsz=512 attr=2, projid32bit=0
data = bsize=4096 blocks=46806016, imaxpct=25
= sunit=0 swidth=0 blks
naming =version 2 bsize=4096 ascii-ci=0
log =internal log bsize=4096 blocks=22854, version=2
= sectsz=512 sunit=0 blks, lazy-count=1
realtime =none extsz=4096 blocks=0, rtextents=0
[root@hp-z210-01 xfstests-dev]#
[root@hp-z210-01 xfstests-dev]# mount /dev/mapper/rhel_hp--z210--01-home /mnt/testarea/scratch
[root@hp-z210-01 xfstests-dev]#
[root@hp-z210-01 xfstests-dev]# mount /dev/mapper/rhel_hp--z210--01-swap /mnt/testarea/test
[root@hp-z210-01 xfstests-dev]# xfs_info $a
meta-data=/dev/mapper/rhel_hp--z210--01-swap isize=256 agcount=4, agsize=251904 blks
= sectsz=512 attr=2
data = bsize=4096 blocks=1007616, imaxpct=25
= sunit=0 swidth=0 blks
naming =version 2 bsize=4096 ascii-ci=0
log =internal bsize=4096 blocks=2560, version=2
= sectsz=512 sunit=0 blks, lazy-count=1
realtime =none extsz=4096 blocks=0, rtextents=0
[root@hp-z210-01 xfstests-dev]# xfs_info $b
meta-data=/dev/mapper/rhel_hp--z210--01-home isize=256 agcount=4, agsize=11701504 blks
= sectsz=512 attr=2
data = bsize=4096 blocks=46806016, imaxpct=25
= sunit=0 swidth=0 blks
naming =version 2 bsize=4096 ascii-ci=0
log =internal bsize=4096 blocks=22854, version=2
= sectsz=512 sunit=0 blks, lazy-count=1
realtime =none extsz=4096 blocks=0, rtextents=0
[root@hp-z210-01 xfstests-dev]# ./check 20
FSTYP -- xfs (non-debug)
PLATFORM -- Linux/x86_64 hp-z210-01 3.9.4
MKFS_OPTIONS -- -f -bsize=4096 /dev/mapper/rhel_hp--z210--01-home
MOUNT_OPTIONS -- -o context=system_u:object_r:nfs_t:s0 /dev/mapper/rhel_hp--z210--01-home /mnt/testarea/scratch
020 <crashed immediately...>
CAI Qian