System hung when running xfstests-dev 013 test case on an s390x guest. Never saw
this on 3.9-rc3 before but need to double-check. Any idea?
CAI Qian
Ý 1113.795759¨ ------------Ý cut here ¨------------
Ý 1113.795771¨ kernel BUG at fs/ext4/inode.c:1591!
Ý 1113.795845¨ illegal operation: 0001 Ý#1¨ SMP
Ý 1113.795848¨ Modules linked in: nf_conntrack_netbios_ns nf_conntrack_broadcast
ipt_MASQUERADE ip6table_mangle ip6t_REJECT nf_conntrack_ipv6 nf_defrag_ipv6 ipt
able_nat nf_nat_ipv4 nf_nat iptable_mangle ipt_REJECT nf_conntrack_ipv4 nf_defra
g_ipv4 xt_conntrack nf_conntrack ebtable_filter ebtables ip6table_filter ip6_tab
les iptable_filter ip_tables sg qeth_l2 vmur xfs libcrc32c dasd_fba_mod dasd_eck
d_mod dasd_mod lcs ctcm qeth fsm qdio ccwgroup dm_mirror dm_region_hash dm_log d
m_mod
Ý 1113.795880¨ CPU: 1 Not tainted 3.9.0-rc4+ #2
Ý 1113.795882¨ Process flush-253:1 (pid: 12418, task: 0000000003f04890, ksp: 000
0000032eab3b8)
Ý 1113.795885¨ Krnl PSW : 0704e00180000000 000000000030bc62 (mpage_da_submit_io+
0x38e/0x3c0)
Ý 1113.795895¨ R:0 T:1 IO:1 EX:1 Key:0 M:1 W:0 P:0 AS:3 CC:2 PM:0 EA:
3
Krnl GPRS: 0000000001000000 000000001b5823a8 0000000000000010 000000001b5823a8
Ý 1113.795900¨ 0000000000000000 00000000000000af 0000000000001000 000
0000032eab9c8
Ý 1113.795902¨ 000003d10053bb40 0000000032eaba30 00000000000000a1 000
0000032eaba98
Ý 1113.795905¨ 00000000000100b0 00000000000000a1 000000000030b9a8 000
0000032eab8a0
Ý 1113.795921¨ Krnl Code: 000000000030bc54: f0dca7f4fecf srp 2036(14,
%r10),3791(%r15),12
MORE... ZVM-RHTS
================================================================================
000000000030bc5a: a7f40001 brc 15,30bc5c
#000000000030bc5e: a7f40001 brc 15,30bc60
>000000000030bc62: a7f40001 brc 15,30bc64
000000000030bc66: a7f40001 brc 15,30bc68
000000000030bc6a: 4120f0e8 la %r2,232(%r15)
000000000030bc6e: a7180000 lhi %r1,0
Ý 1113.796303¨ Call Trace:
Ý 1113.796306¨ (Ý<000000000030b9a8>¨ mpage_da_submit_io+0xd4/0x3c0)
Ý 1113.796311¨ Ý<000000000031204c>¨ mpage_da_map_and_submit+0x150/0x41c
Ý 1113.796314¨ Ý<0000000000312bac>¨ ext4_da_writepages+0x364/0x628
Ý 1113.796317¨ Ý<00000000002a08c8>¨ __writeback_single_inode+0x54/0x27c
Ý 1113.796322¨ Ý<00000000002a2c6c>¨ writeback_sb_inodes+0x284/0x4d8
Ý 1113.796325¨ Ý<00000000002a30b6>¨ wb_writeback+0x10e/0x374
Ý 1113.796327¨ Ý<00000000002a3d22>¨ wb_do_writeback+0x102/0x2e0
Ý 1113.796330¨ Ý<00000000002a3fa2>¨ bdi_writeback_thread+0xa2/0x270
Ý 1113.796333¨ Ý<00000000001581b2>¨ kthread+0xda/0xe4
Ý 1113.796338¨ Ý<0000000000629836>¨ kernel_thread_starter+0x6/0xc
Ý 1113.796342¨ Ý<0000000000629830>¨ kernel_thread_starter+0x0/0xc
Ý 1113.796345¨ Last Breaking-Event-Address:
Ý 1113.796346¨ Ý<000000000030bc5e>¨ mpage_da_submit_io+0x38a/0x3c0
Ý 1113.796349¨
Ý 1113.796351¨ ---Ý end trace 45df5089b835470e ¨---
Ý 1113.796365¨ ------------Ý cut here ¨------------
Ý 1113.796367¨ WARNING: at kernel/exit.c:715
Ý 1113.796369¨ Modules linked in: nf_conntrack_netbios_ns nf_conntrack_broadcast
ipt_MASQUERADE ip6table_mangle ip6t_REJECT nf_conntrack_ipv6 nf_defrag_ipv6 ipt
able_nat nf_nat_ipv4 nf_nat iptable_mangle ipt_REJECT nf_conntrack_ipv4 nf_defra
g_ipv4 xt_conntrack nf_conntrack ebtable_filter ebtables ip6table_filter ip6_tab
les iptable_filter ip_tables sg qeth_l2 vmur xfs libcrc32c dasd_fba_mod dasd_eck
d_mod dasd_mod lcs ctcm qeth fsm qdio ccwgroup dm_mirror dm_region_hash dm_log d
m_mod
Ý 1113.796393¨ CPU: 1 Tainted: G D 3.9.0-rc4+ #2
Ý 1113.796396¨ Process flush-253:1 (pid: 12418, task: 0000000003f04890, ksp: 000
0000032eab3b8)
Ý 1113.796398¨ Krnl PSW : 0704e00180000000 0000000000133074 (do_exit+0x54/0xab8)
Ý 1113.796404¨ R:0 T:1 IO:1 EX:1 Key:0 M:1 W:0 P:0 AS:3 CC:2 PM:0 EA:
3
Krnl GPRS: 0000000000000015 0000000032eabb60 0000000032eabb68 0000000000000000
Ý 1113.796408¨ 0000000000133058 ffffffffffffffff 0000000000000001 00
MORE... ZVM-RHTS
================================================================================
0000032eab9c8
Ý 1113.796411¨ 0704e00180000000 000000000030bc62 0000000003f04890 000
0000000778d38
Ý 1113.796413¨ 000000000000000b 0000000000634140 0000000000133058 000
0000032eab4a0
Ý 1113.796423¨ Krnl Code: 0000000000133066: e32010080020 cg %r2,8(%r
1)
000000000013306c: a78403fa brc 8,133860
#0000000000133070: a7f40001 brc 15,133072
>0000000000133074: e31003180004 lg %r1,792
000000000013307a: 5810101c l %r1,28(%r1)
000000000013307e: c01b01ffff00 nilf %r1,33554176
0000000000133084: a7740481 brc 7,133986
0000000000133088: e310a2bc0012 lt %r1,700(%r10)
Ý 1113.796447¨ Call Trace:
Ý 1113.796449¨ (Ý<0000000000133058>¨ do_exit+0x38/0xab8)
Ý 1113.796451¨ Ý<0000000000100ea4>¨ die+0x158/0x180
Ý 1113.796455¨ Ý<00000000006291fa>¨ do_trap+0x1a2/0x1a4
Ý 1113.796459¨ Ý<00000000006293ac>¨ illegal_op+0xfc/0x144
Ý 1113.796462¨ Ý<0000000000629974>¨ pgm_check_handler+0x138/0x13c
Ý 1113.796464¨ Ý<000000000030bc62>¨ mpage_da_submit_io+0x38e/0x3c0
Ý 1113.796467¨ (Ý<000000000030b9a8>¨ mpage_da_submit_io+0xd4/0x3c0)
Ý 1113.796470¨ Ý<000000000031204c>¨ mpage_da_map_and_submit+0x150/0x41c
Ý 1113.796472¨ Ý<0000000000312bac>¨ ext4_da_writepages+0x364/0x628
Ý 1113.829664¨ Ý<00000000002a08c8>¨ __writeback_single_inode+0x54/0x27c
Ý 1113.829674¨ Ý<00000000002a2c6c>¨ writeback_sb_inodes+0x284/0x4d8
Ý 1113.829681¨ Ý<00000000002a30b6>¨ wb_writeback+0x10e/0x374
Ý 1113.829691¨ Ý<00000000002a3d22>¨ wb_do_writeback+0x102/0x2e0
Ý 1113.829706¨ Ý<00000000002a3fa2>¨ bdi_writeback_thread+0xa2/0x270
Ý 1113.829715¨ Ý<00000000001581b2>¨ kthread+0xda/0xe4
Ý 1113.829725¨ Ý<0000000000629836>¨ kernel_thread_starter+0x6/0xc
Ý 1113.829736¨ Ý<0000000000629830>¨ kernel_thread_starter+0x0/0xc
Ý 1113.829753¨ Last Breaking-Event-Address:
Ý 1113.829759¨ Ý<0000000000133070>¨ do_exit+0x50/0xab8
Ý 1113.829771¨ ---Ý end trace 45df5089b835470f ¨---
----- Original Message -----
> From: "CAI Qian" <[email protected]>
> To: "LKML" <[email protected]>
> Cc: "linux-s390" <[email protected]>, "Steve Best" <[email protected]>, "Theodore Ts'o" <[email protected]>,
> [email protected]
> Sent: Thursday, March 28, 2013 2:40:33 PM
> Subject: s390x: kernel BUG at fs/ext4/inode.c:1591!
>
> System hung when running xfstests-dev 013 test case on an s390x
> guest. Never saw
> this on 3.9-rc3 before but need to double-check. Any idea?
Reproduced; bisect data so far,
# git bisect log
git bisect start
# good: [a937536b868b8369b98967929045f1df54234323] Linux 3.9-rc3
git bisect good a937536b868b8369b98967929045f1df54234323
# bad: [9064171268d838b8f283fe111ef086b9479d059a] Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/sfr/next-fixes
git bisect bad 9064171268d838b8f283fe111ef086b9479d059a
# bad: [38d78e587d4960d0db94add518d27ee74bad2301] mqueue: sys_mq_open: do not call mnt_drop_write() if read-only
git bisect bad 38d78e587d4960d0db94add518d27ee74bad2301
>
> CAI Qian
>
> Ý 1113.795759¨ ------------Ý cut here ¨------------
> Ý 1113.795771¨ kernel BUG at fs/ext4/inode.c:1591!
> Ý 1113.795845¨ illegal operation: 0001 Ý#1¨ SMP
> Ý 1113.795848¨ Modules linked in: nf_conntrack_netbios_ns
> nf_conntrack_broadcast
> ipt_MASQUERADE ip6table_mangle ip6t_REJECT nf_conntrack_ipv6
> nf_defrag_ipv6 ipt
> able_nat nf_nat_ipv4 nf_nat iptable_mangle ipt_REJECT
> nf_conntrack_ipv4 nf_defra
> g_ipv4 xt_conntrack nf_conntrack ebtable_filter ebtables
> ip6table_filter ip6_tab
> les iptable_filter ip_tables sg qeth_l2 vmur xfs libcrc32c
> dasd_fba_mod dasd_eck
> d_mod dasd_mod lcs ctcm qeth fsm qdio ccwgroup dm_mirror
> dm_region_hash dm_log d
> m_mod
> Ý 1113.795880¨ CPU: 1 Not tainted 3.9.0-rc4+ #2
> Ý 1113.795882¨ Process flush-253:1 (pid: 12418, task:
> 0000000003f04890, ksp: 000
> 0000032eab3b8)
> Ý 1113.795885¨ Krnl PSW : 0704e00180000000 000000000030bc62
> (mpage_da_submit_io+
> 0x38e/0x3c0)
> Ý 1113.795895¨ R:0 T:1 IO:1 EX:1 Key:0 M:1 W:0 P:0 AS:3
> CC:2 PM:0 EA:
> 3
> Krnl GPRS: 0000000001000000 000000001b5823a8 0000000000000010
> 000000001b5823a8
> Ý 1113.795900¨ 0000000000000000 00000000000000af
> 0000000000001000 000
> 0000032eab9c8
> Ý 1113.795902¨ 000003d10053bb40 0000000032eaba30
> 00000000000000a1 000
> 0000032eaba98
> Ý 1113.795905¨ 00000000000100b0 00000000000000a1
> 000000000030b9a8 000
> 0000032eab8a0
> Ý 1113.795921¨ Krnl Code: 000000000030bc54: f0dca7f4fecf srp
> 2036(14,
> %r10),3791(%r15),12
>
> MORE...
> ZVM-RHTS
> ================================================================================
> 000000000030bc5a: a7f40001 brc 15,30bc5c
> #000000000030bc5e: a7f40001 brc 15,30bc60
> >000000000030bc62: a7f40001 brc 15,30bc64
> 000000000030bc66: a7f40001 brc 15,30bc68
> 000000000030bc6a: 4120f0e8 la %r2,232(%r15)
> 000000000030bc6e: a7180000 lhi %r1,0
> Ý 1113.796303¨ Call Trace:
> Ý 1113.796306¨ (Ý<000000000030b9a8>¨ mpage_da_submit_io+0xd4/0x3c0)
> Ý 1113.796311¨ Ý<000000000031204c>¨
> mpage_da_map_and_submit+0x150/0x41c
> Ý 1113.796314¨ Ý<0000000000312bac>¨ ext4_da_writepages+0x364/0x628
> Ý 1113.796317¨ Ý<00000000002a08c8>¨
> __writeback_single_inode+0x54/0x27c
> Ý 1113.796322¨ Ý<00000000002a2c6c>¨ writeback_sb_inodes+0x284/0x4d8
> Ý 1113.796325¨ Ý<00000000002a30b6>¨ wb_writeback+0x10e/0x374
> Ý 1113.796327¨ Ý<00000000002a3d22>¨ wb_do_writeback+0x102/0x2e0
> Ý 1113.796330¨ Ý<00000000002a3fa2>¨ bdi_writeback_thread+0xa2/0x270
> Ý 1113.796333¨ Ý<00000000001581b2>¨ kthread+0xda/0xe4
> Ý 1113.796338¨ Ý<0000000000629836>¨ kernel_thread_starter+0x6/0xc
> Ý 1113.796342¨ Ý<0000000000629830>¨ kernel_thread_starter+0x0/0xc
> Ý 1113.796345¨ Last Breaking-Event-Address:
> Ý 1113.796346¨ Ý<000000000030bc5e>¨ mpage_da_submit_io+0x38a/0x3c0
> Ý 1113.796349¨
> Ý 1113.796351¨ ---Ý end trace 45df5089b835470e ¨---
> Ý 1113.796365¨ ------------Ý cut here ¨------------
> Ý 1113.796367¨ WARNING: at kernel/exit.c:715
> Ý 1113.796369¨ Modules linked in: nf_conntrack_netbios_ns
> nf_conntrack_broadcast
> ipt_MASQUERADE ip6table_mangle ip6t_REJECT nf_conntrack_ipv6
> nf_defrag_ipv6 ipt
> able_nat nf_nat_ipv4 nf_nat iptable_mangle ipt_REJECT
> nf_conntrack_ipv4 nf_defra
> g_ipv4 xt_conntrack nf_conntrack ebtable_filter ebtables
> ip6table_filter ip6_tab
> les iptable_filter ip_tables sg qeth_l2 vmur xfs libcrc32c
> dasd_fba_mod dasd_eck
> d_mod dasd_mod lcs ctcm qeth fsm qdio ccwgroup dm_mirror
> dm_region_hash dm_log d
> m_mod
> Ý 1113.796393¨ CPU: 1 Tainted: G D 3.9.0-rc4+ #2
> Ý 1113.796396¨ Process flush-253:1 (pid: 12418, task:
> 0000000003f04890, ksp: 000
> 0000032eab3b8)
> Ý 1113.796398¨ Krnl PSW : 0704e00180000000 0000000000133074
> (do_exit+0x54/0xab8)
>
> Ý 1113.796404¨ R:0 T:1 IO:1 EX:1 Key:0 M:1 W:0 P:0 AS:3
> CC:2 PM:0 EA:
> 3
> Krnl GPRS: 0000000000000015 0000000032eabb60 0000000032eabb68
> 0000000000000000
> Ý 1113.796408¨ 0000000000133058 ffffffffffffffff
> 0000000000000001 00
>
> MORE...
> ZVM-RHTS
> ================================================================================
> 0000032eab9c8
> Ý 1113.796411¨ 0704e00180000000 000000000030bc62
> 0000000003f04890 000
> 0000000778d38
> Ý 1113.796413¨ 000000000000000b 0000000000634140
> 0000000000133058 000
> 0000032eab4a0
> Ý 1113.796423¨ Krnl Code: 0000000000133066: e32010080020 cg
> %r2,8(%r
> 1)
> 000000000013306c: a78403fa brc 8,133860
> #0000000000133070: a7f40001 brc 15,133072
> >0000000000133074: e31003180004 lg %r1,792
> 000000000013307a: 5810101c l %r1,28(%r1)
> 000000000013307e: c01b01ffff00 nilf %r1,33554176
> 0000000000133084: a7740481 brc 7,133986
> 0000000000133088: e310a2bc0012 lt %r1,700(%r10)
> Ý 1113.796447¨ Call Trace:
> Ý 1113.796449¨ (Ý<0000000000133058>¨ do_exit+0x38/0xab8)
> Ý 1113.796451¨ Ý<0000000000100ea4>¨ die+0x158/0x180
> Ý 1113.796455¨ Ý<00000000006291fa>¨ do_trap+0x1a2/0x1a4
> Ý 1113.796459¨ Ý<00000000006293ac>¨ illegal_op+0xfc/0x144
> Ý 1113.796462¨ Ý<0000000000629974>¨ pgm_check_handler+0x138/0x13c
> Ý 1113.796464¨ Ý<000000000030bc62>¨ mpage_da_submit_io+0x38e/0x3c0
> Ý 1113.796467¨ (Ý<000000000030b9a8>¨ mpage_da_submit_io+0xd4/0x3c0)
> Ý 1113.796470¨ Ý<000000000031204c>¨
> mpage_da_map_and_submit+0x150/0x41c
> Ý 1113.796472¨ Ý<0000000000312bac>¨ ext4_da_writepages+0x364/0x628
> Ý 1113.829664¨ Ý<00000000002a08c8>¨
> __writeback_single_inode+0x54/0x27c
> Ý 1113.829674¨ Ý<00000000002a2c6c>¨ writeback_sb_inodes+0x284/0x4d8
> Ý 1113.829681¨ Ý<00000000002a30b6>¨ wb_writeback+0x10e/0x374
> Ý 1113.829691¨ Ý<00000000002a3d22>¨ wb_do_writeback+0x102/0x2e0
> Ý 1113.829706¨ Ý<00000000002a3fa2>¨ bdi_writeback_thread+0xa2/0x270
> Ý 1113.829715¨ Ý<00000000001581b2>¨ kthread+0xda/0xe4
> Ý 1113.829725¨ Ý<0000000000629836>¨ kernel_thread_starter+0x6/0xc
> Ý 1113.829736¨ Ý<0000000000629830>¨ kernel_thread_starter+0x0/0xc
> Ý 1113.829753¨ Last Breaking-Event-Address:
> Ý 1113.829759¨ Ý<0000000000133070>¨ do_exit+0x50/0xab8
> Ý 1113.829771¨ ---Ý end trace 45df5089b835470f ¨---
On Thu, Mar 28, 2013 at 02:40:33AM -0400, CAI Qian wrote:
> System hung when running xfstests-dev 013 test case on an s390x guest. Never saw
> this on 3.9-rc3 before but need to double-check. Any idea?
>
> ? 1113.795759? ------------? cut here ?------------
> ? 1113.795771? kernel BUG at fs/ext4/inode.c:1591!
thanks for the report. What kernel version did this come from? Was
it 3.9-rc4? (line 1591 for 3.9-rc3 doesn't contain a BUG_ON).
If it is indeed 3.9-rc4, it would be helpful, since you can reproduce
the problem, to insert a debugging printk which fires when
bh->b_blocknr != pblock before the BUG_ON, and have it print the
b_blocknr and pblock values.
Thanks,
- Ted
On Thu, 28 Mar 2013 08:05:17 -0400, Theodore Ts'o <[email protected]> wrote:
> On Thu, Mar 28, 2013 at 02:40:33AM -0400, CAI Qian wrote:
> > System hung when running xfstests-dev 013 test case on an s390x guest. Never saw
> > this on 3.9-rc3 before but need to double-check. Any idea?
> >
> > Ý 1113.795759¨ ------------Ý cut here ¨------------
> > Ý 1113.795771¨ kernel BUG at fs/ext4/inode.c:1591!
>
> thanks for the report. What kernel version did this come from? Was
> it 3.9-rc4? (line 1591 for 3.9-rc3 doesn't contain a BUG_ON).
>
> If it is indeed 3.9-rc4, it would be helpful, since you can reproduce
> the problem, to insert a debugging printk which fires when
> bh->b_blocknr != pblock before the BUG_ON, and have it print the
> b_blocknr and pblock values.
I've triggered this bug on before at the time i've worked on
e4defrag functionality, but AFAIK all related issues was aready fixed
and 013 has nothing with e4defrag.
But still bh->b_blocknr under us. So other obvious place I suspect is
puch_hole but this also not true because 013 use fsstress
test in vegetarian mode: "-f rmdir=10 -f link=10 -f creat=10 -f mkdir=10
-f rename=30 -f stat=30 -f unlink=30 -f truncate=20"
So the only place I suspect is some unknown bug in extent status tree
Can you please enable ES_AGGRESSIVE_TEST and rerun xfstest.
>
> Thanks,
>
> - Ted
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to [email protected]
> More majordomo info at http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at http://www.tux.org/lkml/
----- Original Message -----
> From: "Dmitry Monakhov" <[email protected]>
> To: "Theodore Ts'o" <[email protected]>, "CAI Qian" <[email protected]>
> Cc: "LKML" <[email protected]>, "linux-s390" <[email protected]>, "Steve Best"
> <[email protected]>, [email protected]
> Sent: Thursday, March 28, 2013 10:56:37 PM
> Subject: Re: s390x: kernel BUG at fs/ext4/inode.c:1591!
>
> On Thu, 28 Mar 2013 08:05:17 -0400, Theodore Ts'o <[email protected]>
> wrote:
> > On Thu, Mar 28, 2013 at 02:40:33AM -0400, CAI Qian wrote:
> > > System hung when running xfstests-dev 013 test case on an s390x
> > > guest. Never saw
> > > this on 3.9-rc3 before but need to double-check. Any idea?
> > >
> > > Ý 1113.795759¨ ------------Ý cut here ¨------------
> > > Ý 1113.795771¨ kernel BUG at fs/ext4/inode.c:1591!
> >
> > thanks for the report. What kernel version did this come from?
> > Was
> > it 3.9-rc4? (line 1591 for 3.9-rc3 doesn't contain a BUG_ON).
> >
> > If it is indeed 3.9-rc4, it would be helpful, since you can
> > reproduce
> > the problem, to insert a debugging printk which fires when
> > bh->b_blocknr != pblock before the BUG_ON, and have it print the
> > b_blocknr and pblock values.
> I've triggered this bug on before at the time i've worked on
> e4defrag functionality, but AFAIK all related issues was aready fixed
> and 013 has nothing with e4defrag.
> But still bh->b_blocknr under us. So other obvious place I suspect is
> puch_hole but this also not true because 013 use fsstress
> test in vegetarian mode: "-f rmdir=10 -f link=10 -f creat=10 -f
> mkdir=10
> -f rename=30 -f stat=30 -f unlink=30 -f truncate=20"
> So the only place I suspect is some unknown bug in extent status tree
> Can you please enable ES_AGGRESSIVE_TEST and rerun xfstest.
What is ES_AGGRESSIVE_TEST and how can it enable it?
> >
> > Thanks,
> >
> > - Ted
> > --
> > To unsubscribe from this list: send the line "unsubscribe
> > linux-kernel" in
> > the body of a message to [email protected]
> > More majordomo info at http://vger.kernel.org/majordomo-info.html
> > Please read the FAQ at http://www.tux.org/lkml/
>
----- Original Message -----
> From: "Theodore Ts'o" <[email protected]>
> To: "CAI Qian" <[email protected]>
> Cc: "LKML" <[email protected]>, "linux-s390" <[email protected]>, "Steve Best"
> <[email protected]>, [email protected]
> Sent: Thursday, March 28, 2013 8:05:17 PM
> Subject: Re: s390x: kernel BUG at fs/ext4/inode.c:1591!
>
> On Thu, Mar 28, 2013 at 02:40:33AM -0400, CAI Qian wrote:
> > System hung when running xfstests-dev 013 test case on an s390x
> > guest. Never saw
> > this on 3.9-rc3 before but need to double-check. Any idea?
> >
> > Ý 1113.795759¨ ------------Ý cut here ¨------------
> > Ý 1113.795771¨ kernel BUG at fs/ext4/inode.c:1591!
>
> thanks for the report. What kernel version did this come from? Was
> it 3.9-rc4? (line 1591 for 3.9-rc3 doesn't contain a BUG_ON).
Yes, the lastest mainline.
>
> If it is indeed 3.9-rc4, it would be helpful, since you can reproduce
> the problem, to insert a debugging printk which fires when
> bh->b_blocknr != pblock before the BUG_ON, and have it print the
> b_blocknr and pblock values.
bh->b_blocknr=100346
pblock=66797
Bisecting results so far,
git bisect start
# good: [a937536b868b8369b98967929045f1df54234323] Linux 3.9-rc3
git bisect good a937536b868b8369b98967929045f1df54234323
# bad: [9064171268d838b8f283fe111ef086b9479d059a] Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/sfr/next-fixes
git bisect bad 9064171268d838b8f283fe111ef086b9479d059a
# bad: [38d78e587d4960d0db94add518d27ee74bad2301] mqueue: sys_mq_open: do not call mnt_drop_write() if read-only
git bisect bad 38d78e587d4960d0db94add518d27ee74bad2301
# good: [e7489622d3603b7d161b484dcd340d9f678b0c7a] Merge tag 'arm64-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/cmarinas/linux-aarch64
git bisect good e7489622d3603b7d161b484dcd340d9f678b0c7a
# good: [172a271b5e090da7468c66b9ccbcdb3d929eed75] Merge branch 'drm-fixes' of git://people.freedesktop.org/~airlied/linux
git bisect good 172a271b5e090da7468c66b9ccbcdb3d929eed75
# good: [0a7e453103b9718d357688b83bb968ee108cc874] Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/rzhang/linux
git bisect good 0a7e453103b9718d357688b83bb968ee108cc874
# bad: [0e401101db49959f5783f6ee9e676124b5a183ac] ext4: fix memory leakage in mext_check_coverage
git bisect bad 0e401101db49959f5783f6ee9e676124b5a183ac
CAI Qian
>
> Thanks,
>
> - Ted
>
On Fri, 29 Mar 2013 04:53:43 -0400 (EDT), CAI Qian <[email protected]> wrote:
>
>
> ----- Original Message -----
> > From: "Dmitry Monakhov" <[email protected]>
> > To: "Theodore Ts'o" <[email protected]>, "CAI Qian" <[email protected]>
> > Cc: "LKML" <[email protected]>, "linux-s390" <[email protected]>, "Steve Best"
> > <[email protected]>, [email protected]
> > Sent: Thursday, March 28, 2013 10:56:37 PM
> > Subject: Re: s390x: kernel BUG at fs/ext4/inode.c:1591!
> >
> > On Thu, 28 Mar 2013 08:05:17 -0400, Theodore Ts'o <[email protected]>
> > wrote:
> > > On Thu, Mar 28, 2013 at 02:40:33AM -0400, CAI Qian wrote:
> > > > System hung when running xfstests-dev 013 test case on an s390x
> > > > guest. Never saw
> > > > this on 3.9-rc3 before but need to double-check. Any idea?
> > > >
> > > > Ý 1113.795759¨ ------------Ý cut here ¨------------
> > > > Ý 1113.795771¨ kernel BUG at fs/ext4/inode.c:1591!
> > >
> > > thanks for the report. What kernel version did this come from?
> > > Was
> > > it 3.9-rc4? (line 1591 for 3.9-rc3 doesn't contain a BUG_ON).
> > >
> > > If it is indeed 3.9-rc4, it would be helpful, since you can
> > > reproduce
> > > the problem, to insert a debugging printk which fires when
> > > bh->b_blocknr != pblock before the BUG_ON, and have it print the
> > > b_blocknr and pblock values.
> > I've triggered this bug on before at the time i've worked on
> > e4defrag functionality, but AFAIK all related issues was aready fixed
> > and 013 has nothing with e4defrag.
> > But still bh->b_blocknr under us. So other obvious place I suspect is
> > puch_hole but this also not true because 013 use fsstress
> > test in vegetarian mode: "-f rmdir=10 -f link=10 -f creat=10 -f
> > mkdir=10
> > -f rename=30 -f stat=30 -f unlink=30 -f truncate=20"
> > So the only place I suspect is some unknown bug in extent status tree
> > Can you please enable ES_AGGRESSIVE_TEST and rerun xfstest.
> What is ES_AGGRESSIVE_TEST and how can it enable it?
Please apply patch. It should helps to spot an issue
On Fri, 29 Mar 2013 05:27:02 -0400 (EDT), CAI Qian <[email protected]> wrote:
>
I've spent a half of weekend by trying to create s390x guest image,
without any success. Can you please share it.
>
> ----- Original Message -----
> > From: "Theodore Ts'o" <[email protected]>
> > To: "CAI Qian" <[email protected]>
> > Cc: "LKML" <[email protected]>, "linux-s390" <[email protected]>, "Steve Best"
> > <[email protected]>, [email protected]
> > Sent: Thursday, March 28, 2013 8:05:17 PM
> > Subject: Re: s390x: kernel BUG at fs/ext4/inode.c:1591!
> >
> > On Thu, Mar 28, 2013 at 02:40:33AM -0400, CAI Qian wrote:
> > > System hung when running xfstests-dev 013 test case on an s390x
> > > guest. Never saw
> > > this on 3.9-rc3 before but need to double-check. Any idea?
> > >
> > > Ý 1113.795759¨ ------------Ý cut here ¨------------
> > > Ý 1113.795771¨ kernel BUG at fs/ext4/inode.c:1591!
> >
> > thanks for the report. What kernel version did this come from? Was
> > it 3.9-rc4? (line 1591 for 3.9-rc3 doesn't contain a BUG_ON).
> Yes, the lastest mainline.
> >
> > If it is indeed 3.9-rc4, it would be helpful, since you can reproduce
> > the problem, to insert a debugging printk which fires when
> > bh->b_blocknr != pblock before the BUG_ON, and have it print the
> > b_blocknr and pblock values.
> bh->b_blocknr=100346
> pblock=66797
>
> Bisecting results so far,
> git bisect start
> # good: [a937536b868b8369b98967929045f1df54234323] Linux 3.9-rc3
> git bisect good a937536b868b8369b98967929045f1df54234323
> # bad: [9064171268d838b8f283fe111ef086b9479d059a] Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/sfr/next-fixes
> git bisect bad 9064171268d838b8f283fe111ef086b9479d059a
> # bad: [38d78e587d4960d0db94add518d27ee74bad2301] mqueue: sys_mq_open: do not call mnt_drop_write() if read-only
> git bisect bad 38d78e587d4960d0db94add518d27ee74bad2301
> # good: [e7489622d3603b7d161b484dcd340d9f678b0c7a] Merge tag 'arm64-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/cmarinas/linux-aarch64
> git bisect good e7489622d3603b7d161b484dcd340d9f678b0c7a
> # good: [172a271b5e090da7468c66b9ccbcdb3d929eed75] Merge branch 'drm-fixes' of git://people.freedesktop.org/~airlied/linux
> git bisect good 172a271b5e090da7468c66b9ccbcdb3d929eed75
> # good: [0a7e453103b9718d357688b83bb968ee108cc874] Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/rzhang/linux
> git bisect good 0a7e453103b9718d357688b83bb968ee108cc874
> # bad: [0e401101db49959f5783f6ee9e676124b5a183ac] ext4: fix memory leakage in mext_check_coverage
> git bisect bad 0e401101db49959f5783f6ee9e676124b5a183ac
> CAI Qian
> >
> > Thanks,
> >
> > - Ted
> >
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to [email protected]
> More majordomo info at http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at http://www.tux.org/lkml/
----- Original Message -----
> From: "Dmitry Monakhov" <[email protected]>
> To: "CAI Qian" <[email protected]>, "Theodore Ts'o" <[email protected]>
> Cc: "LKML" <[email protected]>, "linux-s390" <[email protected]>, "Steve Best"
> <[email protected]>, [email protected]
> Sent: Monday, April 1, 2013 2:07:35 PM
> Subject: Re: s390x: kernel BUG at fs/ext4/inode.c:1591!
>
> On Fri, 29 Mar 2013 05:27:02 -0400 (EDT), CAI Qian <[email protected]>
> wrote:
> >
> I've spent a half of weekend by trying to create s390x guest image,
> without any success. Can you please share it.
Well, do you have something like IBM system-Z?
http://en.wikipedia.org/wiki/IBM_System_z10
Also can be reproduced on system-p,
http://en.wikipedia.org/wiki/IBM_System_p
Never seen it on x86 so far though.
CAI Qian
> >
> > ----- Original Message -----
> > > From: "Theodore Ts'o" <[email protected]>
> > > To: "CAI Qian" <[email protected]>
> > > Cc: "LKML" <[email protected]>, "linux-s390"
> > > <[email protected]>, "Steve Best"
> > > <[email protected]>, [email protected]
> > > Sent: Thursday, March 28, 2013 8:05:17 PM
> > > Subject: Re: s390x: kernel BUG at fs/ext4/inode.c:1591!
> > >
> > > On Thu, Mar 28, 2013 at 02:40:33AM -0400, CAI Qian wrote:
> > > > System hung when running xfstests-dev 013 test case on an s390x
> > > > guest. Never saw
> > > > this on 3.9-rc3 before but need to double-check. Any idea?
> > > >
> > > > Ý 1113.795759¨ ------------Ý cut here ¨------------
> > > > Ý 1113.795771¨ kernel BUG at fs/ext4/inode.c:1591!
> > >
> > > thanks for the report. What kernel version did this come from? Was
> > > it 3.9-rc4? (line 1591 for 3.9-rc3 doesn't contain a BUG_ON).
> > Yes, the lastest mainline.
> > >
> > > If it is indeed 3.9-rc4, it would be helpful, since you can reproduce
> > > the problem, to insert a debugging printk which fires when
> > > bh->b_blocknr != pblock before the BUG_ON, and have it print the
> > > b_blocknr and pblock values.
> > bh->b_blocknr=100346
> > pblock=66797
> >
> > Bisecting results so far,
> > git bisect start
> > # good: [a937536b868b8369b98967929045f1df54234323] Linux 3.9-rc3
> > git bisect good a937536b868b8369b98967929045f1df54234323
> > # bad: [9064171268d838b8f283fe111ef086b9479d059a] Merge tag 'for-linus' of
> > git://git.kernel.org/pub/scm/linux/kernel/git/sfr/next-fixes
> > git bisect bad 9064171268d838b8f283fe111ef086b9479d059a
> > # bad: [38d78e587d4960d0db94add518d27ee74bad2301] mqueue: sys_mq_open: do
> > not call mnt_drop_write() if read-only
> > git bisect bad 38d78e587d4960d0db94add518d27ee74bad2301
> > # good: [e7489622d3603b7d161b484dcd340d9f678b0c7a] Merge tag 'arm64-fixes'
> > of git://git.kernel.org/pub/scm/linux/kernel/git/cmarinas/linux-aarch64
> > git bisect good e7489622d3603b7d161b484dcd340d9f678b0c7a
> > # good: [172a271b5e090da7468c66b9ccbcdb3d929eed75] Merge branch 'drm-fixes'
> > of git://people.freedesktop.org/~airlied/linux
> > git bisect good 172a271b5e090da7468c66b9ccbcdb3d929eed75
> > # good: [0a7e453103b9718d357688b83bb968ee108cc874] Merge branch 'next' of
> > git://git.kernel.org/pub/scm/linux/kernel/git/rzhang/linux
> > git bisect good 0a7e453103b9718d357688b83bb968ee108cc874
> > # bad: [0e401101db49959f5783f6ee9e676124b5a183ac] ext4: fix memory leakage
> > in mext_check_coverage
> > git bisect bad 0e401101db49959f5783f6ee9e676124b5a183ac
> > CAI Qian
> > >
> > > Thanks,
> > >
> > > - Ted
> > >
> > --
> > To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> > the body of a message to [email protected]
> > More majordomo info at http://vger.kernel.org/majordomo-info.html
> > Please read the FAQ at http://www.tux.org/lkml/
>
On Mon, 1 Apr 2013 02:30:47 -0400 (EDT), CAI Qian <[email protected]> wrote:
>
>
> ----- Original Message -----
> > From: "Dmitry Monakhov" <[email protected]>
> > To: "CAI Qian" <[email protected]>, "Theodore Ts'o" <[email protected]>
> > Cc: "LKML" <[email protected]>, "linux-s390" <[email protected]>, "Steve Best"
> > <[email protected]>, [email protected]
> > Sent: Monday, April 1, 2013 2:07:35 PM
> > Subject: Re: s390x: kernel BUG at fs/ext4/inode.c:1591!
> >
> > On Fri, 29 Mar 2013 05:27:02 -0400 (EDT), CAI Qian <[email protected]>
> > wrote:
> > >
> > I've spent a half of weekend by trying to create s390x guest image,
> > without any success. Can you please share it.
> Well, do you have something like IBM system-Z?
> http://en.wikipedia.org/wiki/IBM_System_z10
>
> Also can be reproduced on system-p,
> http://en.wikipedia.org/wiki/IBM_System_p
>
Ohh... So you are lucky user of this black rocks and you
are the only person who can help to reproduce the issue.
BTW: Have you run kernel with the patch i've sent you?
> Never seen it on x86 so far though.
> CAI Qian
> > >
> > > ----- Original Message -----
> > > > From: "Theodore Ts'o" <[email protected]>
> > > > To: "CAI Qian" <[email protected]>
> > > > Cc: "LKML" <[email protected]>, "linux-s390"
> > > > <[email protected]>, "Steve Best"
> > > > <[email protected]>, [email protected]
> > > > Sent: Thursday, March 28, 2013 8:05:17 PM
> > > > Subject: Re: s390x: kernel BUG at fs/ext4/inode.c:1591!
> > > >
> > > > On Thu, Mar 28, 2013 at 02:40:33AM -0400, CAI Qian wrote:
> > > > > System hung when running xfstests-dev 013 test case on an s390x
> > > > > guest. Never saw
> > > > > this on 3.9-rc3 before but need to double-check. Any idea?
> > > > >
> > > > > Ý 1113.795759¨ ------------Ý cut here ¨------------
> > > > > Ý 1113.795771¨ kernel BUG at fs/ext4/inode.c:1591!
> > > >
> > > > thanks for the report. What kernel version did this come from? Was
> > > > it 3.9-rc4? (line 1591 for 3.9-rc3 doesn't contain a BUG_ON).
> > > Yes, the lastest mainline.
> > > >
> > > > If it is indeed 3.9-rc4, it would be helpful, since you can reproduce
> > > > the problem, to insert a debugging printk which fires when
> > > > bh->b_blocknr != pblock before the BUG_ON, and have it print the
> > > > b_blocknr and pblock values.
> > > bh->b_blocknr=100346
> > > pblock=66797
> > >
> > > Bisecting results so far,
> > > git bisect start
> > > # good: [a937536b868b8369b98967929045f1df54234323] Linux 3.9-rc3
> > > git bisect good a937536b868b8369b98967929045f1df54234323
> > > # bad: [9064171268d838b8f283fe111ef086b9479d059a] Merge tag 'for-linus' of
> > > git://git.kernel.org/pub/scm/linux/kernel/git/sfr/next-fixes
> > > git bisect bad 9064171268d838b8f283fe111ef086b9479d059a
> > > # bad: [38d78e587d4960d0db94add518d27ee74bad2301] mqueue: sys_mq_open: do
> > > not call mnt_drop_write() if read-only
> > > git bisect bad 38d78e587d4960d0db94add518d27ee74bad2301
> > > # good: [e7489622d3603b7d161b484dcd340d9f678b0c7a] Merge tag 'arm64-fixes'
> > > of git://git.kernel.org/pub/scm/linux/kernel/git/cmarinas/linux-aarch64
> > > git bisect good e7489622d3603b7d161b484dcd340d9f678b0c7a
> > > # good: [172a271b5e090da7468c66b9ccbcdb3d929eed75] Merge branch 'drm-fixes'
> > > of git://people.freedesktop.org/~airlied/linux
> > > git bisect good 172a271b5e090da7468c66b9ccbcdb3d929eed75
> > > # good: [0a7e453103b9718d357688b83bb968ee108cc874] Merge branch 'next' of
> > > git://git.kernel.org/pub/scm/linux/kernel/git/rzhang/linux
> > > git bisect good 0a7e453103b9718d357688b83bb968ee108cc874
> > > # bad: [0e401101db49959f5783f6ee9e676124b5a183ac] ext4: fix memory leakage
> > > in mext_check_coverage
> > > git bisect bad 0e401101db49959f5783f6ee9e676124b5a183ac
> > > CAI Qian
> > > >
> > > > Thanks,
> > > >
> > > > - Ted
> > > >
> > > --
> > > To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> > > the body of a message to [email protected]
> > > More majordomo info at http://vger.kernel.org/majordomo-info.html
> > > Please read the FAQ at http://www.tux.org/lkml/
> >
Bisect indicated this is the culprit,
0e401101db49959f5783f6ee9e676124b5a183ac
ext4: fix memory leakage in mext_check_coverage
This following with Dmitry's debug patch applied,
CAI Qian
Ý 101.408610¨ ES cache assertation failed for inode: 753 es_cached ex Ý56/5/744
81/20¨ != found ex Ý56/5/3396400/0¨ retval 0 flags 5
Ý 209.858899¨ ES cache assertation failed for inode: 384 es_cached ex Ý57/7/332
82/20¨ != found ex Ý57/7/3396400/0¨ retval 0 flags 5
Ý 209.860656¨ ES cache assertation failed for inode: 384 es_cached ex Ý25/1/332
50/20¨ != found ex Ý25/1/0/0¨ retval 0 flags 0
Ý 209.893587¨ ES cache assertation failed for inode: 384 es_cached ex Ý22/1/332
47/20¨ != found ex Ý22/1/34838/1000¨ retval 1 flags 0
Ý 209.913482¨ ES cache assertation failed for inode: 384 es_cached ex Ý27/1/329
40/20¨ != found ex Ý27/1/0/0¨ retval 0 flags 0
Ý 209.919950¨ ES cache assertation failed for inode: 384 es_cached ex Ý59/5/338
48/20¨ != found ex Ý59/5/3396400/0¨ retval 0 flags 5
Ý 209.931856¨ ES cache assertation failed for inode: 384 es_cached ex Ý7/1/3292
0/20¨ != found ex Ý7/1/35879/20¨ retval 1 flags 43
Ý 209.969282¨ ES cache assertation failed for inode: 384 es_cached ex Ý35/1/361
97/20¨ != found ex Ý35/1/36197/1000¨ retval 1 flags 0
Ý 209.969290¨ ES cache assertation failed for inode: 384 es_cached ex Ý48/1/362
10/20¨ != found ex Ý48/1/0/0¨ retval 0 flags 0
Ý 209.980724¨ ES cache assertation failed for inode: 384 es_cached ex Ý13/4/334
89/20¨ != found ex Ý13/4/2161372/0¨ retval 0 flags 5
Ý 209.980744¨ ES cache assertation failed for inode: 384 es_cached ex Ý61/3/335
37/20¨ != found ex Ý61/3/3396400/0¨ retval 0 flags 5
Ý 209.983848¨ ES cache assertation failed for inode: 384 es_cached ex Ý44/2/335
20/20¨ != found ex Ý44/2/36216/20¨ retval 2 flags 43
Ý 210.020041¨ ES cache assertation failed for inode: 384 es_cached ex Ý61/3/341
91/20¨ != found ex Ý61/3/3396400/0¨ retval 0 flags 5
Ý 210.050100¨ ES cache assertation failed for inode: 384 es_cached ex Ý22/11/34
565/20¨ != found ex Ý22/11/3396400/0¨ retval 0 flags 5
Ý 210.053271¨ ES cache assertation failed for inode: 384 es_cached ex Ý15/1/334
90/20¨ != found ex Ý15/1/33579/1000¨ retval 1 flags 1
Ý 210.053275¨ mpage_da_submit_io failed block=33490 != b_blocknr=33579
Ý 210.053277¨ ino:384 lbkl:15, b_state=0x00001023, b_size=4096
Ý 210.053320¨ ------------Ý cut here ¨------------
Ý 210.053323¨ kernel BUG at fs/ext4/inode.c:1639!
Ý 210.053402¨ illegal operation: 0001 Ý#1¨ SMP
Ý 210.053405¨ Modules linked in: nf_conntrack_netbios_ns nf_conntrack_broadcast
ipt_MASQUERADE ip6table_mangle ip6t_REJECT nf_conntrack_ipv6 nf_defrag_ipv6 ipt
able_nat nf_nat_ipv4 nf_nat iptable_mangle ipt_REJECT nf_conntrack_ipv4 nf_defra
g_ipv4 xt_conntrack nf_conntrack ebtable_filter ebtables ip6table_filter ip6_tab
les iptable_filter ip_tables sg qeth_l2 vmur xfs libcrc32c dasd_fba_mod dasd_eck
d_mod lcs dasd_mod qeth ctcm qdio ccwgroup fsm dm_mirror dm_region_hash dm_log d
m_mod
Ý 210.053434¨ CPU: 0 Not tainted 3.8.0-rc3+ #16
Ý 210.053436¨ Process fsx (pid: 20565, task: 000000002c358000, ksp: 000000002c0
8f480)
Ý 210.053439¨ Krnl PSW : 0704f00180000000 00000000003033e8 (mpage_da_submit_io
0x3d4/0x408)
Ý 210.053450¨ R:0 T:1 IO:1 EX:1 Key:0 M:1 W:0 P:0 AS:3 CC:3 PM:0 EA:
3
Krnl GPRS: 0000000000000015 0000000000000001 0000000000000030 00000000031b4508
Ý 210.053455¨ 00000000003033e4 0000000000000000 0000000000001000 000
7ffff00001000
Ý 210.053457¨ 000000002c08fa98 000003d100a8c6c0 000000002c08fb68 000
000000000000f
Ý 210.053460¨ 00000000000082d2 000000002204d068 00000000003033e4 000
000002c08f970
Ý 210.053473¨ Krnl Code: 00000000003033d8: c02000215447 larl %r2,72dc
66
00000000003033de: c0e50016788f brasl %r14,5d24fc
#00000000003033e4: a7f40001 brc 15,3033e6
>00000000003033e8: a7f40001 brc 15,3033ea
00000000003033ec: a7f40001 brc 15,3033ee
00000000003033f0: 4120f0e8 la %r2,232(%r15)
00000000003033f4: a7180000 lhi %r1,0
00000000003033f8: 5010f0d8 st %r1,216(%r15)
Ý 210.053497¨ Call Trace:
Ý 210.053498¨ (Ý<00000000003033e4>¨ mpage_da_submit_io+0x3d0/0x408)
Ý 210.053501¨ Ý<0000000000309a48>¨ mpage_da_map_and_submit+0x150/0x41c
Ý 210.053505¨ Ý<000000000030a212>¨ write_cache_pages_da+0x4fe/0x530
Ý 210.053509¨ Ý<000000000030a584>¨ ext4_da_writepages+0x340/0x628
Ý 210.053512¨ Ý<00000000002024d2>¨ __filemap_fdatawrite_range+0x6e/0x7c
Ý 210.053518¨ Ý<00000000002025fc>¨ filemap_write_and_wait_range+0x54/0x8c
Ý 210.053521¨ Ý<00000000002fe0f8>¨ ext4_sync_file+0x7c/0x3d8
Ý 210.053524¨ Ý<000000000023c932>¨ SyS_msync+0x14e/0x1d8
Ý 210.053528¨ Ý<00000000005de66e>¨ sysc_tracego+0x14/0x1a
Ý 210.053533¨ Ý<000003fffd0e1240>¨ 0x3fffd0e1240
Ý 210.053536¨ Last Breaking-Event-Address:
Ý 210.053537¨ Ý<00000000003033e4>¨ mpage_da_submit_io+0x3d0/0x408
Ý 210.053540¨
Ý 210.053542¨ ---Ý end trace f387176e9fcb98d0 ¨---
Ý 210.053546¨ ------------Ý cut here ¨------------
Ý 210.053548¨ WARNING: at kernel/exit.c:713
Ý 210.053550¨ Modules linked in: nf_conntrack_netbios_ns nf_conntrack_broadcast
ipt_MASQUERADE ip6table_mangle ip6t_REJECT nf_conntrack_ipv6 nf_defrag_ipv6 ipt
able_nat nf_nat_ipv4 nf_nat iptable_mangle ipt_REJECT nf_conntrack_ipv4 nf_defra
g_ipv4 xt_conntrack nf_conntrack ebtable_filter ebtables ip6table_filter ip6_tab
les iptable_filter ip_tables sg qeth_l2 vmur xfs libcrc32c dasd_fba_mod dasd_eck
d_mod lcs dasd_mod qeth ctcm qdio ccwgroup fsm dm_mirror dm_region_hash dm_log d
m_mod
Ý 210.053571¨ CPU: 0 Tainted: G D 3.8.0-rc3+ #16
It just disable es_cache lookup feature should. This should helps to
determine whenever this is a es_cache issue or not.
> Ý 210.053275¨ mpage_da_submit_io failed block=33490 != b_blocknr=33579
> Ý 210.053277¨ ino:384 lbkl:15, b_state=0x00001023, b_size=4096
> Ý 210.053320¨ ------------Ý cut here ¨------------
> Ý 210.053323¨ kernel BUG at fs/ext4/inode.c:1639!
> Ý 210.053402¨ illegal operation: 0001 Ý#1¨ SMP
> Ý 210.053405¨ Modules linked in: nf_conntrack_netbios_ns nf_conntrack_broadcast
> ipt_MASQUERADE ip6table_mangle ip6t_REJECT nf_conntrack_ipv6 nf_defrag_ipv6 ipt
> able_nat nf_nat_ipv4 nf_nat iptable_mangle ipt_REJECT nf_conntrack_ipv4 nf_defra
> g_ipv4 xt_conntrack nf_conntrack ebtable_filter ebtables ip6table_filter ip6_tab
> les iptable_filter ip_tables sg qeth_l2 vmur xfs libcrc32c dasd_fba_mod dasd_eck
> d_mod lcs dasd_mod qeth ctcm qdio ccwgroup fsm dm_mirror dm_region_hash dm_log d
> m_mod
> Ý 210.053434¨ CPU: 0 Not tainted 3.8.0-rc3+ #16
> Ý 210.053436¨ Process fsx (pid: 20565, task: 000000002c358000, ksp: 000000002c0
> 8f480)
> Ý 210.053439¨ Krnl PSW : 0704f00180000000 00000000003033e8 (mpage_da_submit_io
> 0x3d4/0x408)
> Ý 210.053450¨ R:0 T:1 IO:1 EX:1 Key:0 M:1 W:0 P:0 AS:3 CC:3 PM:0 EA:
> 3
> Krnl GPRS: 0000000000000015 0000000000000001 0000000000000030 00000000031b4508
> Ý 210.053455¨ 00000000003033e4 0000000000000000 0000000000001000 000
> 7ffff00001000
> Ý 210.053457¨ 000000002c08fa98 000003d100a8c6c0 000000002c08fb68 000
> 000000000000f
> Ý 210.053460¨ 00000000000082d2 000000002204d068 00000000003033e4 000
> 000002c08f970
> Ý 210.053473¨ Krnl Code: 00000000003033d8: c02000215447 larl %r2,72dc
> 66
> 00000000003033de: c0e50016788f brasl %r14,5d24fc
> #00000000003033e4: a7f40001 brc 15,3033e6
> >00000000003033e8: a7f40001 brc 15,3033ea
> 00000000003033ec: a7f40001 brc 15,3033ee
> 00000000003033f0: 4120f0e8 la %r2,232(%r15)
> 00000000003033f4: a7180000 lhi %r1,0
> 00000000003033f8: 5010f0d8 st %r1,216(%r15)
> Ý 210.053497¨ Call Trace:
> Ý 210.053498¨ (Ý<00000000003033e4>¨ mpage_da_submit_io+0x3d0/0x408)
> Ý 210.053501¨ Ý<0000000000309a48>¨ mpage_da_map_and_submit+0x150/0x41c
> Ý 210.053505¨ Ý<000000000030a212>¨ write_cache_pages_da+0x4fe/0x530
> Ý 210.053509¨ Ý<000000000030a584>¨ ext4_da_writepages+0x340/0x628
> Ý 210.053512¨ Ý<00000000002024d2>¨ __filemap_fdatawrite_range+0x6e/0x7c
> Ý 210.053518¨ Ý<00000000002025fc>¨ filemap_write_and_wait_range+0x54/0x8c
> Ý 210.053521¨ Ý<00000000002fe0f8>¨ ext4_sync_file+0x7c/0x3d8
> Ý 210.053524¨ Ý<000000000023c932>¨ SyS_msync+0x14e/0x1d8
> Ý 210.053528¨ Ý<00000000005de66e>¨ sysc_tracego+0x14/0x1a
> Ý 210.053533¨ Ý<000003fffd0e1240>¨ 0x3fffd0e1240
> Ý 210.053536¨ Last Breaking-Event-Address:
> Ý 210.053537¨ Ý<00000000003033e4>¨ mpage_da_submit_io+0x3d0/0x408
> Ý 210.053540¨
> Ý 210.053542¨ ---Ý end trace f387176e9fcb98d0 ¨---
> Ý 210.053546¨ ------------Ý cut here ¨------------
> Ý 210.053548¨ WARNING: at kernel/exit.c:713
> Ý 210.053550¨ Modules linked in: nf_conntrack_netbios_ns nf_conntrack_broadcast
> ipt_MASQUERADE ip6table_mangle ip6t_REJECT nf_conntrack_ipv6 nf_defrag_ipv6 ipt
> able_nat nf_nat_ipv4 nf_nat iptable_mangle ipt_REJECT nf_conntrack_ipv4 nf_defra
> g_ipv4 xt_conntrack nf_conntrack ebtable_filter ebtables ip6table_filter ip6_tab
> les iptable_filter ip_tables sg qeth_l2 vmur xfs libcrc32c dasd_fba_mod dasd_eck
> d_mod lcs dasd_mod qeth ctcm qdio ccwgroup fsm dm_mirror dm_region_hash dm_log d
> m_mod
> Ý 210.053571¨ CPU: 0 Tainted: G D 3.8.0-rc3+ #16
> > [Text Documents:disable-es_lookup_extent.patch]
With this patch, I cannot reproduce it any more.
CAI Qian
On Wed, 3 Apr 2013 03:14:44 -0400 (EDT), CAI Qian <[email protected]> wrote:
>
> > > [Text Documents:disable-es_lookup_extent.patch]
> With this patch, I cannot reproduce it any more.
Ok that is second confirmation that bug caused by issue in es_cache,
This is not a fix though. So still try to investigate that.
>
> CAI Qian
On Wed, 3 Apr 2013, CAI Qian wrote:
> Date: Wed, 3 Apr 2013 00:00:17 -0400 (EDT)
> From: CAI Qian <[email protected]>
> To: Dmitry Monakhov <[email protected]>
> Cc: Theodore Ts'o <[email protected]>, LKML <[email protected]>,
> linux-s390 <[email protected]>, Steve Best <[email protected]>,
> [email protected], Lukas Czerner <[email protected]>,
> [email protected]
> Subject: Re: bisected! (WAS Re: s390x: kernel BUG at fs/ext4/inode.c:1591!)
>
>
>
> ----- Original Message -----
> > From: "Dmitry Monakhov" <[email protected]>
> > To: "CAI Qian" <[email protected]>
> > Cc: "Theodore Ts'o" <[email protected]>, "LKML" <[email protected]>, "linux-s390"
> > <[email protected]>, "Steve Best" <[email protected]>, [email protected]
> > Sent: Tuesday, April 2, 2013 6:01:36 PM
> > Subject: Re: bisected! (WAS Re: s390x: kernel BUG at fs/ext4/inode.c:1591!)
> >
> > On Tue, 2 Apr 2013 00:06:24 -0400 (EDT), CAI Qian <[email protected]> wrote:
> > > Bisect indicated this is the culprit,
> > >
> > > 0e401101db49959f5783f6ee9e676124b5a183ac
> > > ext4: fix memory leakage in mext_check_coverage
> > Strange...
> > It changes a bug in move_extent.c (e4defrag functionality)
> > ASAIU you just previously stopped your bisecting process here. Right?
> > Is this indeed a first bad commit?
> Hmm, bisect went wrong in the first place. Now double-confirmed this is
> the culprit,
>
> 4f42f80a8f08d4c3f52c4267361241885d5dee3a
> ext4: use s_extent_max_zeroout_kb value as number of kb
With this commit we're zeroing parts of uninitialized extents when
converting uninitialized extents to initialized as we should. This is
unlikely to be real cause, though it probably uncover some another bug which
we could not notice before.
-Lukas
>
> > >
> > > This following with Dmitry's debug patch applied,
> > >
> > > CAI Qian
> > >
> > > Ý 101.408610¨ ES cache assertation failed for inode: 753 es_cached ex
> > > Ý56/5/744
> > > 81/20¨ != found ex Ý56/5/3396400/0¨ retval 0 flags 5
> > > Ý 209.858899¨ ES cache assertation failed for inode: 384 es_cached ex
> > > Ý57/7/332
> > > 82/20¨ != found ex Ý57/7/3396400/0¨ retval 0 flags 5
> > > Ý 209.860656¨ ES cache assertation failed for inode: 384 es_cached ex
> > > Ý25/1/332
> > > 50/20¨ != found ex Ý25/1/0/0¨ retval 0 flags 0
> > > Ý 209.893587¨ ES cache assertation failed for inode: 384 es_cached ex
> > > Ý22/1/332
> > > 47/20¨ != found ex Ý22/1/34838/1000¨ retval 1 flags 0
> > > Ý 209.913482¨ ES cache assertation failed for inode: 384 es_cached ex
> > > Ý27/1/329
> > > 40/20¨ != found ex Ý27/1/0/0¨ retval 0 flags 0
> > > Ý 209.919950¨ ES cache assertation failed for inode: 384 es_cached ex
> > > Ý59/5/338
> > > 48/20¨ != found ex Ý59/5/3396400/0¨ retval 0 flags 5
> > > Ý 209.931856¨ ES cache assertation failed for inode: 384 es_cached ex
> > > Ý7/1/3292
> > > 0/20¨ != found ex Ý7/1/35879/20¨ retval 1 flags 43
> > > Ý 209.969282¨ ES cache assertation failed for inode: 384 es_cached ex
> > > Ý35/1/361
> > > 97/20¨ != found ex Ý35/1/36197/1000¨ retval 1 flags 0
> > > Ý 209.969290¨ ES cache assertation failed for inode: 384 es_cached ex
> > > Ý48/1/362
> > > 10/20¨ != found ex Ý48/1/0/0¨ retval 0 flags 0
> > > Ý 209.980724¨ ES cache assertation failed for inode: 384 es_cached ex
> > > Ý13/4/334
> > > 89/20¨ != found ex Ý13/4/2161372/0¨ retval 0 flags 5
> > > Ý 209.980744¨ ES cache assertation failed for inode: 384 es_cached ex
> > > Ý61/3/335
> > > 37/20¨ != found ex Ý61/3/3396400/0¨ retval 0 flags 5
> > > Ý 209.983848¨ ES cache assertation failed for inode: 384 es_cached ex
> > > Ý44/2/335
> > > 20/20¨ != found ex Ý44/2/36216/20¨ retval 2 flags 43
> > > Ý 210.020041¨ ES cache assertation failed for inode: 384 es_cached ex
> > > Ý61/3/341
> > > 91/20¨ != found ex Ý61/3/3396400/0¨ retval 0 flags 5
> > > Ý 210.050100¨ ES cache assertation failed for inode: 384 es_cached ex
> > > Ý22/11/34
> > > 565/20¨ != found ex Ý22/11/3396400/0¨ retval 0 flags 5
> > > Ý 210.053271¨ ES cache assertation failed for inode: 384 es_cached ex
> > > Ý15/1/334
> > > 90/20¨ != found ex Ý15/1/33579/1000¨ retval 1 flags 1
> > It does not looks like bigendian issue, actually I cant find any logical
> > system in the log. The only thing I see is that es_cache is
> > horribly out of sync with extent_tree.
> > Please try this patch:
> I'll test that shortly.
> CAI Qian
> >
> >
> > [Text Documents:disable-es_lookup_extent.patch]
> >
> >
> > It just disable es_cache lookup feature should. This should helps to
> > determine whenever this is a es_cache issue or not.
> > > Ý 210.053275¨ mpage_da_submit_io failed block=33490 != b_blocknr=33579
> > > Ý 210.053277¨ ino:384 lbkl:15, b_state=0x00001023, b_size=4096
> > > Ý 210.053320¨ ------------Ý cut here ¨------------
> > > Ý 210.053323¨ kernel BUG at fs/ext4/inode.c:1639!
> > > Ý 210.053402¨ illegal operation: 0001 Ý#1¨ SMP
> > > Ý 210.053405¨ Modules linked in: nf_conntrack_netbios_ns
> > > nf_conntrack_broadcast
> > > ipt_MASQUERADE ip6table_mangle ip6t_REJECT nf_conntrack_ipv6
> > > nf_defrag_ipv6 ipt
> > > able_nat nf_nat_ipv4 nf_nat iptable_mangle ipt_REJECT nf_conntrack_ipv4
> > > nf_defra
> > > g_ipv4 xt_conntrack nf_conntrack ebtable_filter ebtables ip6table_filter
> > > ip6_tab
> > > les iptable_filter ip_tables sg qeth_l2 vmur xfs libcrc32c dasd_fba_mod
> > > dasd_eck
> > > d_mod lcs dasd_mod qeth ctcm qdio ccwgroup fsm dm_mirror dm_region_hash
> > > dm_log d
> > > m_mod
> > > Ý 210.053434¨ CPU: 0 Not tainted 3.8.0-rc3+ #16
> > > Ý 210.053436¨ Process fsx (pid: 20565, task: 000000002c358000, ksp:
> > > 000000002c0
> > > 8f480)
> > > Ý 210.053439¨ Krnl PSW : 0704f00180000000 00000000003033e8
> > > (mpage_da_submit_io
> > > 0x3d4/0x408)
> > > Ý 210.053450¨ R:0 T:1 IO:1 EX:1 Key:0 M:1 W:0 P:0 AS:3 CC:3
> > > PM:0 EA:
> > > 3
> > > Krnl GPRS: 0000000000000015 0000000000000001 0000000000000030
> > > 00000000031b4508
> > > Ý 210.053455¨ 00000000003033e4 0000000000000000
> > > 0000000000001000 000
> > > 7ffff00001000
> > > Ý 210.053457¨ 000000002c08fa98 000003d100a8c6c0
> > > 000000002c08fb68 000
> > > 000000000000f
> > > Ý 210.053460¨ 00000000000082d2 000000002204d068
> > > 00000000003033e4 000
> > > 000002c08f970
> > > Ý 210.053473¨ Krnl Code: 00000000003033d8: c02000215447 larl
> > > %r2,72dc
> > > 66
> > > 00000000003033de: c0e50016788f brasl %r14,5d24fc
> > > #00000000003033e4: a7f40001 brc 15,3033e6
> > > >00000000003033e8: a7f40001 brc 15,3033ea
> > > 00000000003033ec: a7f40001 brc 15,3033ee
> > > 00000000003033f0: 4120f0e8 la %r2,232(%r15)
> > > 00000000003033f4: a7180000 lhi %r1,0
> > > 00000000003033f8: 5010f0d8 st %r1,216(%r15)
> > > Ý 210.053497¨ Call Trace:
> > > Ý 210.053498¨ (Ý<00000000003033e4>¨ mpage_da_submit_io+0x3d0/0x408)
> > > Ý 210.053501¨ Ý<0000000000309a48>¨ mpage_da_map_and_submit+0x150/0x41c
> > > Ý 210.053505¨ Ý<000000000030a212>¨ write_cache_pages_da+0x4fe/0x530
> > > Ý 210.053509¨ Ý<000000000030a584>¨ ext4_da_writepages+0x340/0x628
> > > Ý 210.053512¨ Ý<00000000002024d2>¨ __filemap_fdatawrite_range+0x6e/0x7c
> > > Ý 210.053518¨ Ý<00000000002025fc>¨ filemap_write_and_wait_range+0x54/0x8c
> > > Ý 210.053521¨ Ý<00000000002fe0f8>¨ ext4_sync_file+0x7c/0x3d8
> > > Ý 210.053524¨ Ý<000000000023c932>¨ SyS_msync+0x14e/0x1d8
> > > Ý 210.053528¨ Ý<00000000005de66e>¨ sysc_tracego+0x14/0x1a
> > > Ý 210.053533¨ Ý<000003fffd0e1240>¨ 0x3fffd0e1240
> > > Ý 210.053536¨ Last Breaking-Event-Address:
> > > Ý 210.053537¨ Ý<00000000003033e4>¨ mpage_da_submit_io+0x3d0/0x408
> > > Ý 210.053540¨
> > > Ý 210.053542¨ ---Ý end trace f387176e9fcb98d0 ¨---
> > > Ý 210.053546¨ ------------Ý cut here ¨------------
> > > Ý 210.053548¨ WARNING: at kernel/exit.c:713
> > > Ý 210.053550¨ Modules linked in: nf_conntrack_netbios_ns
> > > nf_conntrack_broadcast
> > > ipt_MASQUERADE ip6table_mangle ip6t_REJECT nf_conntrack_ipv6
> > > nf_defrag_ipv6 ipt
> > > able_nat nf_nat_ipv4 nf_nat iptable_mangle ipt_REJECT nf_conntrack_ipv4
> > > nf_defra
> > > g_ipv4 xt_conntrack nf_conntrack ebtable_filter ebtables ip6table_filter
> > > ip6_tab
> > > les iptable_filter ip_tables sg qeth_l2 vmur xfs libcrc32c dasd_fba_mod
> > > dasd_eck
> > > d_mod lcs dasd_mod qeth ctcm qdio ccwgroup fsm dm_mirror dm_region_hash
> > > dm_log d
> > > m_mod
> > > Ý 210.053571¨ CPU: 0 Tainted: G D 3.8.0-rc3+ #16
> >
>