Hi,
mkdir failed IO error on pmem DAX ext2/3 fs mount using ext4 module.
This happends only on -next tree, not on Linus' tree,
at least from 4.5.0-rc5-next-20160224.
.config attached.
sh-4.2# uname -r
4.5.0-rc6-next-20160301
sh-4.2# sh -x extmod.sh
+ testmkdir ext2 /dev/pmem0 /daxmnt ext4
+ mkdir -p /daxmnt
+ mkfs.ext2 -Fq /dev/pmem0
+ lsmod
+ grep ext
ext2 73728 0
ext4 585728 0
jbd2 110592 1 ext4
mbcache 16384 2 ext2,ext4
+ mount -t ext4 -o dax /dev/pmem0 /daxmnt
+ mount
+ grep dax
/dev/pmem0 on /daxmnt type ext4
(rw,relatime,seclabel,block_validity,delalloc,barrier,dax,user_xattr,acl)
+ lsmod
+ grep ext
ext2 73728 0
ext4 585728 1
jbd2 110592 1 ext4
mbcache 16384 3 ext2,ext4
+ cd /daxmnt
+ mkdir -p 1/2/3/4
mkdir: cannot create directory ‘1’: Input/output error
+ touch 5
+ cd
+ umount /daxmnt
+ testmkdir ext3 /dev/pmem0 /daxmnt ext4
+ mkdir -p /daxmnt
+ mkfs.ext3 -Fq /dev/pmem0
+ lsmod
+ grep ext
ext2 73728 0
ext4 585728 0
jbd2 110592 1 ext4
mbcache 16384 2 ext2,ext4
+ mount -t ext4 -o dax /dev/pmem0 /daxmnt
+ mount
+ grep dax
/dev/pmem0 on /daxmnt type ext4 (rw,relatime,seclabel,dax,data=ordered)
+ lsmod
+ grep ext
ext2 73728 0
ext4 585728 1
jbd2 110592 1 ext4
mbcache 16384 3 ext2,ext4
+ cd /daxmnt
+ mkdir -p 1/2/3/4
mkdir: cannot create directory ‘1’: Input/output error
+ touch 5
+ cd
+ umount /daxmnt
sh-4.2# cat extmod.sh
#!/bin/bash
# param 1/2/3 mkfs.fstype/pmemdev/mountpoint/mountfstype
function testmkdir()
{
#modprobe -r $1
mkdir -p $3
mkfs.$1 -Fq $2
lsmod | grep ext
mount -t $4 -o dax $2 $3
mount | grep dax
lsmod | grep ext
cd $3
mkdir -p 1/2/3/4
touch 5
cd
umount $3
}
testmkdir ext2 /dev/pmem0 /daxmnt ext4
testmkdir ext3 /dev/pmem0 /daxmnt ext4
#testmkdir ext2 /dev/pmem0 /daxmnt ext2 # pass
#testmkdir ext4 /dev/pmem0 /daxmnt ext4 # pass
sh-4.2#
--
Xiong
On Wed, Mar 02, 2016 at 02:43:04PM +0800, Xiong Zhou wrote:
> Hi,
>
> mkdir failed IO error on pmem DAX ext2/3 fs mount using ext4 module.
>
> This happends only on -next tree, not on Linus' tree,
> at least from 4.5.0-rc5-next-20160224.
I was able to reproduce this and bisect it to the following commit:
commit 1f2d779fed21 ("ext4: optimize group search for inode allocation")
The fact that DAX was enabled in this failure was a red herring - it fails
without DAX as well. The key is just that we're mounting an ext2 formatted
filesystem using the ext4 module, I think.
- Ross
Hi,
I have tested it locally I am also getting same error but not
consistently. I have put some debug logs for debugging, as per the logs
it seems that block_group is not getting initialized properly. I am
not sure weather this is because of my patch. Need to debug more.
commands :
=========
[root@cli02-ws Desktop]# dd if=/dev/zero of=/mnt/test bs=1024 count=100000
[root@cli02-ws Desktop]# losetup /dev/loop4 /mnt/test
[root@cli02-ws Desktop]# mkfs.ext2 /mnt/test
[root@cli02-ws Desktop]# mount -t ext2 -o loop /mnt/test /mnt/test_ext4
[root@cli02-ws Desktop]# cd /mnt/test_ext4/
[root@cli02-ws test_ext4]# mkdir 1
mkdir: cannot create directory `1': Input/output error
[root@cli02-ws test_ext4]# mkdir 2
[root@cli02-ws test_ext4]# mkdir 3
mkdir: cannot create directory `3': Input/output error
[root@cli02-ws test_ext4]# mkdir 4
debug_logs :
=========
[root@cli02-ws Desktop]# tail -f /var/log/messages 2>&1 | tee faillog
Mar 3 13:13:27 cli02-ws kernel: [ 228.167307] EXT4-fs (loop0):
mounting ext2 file system using the ext4 subsystem
Mar 3 13:13:27 cli02-ws kernel: [ 228.169634] EXT4-fs (loop0):
mounted filesystem without journal. Opts: (null)
Mar 3 13:13:42 cli02-ws kernel: [ 243.506280] max_blocks_per_group : 7949
Mar 3 13:13:42 cli02-ws kernel: [ 243.506285] stats.free_clusters : 7949
Mar 3 13:13:42 cli02-ws kernel: [ 243.506287] inodes_per_group*f : 1928
Mar 3 13:13:42 cli02-ws kernel: [ 243.506290] stats.free_inodes : 1928
Mar 3 13:13:42 cli02-ws kernel: [ 243.506292] block_group
:2103884794 <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
Mar 3 13:13:42 cli02-ws kernel: [ 243.506295] group_count : 13
Mar 3 13:13:42 cli02-ws kernel: [ 243.506298] EXT4-fs error (device
loop0): ext4_get_group_desc:291: comm mkdir: block_group >=
groups_count - block_group = 2103884794, groups_count = 13
Mar 3 13:14:09 cli02-ws kernel: [ 270.809734] max_blocks_per_group : 7949
Mar 3 13:14:09 cli02-ws kernel: [ 270.809739] stats.free_clusters : 1452
Mar 3 13:14:09 cli02-ws kernel: [ 270.809742] inodes_per_group*f : 1928
Mar 3 13:14:09 cli02-ws kernel: [ 270.809744] stats.free_inodes : 1928
Mar 3 13:14:09 cli02-ws kernel: [ 270.809747] max_blocks_per_group : 7949
Mar 3 13:14:09 cli02-ws kernel: [ 270.809750] stats.free_clusters : 7677
Mar 3 13:14:09 cli02-ws kernel: [ 270.809752] inodes_per_group*f : 1928
Mar 3 13:14:09 cli02-ws kernel: [ 270.809755] stats.free_inodes : 1917
Mar 3 13:14:09 cli02-ws kernel: [ 270.809757] max_blocks_per_group : 7949
Mar 3 13:14:09 cli02-ws kernel: [ 270.809760] stats.free_clusters : 7691
Mar 3 13:14:09 cli02-ws kernel: [ 270.809762] inodes_per_group*f : 1928
Mar 3 13:14:09 cli02-ws kernel: [ 270.809765] stats.free_inodes : 1928
Mar 3 13:14:09 cli02-ws kernel: [ 270.809767] max_blocks_per_group : 7949
Mar 3 13:14:09 cli02-ws kernel: [ 270.809770] stats.free_clusters : 7949
Mar 3 13:14:09 cli02-ws kernel: [ 270.809772] inodes_per_group*f : 1928
Mar 3 13:14:09 cli02-ws kernel: [ 270.809775] stats.free_inodes : 1928
Mar 3 13:14:38 cli02-ws kernel: [ 299.081995] max_blocks_per_group : 7949
Mar 3 13:14:38 cli02-ws kernel: [ 299.082038] stats.free_clusters : 7949
Mar 3 13:14:38 cli02-ws kernel: [ 299.082040] inodes_per_group*f : 1928
Mar 3 13:14:38 cli02-ws kernel: [ 299.082042] stats.free_inodes : 1928
Mar 3 13:14:38 cli02-ws kernel: [ 299.082045] block_group
:1066167208<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
Mar 3 13:14:38 cli02-ws kernel: [ 299.082048] group_count : 13
Mar 3 13:14:38 cli02-ws kernel: [ 299.082051] EXT4-fs error (device
loop0): ext4_get_group_desc:291: comm mkdir: block_group >=
groups_count - block_group = 1066167208, groups_count = 13
Mar 3 13:14:48 cli02-ws kernel: [ 309.138432] max_blocks_per_group : 7949
Mar 3 13:14:48 cli02-ws kernel: [ 309.138436] stats.free_clusters : 7949
Mar 3 13:14:48 cli02-ws kernel: [ 309.138439] inodes_per_group*f : 1928
Mar 3 13:14:48 cli02-ws kernel: [ 309.138452] stats.free_inodes : 1928
Regards,
Lokesh
On Thu, Mar 3, 2016 at 5:58 AM, Ross Zwisler
<[email protected]> wrote:
> On Wed, Mar 02, 2016 at 02:43:04PM +0800, Xiong Zhou wrote:
>> Hi,
>>
>> mkdir failed IO error on pmem DAX ext2/3 fs mount using ext4 module.
>>
>> This happends only on -next tree, not on Linus' tree,
>> at least from 4.5.0-rc5-next-20160224.
>
> I was able to reproduce this and bisect it to the following commit:
>
> commit 1f2d779fed21 ("ext4: optimize group search for inode allocation")
>
> The fact that DAX was enabled in this failure was a red herring - it fails
> without DAX as well. The key is just that we're mounting an ext2 formatted
> filesystem using the ext4 module, I think.
>
> - Ross
On Thu, Mar 03, 2016 at 10:49:51PM +0530, lokesh jaliminche wrote:
> Hi,
> I have tested it locally I am also getting same error but not
> consistently. I have put some debug logs for debugging, as per the logs
> it seems that block_group is not getting initialized properly. I am
> not sure weather this is because of my patch. Need to debug more.
I'm quite sure it's your patch, so I'm going to drop it for now. Note
that with ext3, we don't have flex_bg's (the flex_bg size is 1).
Here's a reliable repro:
kvm-xfstests --no-log -c ext3 generic/003
For more about kvm-xfstests
https://git.kernel.org/cgit/fs/ext2/xfstests-bld.git/tree/quick-start?h=META
Output from running this test:
BEGIN TEST ext3: Ext4 4k block emulating ext3 Thu Mar 3 18:39:33 EST 2016
DEVICE: /dev/vdd
MK2FS OPTIONS: -q -O ^extents,^flex_bg,^uninit_bg,^64bit,^metadata_csum,^huge_file,^dir_nlink,^extra_isize
MOUNT OPTIONS: -o block_validity,nodelalloc
FSTYP -- ext4
PLATFORM -- Linux/i686 kvm-xfstests 4.5.0-rc2-00026-g1f2d779
MKFS_OPTIONS -- -q -O ^extents,^flex_bg,^uninit_bg,^64bit,^metadata_csum,^huge_file,^dir_nlink,^extra_isize /dev/vdc
MOUNT_OPTIONS -- -o acl,user_xattr -o block_validity,nodelalloc /dev/vdc /vdc
generic/003 [18:39:35][ 13.096353] run fstests generic/003 at 2016-03-03 18:39:35
[ 14.859764] EXT4-fs error (device vdc): ext4_get_group_desc:288: comm mkdir: block_group >= groups_count - block_group = 850480444, groups_count = 40
[ 16.944808] EXT4-fs (vdc): warning: mounting fs with errors, running e2fsck is recommended
[ 16.956276] EXT4-fs error (device vdc): ext4_get_group_desc:288: comm mkdir: block_group >= groups_count - block_group = 3221700656, groups_count = 40
[ 19.023011] EXT4-fs (vdc): warning: mounting fs with errors, running e2fsck is recommended
[ 22.104702] EXT4-fs (vdc): warning: mounting fs with errors, running e2fsck is recommended
[18:39:46] - output mismatch (see /results/results-ext3/generic/003.out.bad)
--- tests/generic/003.out 2016-02-29 22:18:21.000000000 -0500
+++ /results/results-ext3/generic/003.out.bad 2016-03-03 18:39:46.587676694 -0500
@@ -1,2 +1,46 @@
QA output created by 003
+mkdir: cannot create directory '/vdc/dir1': Input/output error
+./tests/generic/003: line 93: /vdc/dir1/file1: No such file or directory
+stat: cannot stat '/vdc/dir1/file1': No such file or directory
+cat: /vdc/dir1/file1: No such file or directory
+stat: cannot stat '/vdc/dir1/file1': No such file or directory
+ERROR: access time has not been updated after accessing file1 first time
...
(Run 'diff -u tests/generic/003.out /results/results-ext3/generic/003.out.bad' to see the entire diff)
Ran: generic/003
Failures: generic/003
Failed 1 of 1 tests
(In fact a huge number of tests fail using the ext3 configuration, but
generic/003 is a quick fast test. Before submitting patches, please
run at the very least "kvm-xfstests smoke". Admittedly, for this bug
the smoke test wouldn't have been enough. You would have needed
"kvm-xfstests-g quick", or preferably, "kvm-xfstests -g auto".
Alternatively, you can use gce-xfstests, which uses Google Compute
Engine where you can just fire off the tests, and it will send you
e-mail with the test results. See
https://git.kernel.org/cgit/fs/ext2/xfstests-bld.git/tree/kvm-xfstests/README.GCE
for more information.)
- Ted
ok, no issues and thanks for the information above.I will take care of
these things while sending corrected version of this patch.
Thanks & Regards,
Lokesh
On Fri, Mar 4, 2016 at 5:19 AM, Theodore Ts'o <[email protected]> wrote:
> On Thu, Mar 03, 2016 at 10:49:51PM +0530, lokesh jaliminche wrote:
>> Hi,
>> I have tested it locally I am also getting same error but not
>> consistently. I have put some debug logs for debugging, as per the logs
>> it seems that block_group is not getting initialized properly. I am
>> not sure weather this is because of my patch. Need to debug more.
>
> I'm quite sure it's your patch, so I'm going to drop it for now. Note
> that with ext3, we don't have flex_bg's (the flex_bg size is 1).
>
> Here's a reliable repro:
>
> kvm-xfstests --no-log -c ext3 generic/003
>
> For more about kvm-xfstests
>
> https://git.kernel.org/cgit/fs/ext2/xfstests-bld.git/tree/quick-start?h=META
>
> Output from running this test:
>
> BEGIN TEST ext3: Ext4 4k block emulating ext3 Thu Mar 3 18:39:33 EST 2016
> DEVICE: /dev/vdd
> MK2FS OPTIONS: -q -O ^extents,^flex_bg,^uninit_bg,^64bit,^metadata_csum,^huge_file,^dir_nlink,^extra_isize
> MOUNT OPTIONS: -o block_validity,nodelalloc
> FSTYP -- ext4
> PLATFORM -- Linux/i686 kvm-xfstests 4.5.0-rc2-00026-g1f2d779
> MKFS_OPTIONS -- -q -O ^extents,^flex_bg,^uninit_bg,^64bit,^metadata_csum,^huge_file,^dir_nlink,^extra_isize /dev/vdc
> MOUNT_OPTIONS -- -o acl,user_xattr -o block_validity,nodelalloc /dev/vdc /vdc
>
> generic/003 [18:39:35][ 13.096353] run fstests generic/003 at 2016-03-03 18:39:35
> [ 14.859764] EXT4-fs error (device vdc): ext4_get_group_desc:288: comm mkdir: block_group >= groups_count - block_group = 850480444, groups_count = 40
> [ 16.944808] EXT4-fs (vdc): warning: mounting fs with errors, running e2fsck is recommended
> [ 16.956276] EXT4-fs error (device vdc): ext4_get_group_desc:288: comm mkdir: block_group >= groups_count - block_group = 3221700656, groups_count = 40
> [ 19.023011] EXT4-fs (vdc): warning: mounting fs with errors, running e2fsck is recommended
> [ 22.104702] EXT4-fs (vdc): warning: mounting fs with errors, running e2fsck is recommended
> [18:39:46] - output mismatch (see /results/results-ext3/generic/003.out.bad)
> --- tests/generic/003.out 2016-02-29 22:18:21.000000000 -0500
> +++ /results/results-ext3/generic/003.out.bad 2016-03-03 18:39:46.587676694 -0500
> @@ -1,2 +1,46 @@
> QA output created by 003
> +mkdir: cannot create directory '/vdc/dir1': Input/output error
> +./tests/generic/003: line 93: /vdc/dir1/file1: No such file or directory
> +stat: cannot stat '/vdc/dir1/file1': No such file or directory
> +cat: /vdc/dir1/file1: No such file or directory
> +stat: cannot stat '/vdc/dir1/file1': No such file or directory
> +ERROR: access time has not been updated after accessing file1 first time
> ...
> (Run 'diff -u tests/generic/003.out /results/results-ext3/generic/003.out.bad' to see the entire diff)
> Ran: generic/003
> Failures: generic/003
> Failed 1 of 1 tests
>
> (In fact a huge number of tests fail using the ext3 configuration, but
> generic/003 is a quick fast test. Before submitting patches, please
> run at the very least "kvm-xfstests smoke". Admittedly, for this bug
> the smoke test wouldn't have been enough. You would have needed
> "kvm-xfstests-g quick", or preferably, "kvm-xfstests -g auto".
> Alternatively, you can use gce-xfstests, which uses Google Compute
> Engine where you can just fire off the tests, and it will send you
> e-mail with the test results. See
> https://git.kernel.org/cgit/fs/ext2/xfstests-bld.git/tree/kvm-xfstests/README.GCE
> for more information.)
>
> - Ted