At a stable 32 bit stable Gentoo with kernel v3.10-rc1-113-ga2c7a54 I cannot umount an (EXT4) fs
which was created in a file located in a tmpfs partition and loop mounted :
That file system was used to hold victims files shared via NFSv4 to an user mode linux
on which trinity was used to fuzz testing a patched UML guest kernel.
n22 ~ # mount
...
nfsd on /proc/fs/nfsd type nfsd (rw,nodev,noexec,nosuid)
/mnt/ramdisk/disk0 on /mnt/trinity type ext4 (rw)
n22 ~ # umount /mnt/trinity
umount: /mnt/trinity: not mounted
n22 ~ # lsof | grep loop
loop0 7028 root cwd DIR 8,19 4096 2 /
loop0 7028 root rtd DIR 8,19 4096 2 /
loop0 7028 root txt unknown /proc/7028/exe
n22 ~ # ps axf | grep 7028 | grep -v grep
7028 ? S< 0:00 \_ [loop0]
n22 ~ # kill -9 7028
n22 ~ # ps axf | grep 7028 | grep -v grep
7028 ? S< 0:00 \_ [loop0]
n22 ~ # umount /mnt/trinity
umount: /mnt/trinity: not mounted
n22 ~ # strace umount /mnt/trinity
execve("/bin/umount", ["umount", "/mnt/trinity"], [/* 40 vars */]) = 0
brk(0) = 0x8b1a000
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb777d000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=148191, ...}) = 0
mmap2(NULL, 148191, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7758000
close(3) = 0
open("/lib/libmount.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0p~\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=236364, ...}) = 0
mmap2(NULL, 243340, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb771c000
mprotect(0xb7754000, 4096, PROT_NONE) = 0
mmap2(0xb7755000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x38) = 0xb7755000
mmap2(0xb7757000, 1676, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb7757000
close(3) = 0
open("/lib/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\240\212\231F4\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=1715512, ...}) = 0
mmap2(0x4697c000, 1727228, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x4697c000
mprotect(0x46b1b000, 4096, PROT_NONE) = 0
mmap2(0x46b1c000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x19f) = 0x46b1c000
mmap2(0x46b1f000, 11004, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x46b1f000
close(3) = 0
open("/lib/libblkid.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\0W\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=219808, ...}) = 0
mmap2(NULL, 226800, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb76e4000
mmap2(0xb7718000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x33) = 0xb7718000
mmap2(0xb771b000, 1520, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb771b000
close(3) = 0
open("/lib/libuuid.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\340\20\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=17892, ...}) = 0
mmap2(NULL, 20672, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb76de000
mmap2(0xb76e2000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3) = 0xb76e2000
close(3) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb76dd000
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb76dc000
set_thread_area({entry_number:-1 -> 6, base_addr:0xb76dc740, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}) = 0
mprotect(0x46b1c000, 8192, PROT_READ) = 0
mprotect(0xb76e2000, 4096, PROT_READ) = 0
mprotect(0xb7718000, 8192, PROT_READ) = 0
mprotect(0xb7755000, 4096, PROT_READ) = 0
mprotect(0x804d000, 4096, PROT_READ) = 0
mprotect(0x46954000, 4096, PROT_READ) = 0
munmap(0xb7758000, 148191) = 0
brk(0) = 0x8b1a000
brk(0x8b3b000) = 0x8b3b000
open("/usr/lib/locale/locale-archive", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=4072928, ...}) = 0
mmap2(NULL, 2097152, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb74dc000
mmap2(NULL, 8192, PROT_READ, MAP_PRIVATE, 3, 0x1ff) = 0xb777b000
close(3) = 0
getuid32() = 0
geteuid32() = 0
getuid32() = 0
geteuid32() = 0
getgid32() = 0
getegid32() = 0
prctl(PR_GET_DUMPABLE) = 1
lstat64("/etc/mtab", {st_mode=S_IFREG|0644, st_size=1604, ...}) = 0
open("/etc/mtab", O_RDWR|O_CREAT|O_LARGEFILE, 0644) = 3
close(3) = 0
lstat64("/etc/mtab", {st_mode=S_IFREG|0644, st_size=1604, ...}) = 0
open("/etc/mtab", O_RDONLY|O_LARGEFILE) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=1604, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb777a000
read(3, "rootfs / rootfs rw 0 0\n/dev/root"..., 4096) = 1604
read(3, "", 4096) = 0
close(3) = 0
munmap(0xb777a000, 4096) = 0
stat64("/sbin/umount.ext4", 0xbf8263cc) = -1 ENOENT (No such file or directory)
stat64("/sbin/fs.d/umount.ext4", 0xbf8263cc) = -1 ENOENT (No such file or directory)
stat64("/sbin/fs/umount.ext4", 0xbf8263cc) = -1 ENOENT (No such file or directory)
stat64("/usr/sbin/umount.ext4", 0xbf8263cc) = -1 ENOENT (No such file or directory)
stat64("/bin/umount.ext4", 0xbf8263cc) = -1 ENOENT (No such file or directory)
stat64("/usr/bin/umount.ext4", 0xbf8263cc) = -1 ENOENT (No such file or directory)
oldumount("/mnt/trinity") = -1 EINVAL (Invalid argument)
open("/usr/share/locale/locale.alias", O_RDONLY|O_CLOEXEC) = 3
fcntl64(3, F_GETFD) = 0x1 (flags FD_CLOEXEC)
fstat64(3, {st_mode=S_IFREG|0644, st_size=2570, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb777a000
read(3, "# Locale name alias data base.\n#"..., 4096) = 2570
read(3, "", 4096) = 0
close(3) = 0
munmap(0xb777a000, 4096) = 0
open("/usr/share/locale/en_US.utf8/LC_MESSAGES/util-linux.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US/LC_MESSAGES/util-linux.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.utf8/LC_MESSAGES/util-linux.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en/LC_MESSAGES/util-linux.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
write(2, "umount: ", 8umount: ) = 8
write(2, "/mnt/trinity: not mounted", 25/mnt/trinity: not mounted) = 25
write(2, "\n", 1
) = 1
close(1) = 0
close(2) = 0
exit_group(32) = ?
+++ exited with 32 +++
--
MfG/Sincerely
Toralf Förster
pgp finger print: 7B1A 07F4 EC82 0F90 D4C2 8936 872A E508 7DB6 9DA3
On 05/14/2013 09:12 PM, Toralf Förster wrote:
> At a stable 32 bit stable Gentoo with kernel v3.10-rc1-113-ga2c7a54 I cannot umount an (EXT4) fs
> which was created in a file located in a tmpfs partition and loop mounted :
>
> That file system was used to hold victims files shared via NFSv4 to an user mode linux
> on which trinity was used to fuzz testing a patched UML guest kernel.
FWIW the numbers are really wrong for /mnt/trinity
# df -m /mnt/trinity/
Filesystem 1M-blocks Used Available Use% Mounted on
/dev/loop0 183851 31165 143325 18% /mnt/trinity
b/c 257 MB were specified for the file in which the EXT4FS was created:
# ls -lh /mnt/ramdisk/disk0
-rw-r--r-- 1 tfoerste users 257M May 14 20:56 /mnt/ramdisk/disk0
here is the corresponding code snippet :
FS="ext4"
dd if=/dev/zero of=/mnt/ramdisk/disk0 bs=1M count=257 2>/dev/null || return 2
yes | /sbin/mkfs.$FS /mnt/ramdisk/disk0 1>/dev/null || return 3
sudo su -c "mount -o loop /mnt/ramdisk/disk0 /mnt/trinity/; chmod 777 /mnt/trinity" || return 4
--
MfG/Sincerely
Toralf Förster
pgp finger print: 7B1A 07F4 EC82 0F90 D4C2 8936 872A E508 7DB6 9DA3
On 5/14/13 2:12 PM, Toralf Förster wrote:
> At a stable 32 bit stable Gentoo with kernel v3.10-rc1-113-ga2c7a54 I cannot umount an (EXT4) fs
> which was created in a file located in a tmpfs partition and loop mounted :
>
> That file system was used to hold victims files shared via NFSv4 to an user mode linux
> on which trinity was used to fuzz testing a patched UML guest kernel.
>
> n22 ~ # mount
> ...
> nfsd on /proc/fs/nfsd type nfsd (rw,nodev,noexec,nosuid)
> /mnt/ramdisk/disk0 on /mnt/trinity type ext4 (rw)
Is your "mount" looking at /etc/mtab ot at /proc/mounts?
What does /proc/mounts say, does it contain this device or mountpoint?
> n22 ~ # umount /mnt/trinity
> umount: /mnt/trinity: not mounted
-Eric
On 05/14/2013 11:12 PM, Eric Sandeen wrote:
> Is your "mount" looking at /etc/mtab ot at /proc/mounts?
I dunno, it is package sys-apps/util-linux-2.22.2 - maybe the Gentoo devs know more (it is a stable 32bit Gentoo).
FWIW today I got after another test run (kernel 3.10.-rc1+) :
2013-05-16T22:17:20.984+02:00 n22 kernel: EXT4-fs (loop0): sb orphan head is 32017
2013-05-16T22:17:20.984+02:00 n22 kernel: sb_info orphan list:
2013-05-16T22:17:20.984+02:00 n22 kernel: inode loop0:32017 at d9ed5a50: mode 103777, nlink 0, next 32003
2013-05-16T22:17:20.984+02:00 n22 kernel: inode loop0:32003 at dfde6d10: mode 102777, nlink 0, next 32021
2013-05-16T22:17:20.984+02:00 n22 kernel: inode loop0:32021 at d9ed6158: mode 103777, nlink 0, next 32013
2013-05-16T22:17:20.984+02:00 n22 kernel: inode loop0:32013 at d9ed55a0: mode 103777, nlink 0, next 0
2013-05-16T22:17:20.984+02:00 n22 kernel: ------------[ cut here ]------------
2013-05-16T22:17:20.984+02:00 n22 kernel: kernel BUG at fs/ext4/super.c:804!
2013-05-16T22:17:20.984+02:00 n22 kernel: invalid opcode: 0000 [#1] SMP
2013-05-16T22:17:20.984+02:00 n22 kernel: Modules linked in: dm_crypt cpufreq_stats loop nfsd auth_rpcgss oid_registry lockd sunrpc ipt_MASQUERADE xt_owner xt_multiport ipt_REJECT xt_tcpudp xt_recent xt_conntrack nf_conntrack_ftp xt_limit xt_LOG iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_filter ip_tables x_tables af_packet pppoe pppox ppp_generic slhc bridge stp llc tun fuse dm_mod coretemp kvm_intel kvm hid_cherry aesni_intel uvcvideo xts aes_i586 videobuf2_vmalloc lrw videobuf2_memops gf128mul ablk_helper hid_generic arc4 usbhid videobuf2_core cryptd iwldvm mac80211 usblp videodev hid i915 snd_hda_codec_conexant snd_hda_intel iwlwifi cfg80211 cfbfillrect cfbimgblt i2c_algo_bit cfbcopyarea intel_agp intel_gtt snd_hda_codec snd_pcm fbcon snd_page_alloc bitblit softcursor font drm_kms_helper drm snd_timer psmouse tpm_tis 8250_pci thinkpad_acpi agpgart nvram sr_mod evdev snd tpm wmi 8250 serial_core i2c_i801 i2c_core cdrom sdhci_pci sdhci mmc_co
re soundcore acpi_cpufreq thermal mperf battery ac tpm_bios e1000e rfkill ptp video fb button fbdev pps_core processor thermal_sys hwmon [last unloaded: microcode]
2013-05-16T22:17:20.984+02:00 n22 kernel: CPU: 1 PID: 9391 Comm: umount Not tainted 3.10.0-rc1+ #1
2013-05-16T22:17:20.984+02:00 n22 kernel: Hardware name: LENOVO 4180F65/4180F65, BIOS 83ET73WW (1.43 ) 11/30/2012
2013-05-16T22:17:20.984+02:00 n22 kernel: task: eab54290 ti: d6cfc000 task.ti: d6cfc000
2013-05-16T22:17:20.984+02:00 n22 kernel: EIP: 0060:[<c11bcb8c>] EFLAGS: 00010283 CPU: 1
2013-05-16T22:17:20.984+02:00 n22 kernel: EIP is at ext4_put_super+0x2dc/0x2e0
2013-05-16T22:17:20.984+02:00 n22 kernel: EAX: 0000003d EBX: e7adbc00 ECX: e7adbd50 EDX: e7adbd50
2013-05-16T22:17:20.984+02:00 n22 kernel: ESI: e7addc00 EDI: e7adbd14 EBP: d6cfdefc ESP: d6cfdecc
2013-05-16T22:17:20.984+02:00 n22 kernel: DS: 007b ES: 007b FS: 00d8 GS: 00e0 SS: 0068
2013-05-16T22:17:20.984+02:00 n22 kernel: CR0: 80050033 CR2: b76bd0c0 CR3: 2f41e000 CR4: 000407f0
2013-05-16T22:17:20.984+02:00 n22 kernel: DR0: 00000000 DR1: 00000000 DR2: 00000000 DR3: 00000000
2013-05-16T22:17:20.984+02:00 n22 kernel: DR6: ffff0ff0 DR7: 00000400
2013-05-16T22:17:20.984+02:00 n22 kernel: Stack:
2013-05-16T22:17:20.984+02:00 n22 kernel: c155c130 e7adddbc 00007d0d d9ed55a0 000087ff 00000000 00000000 d9ed5580
2013-05-16T22:17:20.984+02:00 n22 kernel: e7adbd50 e7addc00 e7addc58 c14957a0 d6cfdf18 c111f441 d6cfdf28 d6cfdf18
2013-05-16T22:17:20.984+02:00 n22 kernel: f1fd4200 00000083 e7addc00 d6cfdf28 c111f4e9 e7addc00 c15f1e28 d6cfdf38
2013-05-16T22:17:20.984+02:00 n22 kernel: Call Trace:
2013-05-16T22:17:20.984+02:00 n22 kernel: [<c111f441>] generic_shutdown_super+0x51/0xd0
2013-05-16T22:17:20.984+02:00 n22 kernel: [<c111f4e9>] kill_block_super+0x29/0x70
2013-05-16T22:17:20.984+02:00 n22 kernel: [<c111f734>] deactivate_locked_super+0x44/0x70
2013-05-16T22:17:20.984+02:00 n22 kernel: [<c1120107>] deactivate_super+0x47/0x60
2013-05-16T22:17:20.985+02:00 n22 kernel: [<c1136e8d>] mntput_no_expire+0xcd/0x120
2013-05-16T22:17:20.985+02:00 n22 kernel: [<c1137d4e>] SyS_umount+0xae/0x330
2013-05-16T22:17:20.985+02:00 n22 kernel: [<c1137fee>] SyS_oldumount+0x1e/0x20
2013-05-16T22:17:20.985+02:00 n22 kernel: [<c1477601>] sysenter_do_call+0x12/0x22
2013-05-16T22:17:20.985+02:00 n22 kernel: Code: 24 30 c1 55 c1 05 bc 01 00 00 89 44 24 04 e8 4d 22 2b 00 8b 4d ec 8b 55 f0 8b 09 39 ca 75 b2 39 93 50 01 00 00 0f 84 9a fe ff ff <0f> 0b 66 90 55 89 e5 83 ec 20 66 66 66 66 90 8d 45 18 c7 04 24
2013-05-16T22:17:20.985+02:00 n22 kernel: EIP: [<c11bcb8c>] ext4_put_super+0x2dc/0x2e0 SS:ESP 0068:d6cfdecc
2013-05-16T22:17:20.985+02:00 n22 kernel: ---[ end trace 0970735cee52720b ]---
--
MfG/Sincerely
Toralf Förster
pgp finger print: 7B1A 07F4 EC82 0F90 D4C2 8936 872A E508 7DB6 9DA3
On 5/16/13 3:23 PM, Toralf Förster wrote:
> On 05/14/2013 11:12 PM, Eric Sandeen wrote:
>> Is your "mount" looking at /etc/mtab ot at /proc/mounts?
> I dunno, it is package sys-apps/util-linux-2.22.2 - maybe the Gentoo devs know more (it is a stable 32bit Gentoo).
<dropping fsdevel & LKML from cc:>
That's why I then asked:
>> What does /proc/mounts say, does it contain this device or mountpoint?
so what does your /proc/mounts contain?
> FWIW today I got after another test run (kernel 3.10.-rc1+) :
which seems to probably be a different problem, although since open-but-unlinked
inodes, which populate the orphan inode list, would also prevent an umount,
this might be related.
You hit this in ext4_put_super() during umount:
/* Debugging code just in case the in-memory inode orphan list
* isn't empty. The on-disk one can be non-empty if we've
* detected an error and taken the fs readonly, but the
* in-memory list had better be clean by this point. */
if (!list_empty(&sbi->s_orphan))
dump_orphan_list(sb, sbi);
J_ASSERT(list_empty(&sbi->s_orphan));
any chance you've got anything crazy going on like mounting the same loop
file on 2 machines via an nfs export, or anything else out of the ordinary?
If you mount it and do "find -inum 32017 /mount/point" for each of the inode
numbers below, do the files in question have anything unique going on?
-Eric
>
> 2013-05-16T22:17:20.984+02:00 n22 kernel: EXT4-fs (loop0): sb orphan head is 32017
> 2013-05-16T22:17:20.984+02:00 n22 kernel: sb_info orphan list:
> 2013-05-16T22:17:20.984+02:00 n22 kernel: inode loop0:32017 at d9ed5a50: mode 103777, nlink 0, next 32003
> 2013-05-16T22:17:20.984+02:00 n22 kernel: inode loop0:32003 at dfde6d10: mode 102777, nlink 0, next 32021
> 2013-05-16T22:17:20.984+02:00 n22 kernel: inode loop0:32021 at d9ed6158: mode 103777, nlink 0, next 32013
> 2013-05-16T22:17:20.984+02:00 n22 kernel: inode loop0:32013 at d9ed55a0: mode 103777, nlink 0, next 0
> 2013-05-16T22:17:20.984+02:00 n22 kernel: ------------[ cut here ]------------
> 2013-05-16T22:17:20.984+02:00 n22 kernel: kernel BUG at fs/ext4/super.c:804!
> 2013-05-16T22:17:20.984+02:00 n22 kernel: invalid opcode: 0000 [#1] SMP
> 2013-05-16T22:17:20.984+02:00 n22 kernel: Modules linked in: dm_crypt cpufreq_stats loop nfsd auth_rpcgss oid_registry lockd sunrpc ipt_MASQUERADE xt_owner xt_multiport ipt_REJECT xt_tcpudp xt_recent xt_conntrack nf_conntrack_ftp xt_limit xt_LOG iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_filter ip_tables x_tables af_packet pppoe pppox ppp_generic slhc bridge stp llc tun fuse dm_mod coretemp kvm_intel kvm hid_cherry aesni_intel uvcvideo xts aes_i586 videobuf2_vmalloc lrw videobuf2_memops gf128mul ablk_helper hid_generic arc4 usbhid videobuf2_core cryptd iwldvm mac80211 usblp videodev hid i915 snd_hda_codec_conexant snd_hda_intel iwlwifi cfg80211 cfbfillrect cfbimgblt i2c_algo_bit cfbcopyarea intel_agp intel_gtt snd_hda_codec snd_pcm fbcon snd_page_alloc bitblit softcursor font drm_kms_helper drm snd_timer psmouse tpm_tis 8250_pci thinkpad_acpi agpgart nvram sr_mod evdev snd tpm wmi 8250 serial_core i2c_i801 i2c_core cdrom sdhci_pci !
sd!
> hci mmc_co
> re soundcore acpi_cpufreq thermal mperf battery ac tpm_bios e1000e rfkill ptp video fb button fbdev pps_core processor thermal_sys hwmon [last unloaded: microcode]
> 2013-05-16T22:17:20.984+02:00 n22 kernel: CPU: 1 PID: 9391 Comm: umount Not tainted 3.10.0-rc1+ #1
> 2013-05-16T22:17:20.984+02:00 n22 kernel: Hardware name: LENOVO 4180F65/4180F65, BIOS 83ET73WW (1.43 ) 11/30/2012
> 2013-05-16T22:17:20.984+02:00 n22 kernel: task: eab54290 ti: d6cfc000 task.ti: d6cfc000
> 2013-05-16T22:17:20.984+02:00 n22 kernel: EIP: 0060:[<c11bcb8c>] EFLAGS: 00010283 CPU: 1
> 2013-05-16T22:17:20.984+02:00 n22 kernel: EIP is at ext4_put_super+0x2dc/0x2e0
> 2013-05-16T22:17:20.984+02:00 n22 kernel: EAX: 0000003d EBX: e7adbc00 ECX: e7adbd50 EDX: e7adbd50
> 2013-05-16T22:17:20.984+02:00 n22 kernel: ESI: e7addc00 EDI: e7adbd14 EBP: d6cfdefc ESP: d6cfdecc
> 2013-05-16T22:17:20.984+02:00 n22 kernel: DS: 007b ES: 007b FS: 00d8 GS: 00e0 SS: 0068
> 2013-05-16T22:17:20.984+02:00 n22 kernel: CR0: 80050033 CR2: b76bd0c0 CR3: 2f41e000 CR4: 000407f0
> 2013-05-16T22:17:20.984+02:00 n22 kernel: DR0: 00000000 DR1: 00000000 DR2: 00000000 DR3: 00000000
> 2013-05-16T22:17:20.984+02:00 n22 kernel: DR6: ffff0ff0 DR7: 00000400
> 2013-05-16T22:17:20.984+02:00 n22 kernel: Stack:
> 2013-05-16T22:17:20.984+02:00 n22 kernel: c155c130 e7adddbc 00007d0d d9ed55a0 000087ff 00000000 00000000 d9ed5580
> 2013-05-16T22:17:20.984+02:00 n22 kernel: e7adbd50 e7addc00 e7addc58 c14957a0 d6cfdf18 c111f441 d6cfdf28 d6cfdf18
> 2013-05-16T22:17:20.984+02:00 n22 kernel: f1fd4200 00000083 e7addc00 d6cfdf28 c111f4e9 e7addc00 c15f1e28 d6cfdf38
> 2013-05-16T22:17:20.984+02:00 n22 kernel: Call Trace:
> 2013-05-16T22:17:20.984+02:00 n22 kernel: [<c111f441>] generic_shutdown_super+0x51/0xd0
> 2013-05-16T22:17:20.984+02:00 n22 kernel: [<c111f4e9>] kill_block_super+0x29/0x70
> 2013-05-16T22:17:20.984+02:00 n22 kernel: [<c111f734>] deactivate_locked_super+0x44/0x70
> 2013-05-16T22:17:20.984+02:00 n22 kernel: [<c1120107>] deactivate_super+0x47/0x60
> 2013-05-16T22:17:20.985+02:00 n22 kernel: [<c1136e8d>] mntput_no_expire+0xcd/0x120
> 2013-05-16T22:17:20.985+02:00 n22 kernel: [<c1137d4e>] SyS_umount+0xae/0x330
> 2013-05-16T22:17:20.985+02:00 n22 kernel: [<c1137fee>] SyS_oldumount+0x1e/0x20
> 2013-05-16T22:17:20.985+02:00 n22 kernel: [<c1477601>] sysenter_do_call+0x12/0x22
> 2013-05-16T22:17:20.985+02:00 n22 kernel: Code: 24 30 c1 55 c1 05 bc 01 00 00 89 44 24 04 e8 4d 22 2b 00 8b 4d ec 8b 55 f0 8b 09 39 ca 75 b2 39 93 50 01 00 00 0f 84 9a fe ff ff <0f> 0b 66 90 55 89 e5 83 ec 20 66 66 66 66 90 8d 45 18 c7 04 24
> 2013-05-16T22:17:20.985+02:00 n22 kernel: EIP: [<c11bcb8c>] ext4_put_super+0x2dc/0x2e0 SS:ESP 0068:d6cfdecc
> 2013-05-16T22:17:20.985+02:00 n22 kernel: ---[ end trace 0970735cee52720b ]---
>
On 05/16/2013 10:36 PM, Eric Sandeen wrote:
> any chance you've got anything crazy going on like mounting the same loop
> file on 2 machines via an nfs export, or anything else out of the ordinary?
>
I mount that directory only from 1 UML guest and only once.
The I run a lot of trinity test in the UML guest suing that share.
After that the UML guest was shutdowned.
Finally I stoped teh NFS daemon at the host.
Then I tried to unmount the the drive.
> If you mount it and do "find -inum 32017 /mount/point" for each of the inode
> numbers below, do the files in question have anything unique going on?
n22 ~ # find /mnt/trinity/ -inum 32017
FWIW :
n22 ~ # umount /mnt/trinity/
umount: /mnt/trinity: not mounted
n22 ~ # grep trinity /etc/mtab /proc/mounts
/etc/mtab:/dev/loop0 /mnt/trinity ext4 rw 0 0
--
MfG/Sincerely
Toralf Förster
pgp finger print: 7B1A 07F4 EC82 0F90 D4C2 8936 872A E508 7DB6 9DA3
Forget to mention, that now df shows for the loop the same values as for / :
n22 ~ # df
Filesystem 1K-blocks Used Available Use% Mounted on
rootfs 188262740 32904632 145771804 19% /
/dev/root 188262740 32904632 145771804 19% /
devtmpfs 4086480 0 4086480 0% /dev
tmpfs 4086744 888 4085856 1% /run
shm 4086744 0 4086744 0% /dev/shm
cgroup_root 10240 0 10240 0% /sys/fs/cgroup
/dev/sdb4 530428688 24466632 505962056 5% /mnt/E
tmpfs 3145728 243872 2901856 8% /mnt/ramdisk
/dev/loop0 188262740 32904632 145771804 19% /mnt/trinity
--
MfG/Sincerely
Toralf Förster
pgp finger print: 7B1A 07F4 EC82 0F90 D4C2 8936 872A E508 7DB6 9DA3
On 5/16/13 3:43 PM, Toralf Förster wrote:
> On 05/16/2013 10:36 PM, Eric Sandeen wrote:
>> any chance you've got anything crazy going on like mounting the same loop
>> file on 2 machines via an nfs export, or anything else out of the ordinary?
>>
> I mount that directory only from 1 UML guest and only once.
> The I run a lot of trinity test in the UML guest suing that share.
which apparently manages to break things - as intended. ;)
Can you narrow down which syscalls cause the problem?
> After that the UML guest was shutdowned.
> Finally I stoped teh NFS daemon at the host.
> Then I tried to unmount the the drive.
>
>> If you mount it and do "find -inum 32017 /mount/point" for each of the inode
>> numbers below, do the files in question have anything unique going on?
>
>
> n22 ~ # find /mnt/trinity/ -inum 32017
>
>
> FWIW :
>
> n22 ~ # umount /mnt/trinity/
> umount: /mnt/trinity: not mounted
Is this the very first umount call . . . ?
> n22 ~ # grep trinity /etc/mtab /proc/mounts
> /etc/mtab:/dev/loop0 /mnt/trinity ext4 rw 0 0
... because it seems to already be unmounted, if it's
not in /proc/mounts (though checking for loop0 in /proc/mounts
might be worthwhile too)
-Eric
>
>
On 05/16/2013 10:36 PM, Eric Sandeen wrote:
> so what does your /proc/mounts contain?
n22 / # cat /proc/mounts
rootfs / rootfs rw 0 0
/dev/root / ext4 rw,noatime,data=ordered 0 0
devtmpfs /dev devtmpfs rw,relatime,size=4086480k,nr_inodes=203718,mode=755 0 0
proc /proc proc rw,relatime 0 0
tmpfs /run tmpfs rw,nosuid,nodev,relatime,mode=755 0 0
devpts /dev/pts devpts rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0
shm /dev/shm tmpfs rw,nodev,relatime 0 0
sysfs /sys sysfs rw,nosuid,nodev,noexec,relatime 0 0
securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0
debugfs /sys/kernel/debug debugfs rw,nosuid,nodev,noexec,relatime 0 0
cgroup_root /sys/fs/cgroup tmpfs rw,nosuid,nodev,noexec,relatime,size=10240k,mode=755 0 0
openrc /sys/fs/cgroup/openrc cgroup rw,nosuid,nodev,noexec,relatime,release_agent=/lib/rc/sh/cgroup-release-agent.sh,name=openrc 0 0
cpuset /sys/fs/cgroup/cpuset cgroup rw,nosuid,nodev,noexec,relatime,cpuset 0 0
cpu /sys/fs/cgroup/cpu cgroup rw,nosuid,nodev,noexec,relatime,cpu 0 0
cpuacct /sys/fs/cgroup/cpuacct cgroup rw,nosuid,nodev,noexec,relatime,cpuacct 0 0
memory /sys/fs/cgroup/memory cgroup rw,nosuid,nodev,noexec,relatime,memory 0 0
devices /sys/fs/cgroup/devices cgroup rw,nosuid,nodev,noexec,relatime,devices 0 0
freezer /sys/fs/cgroup/freezer cgroup rw,nosuid,nodev,noexec,relatime,freezer 0 0
blkio /sys/fs/cgroup/blkio cgroup rw,nosuid,nodev,noexec,relatime,blkio 0 0
/dev/sdb4 /mnt/E fuseblk rw,nosuid,nodev,noexec,relatime,user_id=0,group_id=0,default_permissions,allow_other,blksize=4096 0 0
tmpfs /mnt/ramdisk tmpfs rw,relatime,size=3145728k 0 0
rpc_pipefs /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0
nfsd /proc/fs/nfsd nfsd rw,nosuid,nodev,noexec,relatime 0 0
--
MfG/Sincerely
Toralf Förster
pgp finger print: 7B1A 07F4 EC82 0F90 D4C2 8936 872A E508 7DB6 9DA3
On 5/16/13 3:45 PM, Toralf Förster wrote:
> Forget to mention, that now df shows for the loop the same values as for / :
because it's "half unmounted" - mtab still has it, so df is
doing a statfs of it, but it's not really mounted (as /proc/mounts
shows) so you get a 2nd statfs on /
-Eric
> n22 ~ # df
> Filesystem 1K-blocks Used Available Use% Mounted on
> rootfs 188262740 32904632 145771804 19% /
> /dev/root 188262740 32904632 145771804 19% /
> devtmpfs 4086480 0 4086480 0% /dev
> tmpfs 4086744 888 4085856 1% /run
> shm 4086744 0 4086744 0% /dev/shm
> cgroup_root 10240 0 10240 0% /sys/fs/cgroup
> /dev/sdb4 530428688 24466632 505962056 5% /mnt/E
> tmpfs 3145728 243872 2901856 8% /mnt/ramdisk
> /dev/loop0 188262740 32904632 145771804 19% /mnt/trinity
>
>
On 05/16/2013 10:49 PM, Eric Sandeen wrote:
> On 5/16/13 3:43 PM, Toralf Förster wrote:
>> On 05/16/2013 10:36 PM, Eric Sandeen wrote:
>>> any chance you've got anything crazy going on like mounting the same loop
>>> file on 2 machines via an nfs export, or anything else out of the ordinary?
>>>
>> I mount that directory only from 1 UML guest and only once.
>> The I run a lot of trinity test in the UML guest suing that share.
>
> which apparently manages to break things - as intended. ;)
yes - I'm calling for trouble and I get it - but by this (uncommon) scenario few real NFSv4 and UML bugs were found.
> Can you narrow down which syscalls cause the problem?
Till now not to a single one (would be nice, I know).
Currently I observed that issue not with host kernel 3.9, but with host kernel 3.10-rc1.
Furthermore on the UML guest trinity has to be run long enough (hours) to produce warnings like :
# zgrep "v4 server returned" /var/log/messages* | cut -f3- -d' '
kernel: NFS: v4 server returned a bad sequence-id error on an unconfirmed sequence 0c6a41f0!
kernel: NFS: v4 server returned a bad sequence-id error on an unconfirmed sequence 3fe4fa60!
kernel: NFS: v4 server returned a bad sequence-id error on an unconfirmed sequence 4143f790!
kernel: NFS: v4 server returned a bad sequence-id error on an unconfirmed sequence 48f401f0!
...
> Is this the very first umount call . . . ?
yes, before that probably the NFS service was either restarted or just halted.
--
MfG/Sincerely
Toralf Förster
pgp finger print: 7B1A 07F4 EC82 0F90 D4C2 8936 872A E508 7DB6 9DA3