noisy:/usr/src/linux # uname -a
Linux noisy 2.4.3-XFS #8 SMP Sam Mai 19 18:21:36 CEST 2001 i686 unknown
dual board with just one processor
/usr/local and /home on LogicalVolumes
tried also 2.4.9-XFS (but see below)
------------ oops -------------
Warning (compare_maps): ksyms_base symbol __rta_fill_R__ver___rta_fill not found in
System.map. Ignoring ksyms_base entry
[other warnings stripped]
Oops: 0000
CPU: 0
EIP: 0010:<c016b3c4>
Using defaults from ksymoops -t elf32-i386 -a i386
EFLAGS: 00010246
eax: 00000000 ebx: 00000000 ecx: cff805b8 edx: 00000010
esi: ca018260 edi: ca0182e0 ebp: cb9e5800 esp: cb9edeec
ds: 0018 es: 0018 ss: 0018
Stack: 0100708b ca018360 c016b7d8 ca0182e0 00000002 cba07000 cb9efc00 cb9e5800
ce078bc4 ca018360 00000000 ca018360 c016bb70 ce078a80 0102a048 00000005
Call Trace: <c01bb7d8> <c016bb70> <c016a0b5>c0169b73> <c02a60bf> <c016999d> <c01054c4>
Code: 8b 40 10 39 d0 74 21 8d 58 c8 39 f3 75 06 8b 5a 04 83 c3 c8
>>EIP; c016b3c4 <nfsd_findparent+78/e8> <=====
Trace; c01bb7d8 <xfs_buf_item_bits+38/58>
Trace; c016bb70 <fh_verify+26c/48c>
Trace; c016a0b5 <nfsd_proc_getattr+91/98>
Code; c016b3c4 <nfsd_findparent+78/e8>
00000000 <_EIP>:
Code; c016b3c4 <nfsd_findparent+78/e8> <=====
0: 8b 40 10 mov 0x10(%eax),%eax <=====
Code; c016b3c7 <nfsd_findparent+7b/e8>
3: 39 d0 cmp %edx,%eax
Code; c016b3c9 <nfsd_findparent+7d/e8>
5: 74 21 je 28 <_EIP+0x28> c016b3ec <nfsd_findparent+a0/e8>
Code; c016b3cb <nfsd_findparent+7f/e8>
7: 8d 58 c8 lea 0xffffffc8(%eax),%ebx
Code; c016b3ce <nfsd_findparent+82/e8>
a: 39 f3 cmp %esi,%ebx
Code; c016b3d0 <nfsd_findparent+84/e8>
c: 75 06 jne 14 <_EIP+0x14> c016b3d8 <nfsd_findparent+8c/e8>
Code; c016b3d2 <nfsd_findparent+86/e8>
e: 8b 5a 04 mov 0x4(%edx),%ebx
Code; c016b3d5 <nfsd_findparent+89/e8>
11: 83 c3 c8 add $0xffffffc8,%ebx
31 warnings issued. Results may not be reliable.
------------ oops -------------
Why do I get all those warnings about symbol mismatch?
The code causing the Oops:
struct dentry *nfsd_findparent(struct dentry *child)
{
struct dentry *tdentry, *pdentry;
tdentry = d_alloc(child, &(const struct qstr) {"..", 2, 0});
if (!tdentry)
return ERR_PTR(-ENOMEM);
/* I'm going to assume that if the returned dentry is different, then
* it is well connected. But nobody returns different dentrys do they?
*/
pdentry = child->d_inode->i_op->lookup(child->d_inode, tdentry);
d_drop(tdentry); /* we never want ".." hashed */
if (!pdentry) {
/* I don't want to return a ".." dentry.
* I would prefer to return an unconnected "IS_ROOT" dentry,
* though a properly connected dentry is even better
*/
/* if first or last of alias list is not tdentry, use that
* else make a root dentry
*/
struct list_head *aliases = &tdentry->d_inode->i_dentry;
spin_lock(&dcache_lock);
if (aliases->next != aliases) { <=========== CRASH
!!!!!!!!!!!!!!!!!!!!!!!
pdentry = list_entry(aliases->next, struct dentry, d_alias);
if (pdentry == tdentry)
pdentry = list_entry(aliases->prev, struct dentry,
d_alias);
if (pdentry == tdentry)
pdentry = NULL;
if (pdentry) dget_locked(pdentry);
}
spin_unlock(&dcache_lock);
if (pdentry == NULL) {
pdentry = d_alloc_root(igrab(tdentry->d_inode));
if (pdentry) {
pdentry->d_flags |= DCACHE_NFSD_DISCONNECTED;
d_rehash(pdentry);
}
}
if (pdentry == NULL)
pdentry = ERR_PTR(-ENOMEM);
}
dput(tdentry); /* it is not hashed, it will be discarded */
return pdentry;
}
(gdb) disass nfsd_findparent
Dump of assembler code for function nfsd_findparent:
0xc016b34c <nfsd_findparent>: push %esi
0xc016b34d <nfsd_findparent+1>: push %ebx
0xc016b34e <nfsd_findparent+2>: mov 0xc(%esp,1),%ebx
0xc016b352 <nfsd_findparent+6>: push $0xc02ce724
0xc016b357 <nfsd_findparent+11>: push %ebx
0xc016b358 <nfsd_findparent+12>: call 0xc0144608 <d_alloc>
0xc016b35d <nfsd_findparent+17>: mov %eax,%esi
0xc016b35f <nfsd_findparent+19>: add $0x8,%esp
tdentry = d_alloc(child, &(const struct qstr) {"..", 2, 0});
0xc016b362 <nfsd_findparent+22>: test %esi,%esi
if (!tdentry)
0xc016b364 <nfsd_findparent+24>: jne 0xc016b370 <nfsd_findparent+36>
0xc016b366 <nfsd_findparent+26>: mov $0xfffffff4,%eax
0xc016b36b <nfsd_findparent+31>: jmp 0xc016b431 <nfsd_findparent+229>
return ERR_PTR(-ENOMEM);
0xc016b370 <nfsd_findparent+36>: mov 0x8(%ebx),%eax
0xc016b373 <nfsd_findparent+39>: mov 0x84(%eax),%edx
0xc016b379 <nfsd_findparent+45>: push %esi
0xc016b37a <nfsd_findparent+46>: push %eax
0xc016b37b <nfsd_findparent+47>: mov 0x4(%edx),%eax
0xc016b37e <nfsd_findparent+50>: call *%eax
0xc016b380 <nfsd_findparent+52>: mov %eax,%ebx
0xc016b382 <nfsd_findparent+54>: add $0x8,%esp
pdentry = child->d_inode->i_op->lookup(child->d_inode, tdentry);
static __inline__ void d_drop(struct dentry * dentry)
{
spin_lock(&dcache_lock);
list_del(&dentry->d_hash);
INIT_LIST_HEAD(&dentry->d_hash);
spin_unlock(&dcache_lock);
}
0xc016b385 <nfsd_findparent+57>: lock decb 0xc0329898
0xc016b38c <nfsd_findparent+64>: js 0xc02aefa4 <stext_lock+12548>
0xc016b392 <nfsd_findparent+70>: lea 0x18(%esi),%eax
0xc016b395 <nfsd_findparent+73>: mov 0x4(%eax),%ecx
0xc016b398 <nfsd_findparent+76>: mov 0x18(%esi),%edx
0xc016b39b <nfsd_findparent+79>: mov %ecx,0x4(%edx)
0xc016b39e <nfsd_findparent+82>: mov %edx,(%ecx)
0xc016b3a0 <nfsd_findparent+84>: mov %eax,0x18(%esi)
0xc016b3a3 <nfsd_findparent+87>: mov %eax,0x1c(%esi)
0xc016b3a6 <nfsd_findparent+90>: movb $0x1,0xc0329898
d_drop(tdentry); /* we never want ".." hashed */
0xc016b3ad <nfsd_findparent+97>: test %ebx,%ebx
0xc016b3af <nfsd_findparent+99>: jne 0xc016b426 <nfsd_findparent+218>
if (!pdentry) {
0xc016b3b1 <nfsd_findparent+101>: mov 0x8(%esi),%eax
0xc016b3b4 <nfsd_findparent+104>: lea 0x10(%eax),%edx // edx holds aliases
struct list_head *aliases = &tdentry->d_inode->i_dentry;
// is tdentry->d_inode->i_dentry not valid anymore? aliases gets NULL
0xc016b3b7 <nfsd_findparent+107>: lock decb 0xc0329898
0xc016b3be <nfsd_findparent+114>: js 0xc02aefb4 <stext_lock+12564>
spin_lock(&dcache_lock);
0xc016b3c4 <nfsd_findparent+120>: mov 0x10(%eax),%eax <=====================
CRASH !!!
0xc016b3c7 <nfsd_findparent+123>: cmp %edx,%eax // eax holds aliases->next
if (aliases->next != aliases) {
0xc016b3c9 <nfsd_findparent+125>: je 0xc016b3ec <nfsd_findparent+160>
0xc016b3cb <nfsd_findparent+127>: lea 0xffffffc8(%eax),%ebx
pdentry = list_entry(aliases->next, struct dentry, d_alias);
0xc016b3ce <nfsd_findparent+130>: cmp %esi,%ebx
0xc016b3d0 <nfsd_findparent+132>: jne 0xc016b3d8 <nfsd_findparent+140>
0xc016b3d2 <nfsd_findparent+134>: mov 0x4(%edx),%ebx
0xc016b3d5 <nfsd_findparent+137>: add $0xffffffc8,%ebx
0xc016b3d8 <nfsd_findparent+140>: xor %eax,%eax
0xc016b3da <nfsd_findparent+142>: cmp %esi,%ebx
0xc016b3dc <nfsd_findparent+144>: cmove %eax,%ebx
0xc016b3df <nfsd_findparent+147>: test %ebx,%ebx
0xc016b3e1 <nfsd_findparent+149>: je 0xc016b3ec <nfsd_findparent+160>
0xc016b3e3 <nfsd_findparent+151>: push %ebx
0xc016b3e4 <nfsd_findparent+152>: call 0xc0144080 <dget_locked>
0xc016b3e9 <nfsd_findparent+157>: add $0x4,%esp
if (pdentry) dget_locked(pdentry);
0xc016b3ec <nfsd_findparent+160>: movb $0x1,0xc0329898
0xc016b3f3 <nfsd_findparent+167>: test %ebx,%ebx
0xc016b3f5 <nfsd_findparent+169>: jne 0xc016b41c <nfsd_findparent+208>
spin_unlock(&dcache_lock);
0xc016b3f7 <nfsd_findparent+171>: mov 0x8(%esi),%eax
0xc016b3fa <nfsd_findparent+174>: push %eax
0xc016b3fb <nfsd_findparent+175>: call 0xc0145ee8 <igrab>
0xc016b400 <nfsd_findparent+180>: push %eax
0xc016b401 <nfsd_findparent+181>: call 0xc01447f4 <d_alloc_root>
0xc016b406 <nfsd_findparent+186>: mov %eax,%ebx
0xc016b408 <nfsd_findparent+188>: add $0x8,%esp
pdentry = d_alloc_root(igrab(tdentry->d_inode));
0xc016b40b <nfsd_findparent+191>: test %ebx,%ebx
0xc016b40d <nfsd_findparent+193>: je 0xc016b41c <nfsd_findparent+208>
0xc016b40f <nfsd_findparent+195>: orb $0x4,0x4(%ebx)
0xc016b413 <nfsd_findparent+199>: push %ebx
0xc016b414 <nfsd_findparent+200>: call 0xc0144ab4 <d_rehash>
0xc016b419 <nfsd_findparent+205>: add $0x4,%esp
0xc016b41c <nfsd_findparent+208>: mov $0xfffffff4,%eax
0xc016b421 <nfsd_findparent+213>: test %ebx,%ebx
0xc016b423 <nfsd_findparent+215>: cmove %eax,%ebx
0xc016b426 <nfsd_findparent+218>: push %esi
} /* if (!pdentry) */
0xc016b427 <nfsd_findparent+219>: call 0xc0143e90 <dput>
0xc016b42c <nfsd_findparent+224>: mov %ebx,%eax
0xc016b42e <nfsd_findparent+226>: add $0x4,%esp
0xc016b431 <nfsd_findparent+229>: pop %ebx
0xc016b432 <nfsd_findparent+230>: pop %esi
0xc016b433 <nfsd_findparent+231>: ret
End of assembler dump.
(gdb)
2.4.3-xfs is compiled with gcc 2.95.2
2.4.9-XFS is compiled with gcc 2.91.6
After the crash I tried to reboot, but it quickly failed again.
reboot system boot 2.4.3-XFS Mon Oct 15 18:20 (21:49)
reboot system boot 2.4.3-XFS Mon Oct 15 18:17 (21:52)
reboot system boot 2.4.3-XFS Mon Oct 15 18:13 (21:57)
reboot system boot 2.4.3-XFS Thu Oct 11 16:51 (4+23:18)
reboot system boot 2.4.3-XFS Thu Oct 11 16:32 (4+23:37)
reboot system boot 2.4.3-XFS Thu Oct 11 16:28 (4+23:41)
reboot system boot 2.4.3-XFS Thu Oct 11 16:23 (4+23:46)
reboot system boot 2.4.3-XFS Thu Oct 11 16:12 (4+23:57)
reboot system boot 2.4.3-XFS Thu Oct 11 15:05 (5+01:04)
reboot system boot 2.4.9-xfs Thu Oct 4 17:02 (11+23:08)
reboot system boot 2.4.9-xfs Thu Oct 4 16:54 (11+23:15)
reboot system boot 2.4.3-XFS Thu Oct 4 16:47 (11+23:23)
reboot system boot 2.4.9-xfs Thu Oct 4 16:39 (11+23:31)
reboot system boot 2.4.9-xfs Thu Oct 4 16:14 (11+23:55)
To recover I had to unplug/powerdown all connected clients. :-( Uhh.
I think 2.4.9 crashed at the same instructions. But I can't reproduce
it on demand (hey, in the meantime we can work!).
Seeing the NFS related changes in 2.4.10: should I upgrade?
We use only NFSv2. Is NFSv3 more stable, if that matters here?
The machine with 2.4.3 was up for several months - with light load.
Now the number of crashes (with 6 NFS clients using /home + cvs) went up.
We do mount our compile environment in a strange way:
NOTE: /server is a symlink to /usr/local/export
# See exports(5) for a description.
# This file contains a list of all directories exported to other computers.
# It is used by rpc.nfsd and rpc.mountd.
/opt/xxx *.xxx(ro)
/home *.xxx(rw)
/tmp *.xxx(rw)
/usr/local/export *.xxx(rw)
/etc/fstab
devserv:/server/compileenv/lib/lib \
/compenv/xxx/lib nfs ro,auto 0 0
devserv:/server/compileenv/usr_lib/usr/lib \
/compenv/xxx/usr_lib nfs ro,auto 0 0
devserv:/server/compileenv/gcc-lib \
/compenv/xxx/gcc-lib nfs ro,exec,auto 0 0
devserv:/server/compileenv/usr_include/usr/include \
/compenv/xxx/usr_include nfs ro,auto 0 0
devserv:/server/compileenv/usr_linux/usr/linux/include \
/compenv/xxx/usr_linux_include nfs ro,auto 0 0
Would it help to mount a single /usr/local/export and work with a
symlink tree (as I actually do with autofs) ?
What can cause the "invalid" dentries? Did someone remove some files
or were some dentries de'hashed because of dcache growth?
As workaround I will do:
struct list_head *aliases = &tdentry->d_inode->i_dentry;
if (aliases && (aliases->next != aliases) ) {
spin_lock(&dcache_lock);
pdentry = list_entry(aliases->next, struct dentry, d_alias);
if (pdentry == tdentry)
pdentry = list_entry(aliases->prev, struct dentry,
d_alias);
if (pdentry == tdentry)
pdentry = NULL;
if (pdentry) dget_locked(pdentry);
spin_unlock(&dcache_lock);
} else
pdentry=NULL;
Where did you get your kernel (the 2.4.9 version that is) this problem
sounds familiar, but I am pretty sure we fixed this case in XFS somewhere
between 2.4.3 and 2.4.9.
Steve
> noisy:/usr/src/linux # uname -a
> Linux noisy 2.4.3-XFS #8 SMP Sam Mai 19 18:21:36 CEST 2001 i686 unknown
>
> dual board with just one processor
> /usr/local and /home on LogicalVolumes
> tried also 2.4.9-XFS (but see below)
>
> ------------ oops -------------
> Warning (compare_maps): ksyms_base symbol __rta_fill_R__ver___rta_fill not fo
> und in
> System.map. Ignoring ksyms_base entry
> [other warnings stripped]
> Oops: 0000
> CPU: 0
> EIP: 0010:<c016b3c4>
> Using defaults from ksymoops -t elf32-i386 -a i386
> EFLAGS: 00010246
> eax: 00000000 ebx: 00000000 ecx: cff805b8 edx: 00000010
> esi: ca018260 edi: ca0182e0 ebp: cb9e5800 esp: cb9edeec
> ds: 0018 es: 0018 ss: 0018
> Stack: 0100708b ca018360 c016b7d8 ca0182e0 00000002 cba07000 cb9efc00 cb9e580
> 0
> ce078bc4 ca018360 00000000 ca018360 c016bb70 ce078a80 0102a048 0000000
> 5
> Call Trace: <c01bb7d8> <c016bb70> <c016a0b5>c0169b73> <c02a60bf> <c016999d> <
> c01054c4>
> Code: 8b 40 10 39 d0 74 21 8d 58 c8 39 f3 75 06 8b 5a 04 83 c3 c8
>
> >>EIP; c016b3c4 <nfsd_findparent+78/e8> <=====
> Trace; c01bb7d8 <xfs_buf_item_bits+38/58>
> Trace; c016bb70 <fh_verify+26c/48c>
> Trace; c016a0b5 <nfsd_proc_getattr+91/98>
> Code; c016b3c4 <nfsd_findparent+78/e8>
> 00000000 <_EIP>:
> Code; c016b3c4 <nfsd_findparent+78/e8> <=====
> 0: 8b 40 10 mov 0x10(%eax),%eax <=====
> Code; c016b3c7 <nfsd_findparent+7b/e8>
> 3: 39 d0 cmp %edx,%eax
> Code; c016b3c9 <nfsd_findparent+7d/e8>
> 5: 74 21 je 28 <_EIP+0x28> c016b3ec <nfsd_findpa
> rent+a0/e8>
> Code; c016b3cb <nfsd_findparent+7f/e8>
> 7: 8d 58 c8 lea 0xffffffc8(%eax),%ebx
> Code; c016b3ce <nfsd_findparent+82/e8>
> a: 39 f3 cmp %esi,%ebx
> Code; c016b3d0 <nfsd_findparent+84/e8>
> c: 75 06 jne 14 <_EIP+0x14> c016b3d8 <nfsd_findpa
> rent+8c/e8>
> Code; c016b3d2 <nfsd_findparent+86/e8>
> e: 8b 5a 04 mov 0x4(%edx),%ebx
> Code; c016b3d5 <nfsd_findparent+89/e8>
> 11: 83 c3 c8 add $0xffffffc8,%ebx
>
>
> 31 warnings issued. Results may not be reliable.
> ------------ oops -------------
>
> Why do I get all those warnings about symbol mismatch?
>
>
> The code causing the Oops:
>
> struct dentry *nfsd_findparent(struct dentry *child)
> {
> struct dentry *tdentry, *pdentry;
> tdentry = d_alloc(child, &(const struct qstr) {"..", 2, 0});
> if (!tdentry)
> return ERR_PTR(-ENOMEM);
>
> /* I'm going to assume that if the returned dentry is different, then
> * it is well connected. But nobody returns different dentrys do the
> y?
> */
> pdentry = child->d_inode->i_op->lookup(child->d_inode, tdentry);
> d_drop(tdentry); /* we never want ".." hashed */
> if (!pdentry) {
> /* I don't want to return a ".." dentry.
> * I would prefer to return an unconnected "IS_ROOT" dentry,
> * though a properly connected dentry is even better
> */
> /* if first or last of alias list is not tdentry, use that
> * else make a root dentry
> */
> struct list_head *aliases = &tdentry->d_inode->i_dentry;
> spin_lock(&dcache_lock);
> if (aliases->next != aliases) { <=========== CRASH
> !!!!!!!!!!!!!!!!!!!!!!!
> pdentry = list_entry(aliases->next, struct dentry, d_
> alias);
> if (pdentry == tdentry)
> pdentry = list_entry(aliases->prev, struct de
> ntry,
> d_alias);
> if (pdentry == tdentry)
> pdentry = NULL;
> if (pdentry) dget_locked(pdentry);
> }
> spin_unlock(&dcache_lock);
> if (pdentry == NULL) {
> pdentry = d_alloc_root(igrab(tdentry->d_inode));
> if (pdentry) {
> pdentry->d_flags |= DCACHE_NFSD_DISCONNECTED;
> d_rehash(pdentry);
> }
> }
> if (pdentry == NULL)
> pdentry = ERR_PTR(-ENOMEM);
> }
> dput(tdentry); /* it is not hashed, it will be discarded */
> return pdentry;
> }
>
>
> (gdb) disass nfsd_findparent
> Dump of assembler code for function nfsd_findparent:
> 0xc016b34c <nfsd_findparent>: push %esi
> 0xc016b34d <nfsd_findparent+1>: push %ebx
> 0xc016b34e <nfsd_findparent+2>: mov 0xc(%esp,1),%ebx
> 0xc016b352 <nfsd_findparent+6>: push $0xc02ce724
> 0xc016b357 <nfsd_findparent+11>: push %ebx
> 0xc016b358 <nfsd_findparent+12>: call 0xc0144608 <d_alloc>
> 0xc016b35d <nfsd_findparent+17>: mov %eax,%esi
> 0xc016b35f <nfsd_findparent+19>: add $0x8,%esp
> tdentry = d_alloc(child, &(const struct qstr) {"..", 2, 0});
>
> 0xc016b362 <nfsd_findparent+22>: test %esi,%esi
> if (!tdentry)
> 0xc016b364 <nfsd_findparent+24>: jne 0xc016b370 <nfsd_findparent+36
> >
> 0xc016b366 <nfsd_findparent+26>: mov $0xfffffff4,%eax
> 0xc016b36b <nfsd_findparent+31>: jmp 0xc016b431 <nfsd_findparent+22
> 9>
> return ERR_PTR(-ENOMEM);
>
>
> 0xc016b370 <nfsd_findparent+36>: mov 0x8(%ebx),%eax
> 0xc016b373 <nfsd_findparent+39>: mov 0x84(%eax),%edx
> 0xc016b379 <nfsd_findparent+45>: push %esi
> 0xc016b37a <nfsd_findparent+46>: push %eax
> 0xc016b37b <nfsd_findparent+47>: mov 0x4(%edx),%eax
> 0xc016b37e <nfsd_findparent+50>: call *%eax
> 0xc016b380 <nfsd_findparent+52>: mov %eax,%ebx
> 0xc016b382 <nfsd_findparent+54>: add $0x8,%esp
> pdentry = child->d_inode->i_op->lookup(child->d_inode, tdentry);
>
> static __inline__ void d_drop(struct dentry * dentry)
> {
> spin_lock(&dcache_lock);
> list_del(&dentry->d_hash);
> INIT_LIST_HEAD(&dentry->d_hash);
> spin_unlock(&dcache_lock);
> }
>
> 0xc016b385 <nfsd_findparent+57>: lock decb 0xc0329898
> 0xc016b38c <nfsd_findparent+64>: js 0xc02aefa4 <stext_lock+12548>
> 0xc016b392 <nfsd_findparent+70>: lea 0x18(%esi),%eax
> 0xc016b395 <nfsd_findparent+73>: mov 0x4(%eax),%ecx
> 0xc016b398 <nfsd_findparent+76>: mov 0x18(%esi),%edx
> 0xc016b39b <nfsd_findparent+79>: mov %ecx,0x4(%edx)
> 0xc016b39e <nfsd_findparent+82>: mov %edx,(%ecx)
> 0xc016b3a0 <nfsd_findparent+84>: mov %eax,0x18(%esi)
> 0xc016b3a3 <nfsd_findparent+87>: mov %eax,0x1c(%esi)
> 0xc016b3a6 <nfsd_findparent+90>: movb $0x1,0xc0329898
>
> d_drop(tdentry); /* we never want ".." hashed */
>
>
> 0xc016b3ad <nfsd_findparent+97>: test %ebx,%ebx
> 0xc016b3af <nfsd_findparent+99>: jne 0xc016b426 <nfsd_findparent+21
> 8>
> if (!pdentry) {
>
> 0xc016b3b1 <nfsd_findparent+101>: mov 0x8(%esi),%eax
> 0xc016b3b4 <nfsd_findparent+104>: lea 0x10(%eax),%edx // edx holds
> aliases
> struct list_head *aliases = &tdentry->d_inode->i_dentry;
>
> // is tdentry->d_inode->i_dentry not valid anymore? aliases ge
> ts NULL
>
>
> 0xc016b3b7 <nfsd_findparent+107>: lock decb 0xc0329898
> 0xc016b3be <nfsd_findparent+114>: js 0xc02aefb4 <stext_lock+12564>
> spin_lock(&dcache_lock);
>
> 0xc016b3c4 <nfsd_findparent+120>: mov 0x10(%eax),%eax <==========
> ===========
> CRASH !!!
> 0xc016b3c7 <nfsd_findparent+123>: cmp %edx,%eax // eax holds alias
> es->next
> if (aliases->next != aliases) {
>
> 0xc016b3c9 <nfsd_findparent+125>: je 0xc016b3ec <nfsd_findparent+16
> 0>
> 0xc016b3cb <nfsd_findparent+127>: lea 0xffffffc8(%eax),%ebx
> pdentry = list_entry(aliases->next, struct dentry, d_
> alias);
>
> 0xc016b3ce <nfsd_findparent+130>: cmp %esi,%ebx
> 0xc016b3d0 <nfsd_findparent+132>: jne 0xc016b3d8 <nfsd_findparent+14
> 0>
> 0xc016b3d2 <nfsd_findparent+134>: mov 0x4(%edx),%ebx
> 0xc016b3d5 <nfsd_findparent+137>: add $0xffffffc8,%ebx
>
> 0xc016b3d8 <nfsd_findparent+140>: xor %eax,%eax
> 0xc016b3da <nfsd_findparent+142>: cmp %esi,%ebx
> 0xc016b3dc <nfsd_findparent+144>: cmove %eax,%ebx
> 0xc016b3df <nfsd_findparent+147>: test %ebx,%ebx
> 0xc016b3e1 <nfsd_findparent+149>: je 0xc016b3ec <nfsd_findparent+16
> 0>
> 0xc016b3e3 <nfsd_findparent+151>: push %ebx
> 0xc016b3e4 <nfsd_findparent+152>: call 0xc0144080 <dget_locked>
> 0xc016b3e9 <nfsd_findparent+157>: add $0x4,%esp
> if (pdentry) dget_locked(pdentry);
>
> 0xc016b3ec <nfsd_findparent+160>: movb $0x1,0xc0329898
> 0xc016b3f3 <nfsd_findparent+167>: test %ebx,%ebx
> 0xc016b3f5 <nfsd_findparent+169>: jne 0xc016b41c <nfsd_findparent+20
> 8>
> spin_unlock(&dcache_lock);
>
> 0xc016b3f7 <nfsd_findparent+171>: mov 0x8(%esi),%eax
> 0xc016b3fa <nfsd_findparent+174>: push %eax
> 0xc016b3fb <nfsd_findparent+175>: call 0xc0145ee8 <igrab>
> 0xc016b400 <nfsd_findparent+180>: push %eax
> 0xc016b401 <nfsd_findparent+181>: call 0xc01447f4 <d_alloc_root>
> 0xc016b406 <nfsd_findparent+186>: mov %eax,%ebx
> 0xc016b408 <nfsd_findparent+188>: add $0x8,%esp
> pdentry = d_alloc_root(igrab(tdentry->d_inode));
>
> 0xc016b40b <nfsd_findparent+191>: test %ebx,%ebx
> 0xc016b40d <nfsd_findparent+193>: je 0xc016b41c <nfsd_findparent+20
> 8>
> 0xc016b40f <nfsd_findparent+195>: orb $0x4,0x4(%ebx)
> 0xc016b413 <nfsd_findparent+199>: push %ebx
> 0xc016b414 <nfsd_findparent+200>: call 0xc0144ab4 <d_rehash>
> 0xc016b419 <nfsd_findparent+205>: add $0x4,%esp
> 0xc016b41c <nfsd_findparent+208>: mov $0xfffffff4,%eax
> 0xc016b421 <nfsd_findparent+213>: test %ebx,%ebx
> 0xc016b423 <nfsd_findparent+215>: cmove %eax,%ebx
> 0xc016b426 <nfsd_findparent+218>: push %esi
> } /* if (!pdentry) */
> 0xc016b427 <nfsd_findparent+219>: call 0xc0143e90 <dput>
> 0xc016b42c <nfsd_findparent+224>: mov %ebx,%eax
> 0xc016b42e <nfsd_findparent+226>: add $0x4,%esp
> 0xc016b431 <nfsd_findparent+229>: pop %ebx
> 0xc016b432 <nfsd_findparent+230>: pop %esi
> 0xc016b433 <nfsd_findparent+231>: ret
> End of assembler dump.
> (gdb)
>
>
> 2.4.3-xfs is compiled with gcc 2.95.2
> 2.4.9-XFS is compiled with gcc 2.91.6
>
> After the crash I tried to reboot, but it quickly failed again.
> reboot system boot 2.4.3-XFS Mon Oct 15 18:20 (21:49)
> reboot system boot 2.4.3-XFS Mon Oct 15 18:17 (21:52)
> reboot system boot 2.4.3-XFS Mon Oct 15 18:13 (21:57)
>
> reboot system boot 2.4.3-XFS Thu Oct 11 16:51 (4+23:18)
> reboot system boot 2.4.3-XFS Thu Oct 11 16:32 (4+23:37)
> reboot system boot 2.4.3-XFS Thu Oct 11 16:28 (4+23:41)
> reboot system boot 2.4.3-XFS Thu Oct 11 16:23 (4+23:46)
> reboot system boot 2.4.3-XFS Thu Oct 11 16:12 (4+23:57)
> reboot system boot 2.4.3-XFS Thu Oct 11 15:05 (5+01:04)
>
> reboot system boot 2.4.9-xfs Thu Oct 4 17:02 (11+23:08)
> reboot system boot 2.4.9-xfs Thu Oct 4 16:54 (11+23:15)
> reboot system boot 2.4.3-XFS Thu Oct 4 16:47 (11+23:23)
> reboot system boot 2.4.9-xfs Thu Oct 4 16:39 (11+23:31)
> reboot system boot 2.4.9-xfs Thu Oct 4 16:14 (11+23:55)
>
>
> To recover I had to unplug/powerdown all connected clients. :-( Uhh.
> I think 2.4.9 crashed at the same instructions. But I can't reproduce
> it on demand (hey, in the meantime we can work!).
> Seeing the NFS related changes in 2.4.10: should I upgrade?
> We use only NFSv2. Is NFSv3 more stable, if that matters here?
>
> The machine with 2.4.3 was up for several months - with light load.
> Now the number of crashes (with 6 NFS clients using /home + cvs) went up.
>
> We do mount our compile environment in a strange way:
>
> NOTE: /server is a symlink to /usr/local/export
> # See exports(5) for a description.
> # This file contains a list of all directories exported to other computers.
> # It is used by rpc.nfsd and rpc.mountd.
> /opt/xxx *.xxx(ro)
> /home *.xxx(rw)
> /tmp *.xxx(rw)
> /usr/local/export *.xxx(rw)
>
>
> /etc/fstab
> devserv:/server/compileenv/lib/lib \
> /compenv/xxx/lib nfs ro,auto 0 0
> devserv:/server/compileenv/usr_lib/usr/lib \
> /compenv/xxx/usr_lib nfs ro,auto 0 0
> devserv:/server/compileenv/gcc-lib \
> /compenv/xxx/gcc-lib nfs ro,exec,auto 0 0
> devserv:/server/compileenv/usr_include/usr/include \
> /compenv/xxx/usr_include nfs ro,auto 0 0
> devserv:/server/compileenv/usr_linux/usr/linux/include \
> /compenv/xxx/usr_linux_include nfs ro,auto 0 0
>
> Would it help to mount a single /usr/local/export and work with a
> symlink tree (as I actually do with autofs) ?
> What can cause the "invalid" dentries? Did someone remove some files
> or were some dentries de'hashed because of dcache growth?
>
> As workaround I will do:
>
> struct list_head *aliases = &tdentry->d_inode->i_dentry;
>
> if (aliases && (aliases->next != aliases) ) {
> spin_lock(&dcache_lock);
> pdentry = list_entry(aliases->next, struct dentry, d_
> alias);
> if (pdentry == tdentry)
> pdentry = list_entry(aliases->prev, struct de
> ntry,
> d_alias);
> if (pdentry == tdentry)
> pdentry = NULL;
> if (pdentry) dget_locked(pdentry);
> spin_unlock(&dcache_lock);
> } else
> pdentry=NULL;
> -
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to [email protected]
> More majordomo info at http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at http://www.tux.org/lkml/
>
Steve Lord wrote:
>
> Where did you get your kernel (the 2.4.9 version that is) this problem
> sounds familiar, but I am pretty sure we fixed this case in XFS somewhere
> between 2.4.3 and 2.4.9.
>
ftp://oss.sgi.com/projects/xfs/download/patches/linux-2.4.9-xfs-2001-08-26.patch.bz2
the 2.4.3 was there in May, 2001.
-rwxr-xr-x 1 peewee peewee 47713 Mai 17 16:15 linux-2.4.3-core-xfs-1.0.patch.gz
I cannot confirm the Oops in 2.4.9-xfs, but I _think_ it was in the same place.
Do you think, it's xfs related? Some dentries vanashing beneath xfs?
What version do you want me to try, 2.4.12-xfs?
I will apply my "safety" patch anyway (with a printk) ;-)
>
> > noisy:/usr/src/linux # uname -a
> > Linux noisy 2.4.3-XFS #8 SMP Sam Mai 19 18:21:36 CEST 2001 i686 unknown
> >
> > dual board with just one processor
> > /usr/local and /home on LogicalVolumes
> > tried also 2.4.9-XFS (but see below)
> >
> > ------------ oops -------------
> > Warning (compare_maps): ksyms_base symbol __rta_fill_R__ver___rta_fill not fo
> > und in
> > System.map. Ignoring ksyms_base entry
> > [other warnings stripped]
> > Oops: 0000
> > CPU: 0
> > EIP: 0010:<c016b3c4>
> > Using defaults from ksymoops -t elf32-i386 -a i386
> > EFLAGS: 00010246
> > eax: 00000000 ebx: 00000000 ecx: cff805b8 edx: 00000010
> > esi: ca018260 edi: ca0182e0 ebp: cb9e5800 esp: cb9edeec
> > ds: 0018 es: 0018 ss: 0018
> > Stack: 0100708b ca018360 c016b7d8 ca0182e0 00000002 cba07000 cb9efc00 cb9e580
> > 0
> > ce078bc4 ca018360 00000000 ca018360 c016bb70 ce078a80 0102a048 0000000
> > 5
> > Call Trace: <c01bb7d8> <c016bb70> <c016a0b5>c0169b73> <c02a60bf> <c016999d> <
> > c01054c4>
> > Code: 8b 40 10 39 d0 74 21 8d 58 c8 39 f3 75 06 8b 5a 04 83 c3 c8
> >
> > >>EIP; c016b3c4 <nfsd_findparent+78/e8> <=====
> > Trace; c01bb7d8 <xfs_buf_item_bits+38/58>
> > Trace; c016bb70 <fh_verify+26c/48c>
> > Trace; c016a0b5 <nfsd_proc_getattr+91/98>
> > Code; c016b3c4 <nfsd_findparent+78/e8>
> > 00000000 <_EIP>:
> > Code; c016b3c4 <nfsd_findparent+78/e8> <=====
> > 0: 8b 40 10 mov 0x10(%eax),%eax <=====
> > Code; c016b3c7 <nfsd_findparent+7b/e8>
> > 3: 39 d0 cmp %edx,%eax
> > Code; c016b3c9 <nfsd_findparent+7d/e8>
> > 5: 74 21 je 28 <_EIP+0x28> c016b3ec <nfsd_findpa
> > rent+a0/e8>
> > Code; c016b3cb <nfsd_findparent+7f/e8>
> > 7: 8d 58 c8 lea 0xffffffc8(%eax),%ebx
> > Code; c016b3ce <nfsd_findparent+82/e8>
> > a: 39 f3 cmp %esi,%ebx
> > Code; c016b3d0 <nfsd_findparent+84/e8>
> > c: 75 06 jne 14 <_EIP+0x14> c016b3d8 <nfsd_findpa
> > rent+8c/e8>
> > Code; c016b3d2 <nfsd_findparent+86/e8>
> > e: 8b 5a 04 mov 0x4(%edx),%ebx
> > Code; c016b3d5 <nfsd_findparent+89/e8>
> > 11: 83 c3 c8 add $0xffffffc8,%ebx
> >
> >
> > 31 warnings issued. Results may not be reliable.
> > ------------ oops -------------
> >
[...]
Hi,
In my previous posts I was thinking why my PII/375
system can't handle more than 25000 packet/s. I repaired
kernel's NET_PROFILEr and got an answer.
So that for curious, first gprof output of program doing
send/poll/recv of 200 byte packets loop on the loopback
interface using PF_PACKET (total running time was 20 sec):
49.67 9.93 9.93 549384 0.02 0.02 sendto
17.71 13.47 3.54 549384 0.01 0.01 recv
12.01 15.87 2.40 553784 0.00 0.00 poll
Now we compare them with kernel profile stats:
af_packet_send: 7876 ms
nested dev_queue_xmit: 3714 ms
net_rx_action: 1464 ms
af_packet_recv: 411 ms
For send gprof tells 9930 ms and kernel 7876 so 2 sec in difference
is generic socket and system call handling overhead (and measure error).
The recv gproffed is 3540 ms, kernel 1464+411 so there is again about
2 sec overhead.
>From this analysis I can finally say that there is not error in my
netflow simulator :) but the 2.4 on PII/375 can handle up to 30000 pps
in userspace loop.
Without userspace overhead and not bouncing packets (like in my test)
the router based on 2.4 could handle about 70000 pps without qos,routing
and filtering overhead ..
Someone tested in real world ?
best regards, devik
Steve Lord wrote:
>
> Where did you get your kernel (the 2.4.9 version that is) this problem
> sounds familiar, but I am pretty sure we fixed this case in XFS somewhere
> between 2.4.3 and 2.4.9.
>
The following diff was made in 2.4.4.
diff -u --recursive --new-file v2.4.4/linux/fs/nfsd/nfsfh.c linux/fs/nfsd/nfsfh.c
--- v2.4.4/linux/fs/nfsd/nfsfh.c Fri Feb 9 11:29:44 2001
+++ linux/fs/nfsd/nfsfh.c Sat May 19 17:47:55 2001
@@ -244,6 +245,11 @@
*/
pdentry = child->d_inode->i_op->lookup(child->d_inode, tdentry);
d_drop(tdentry); /* we never want ".." hashed */
+ if (!pdentry && tdentry->d_inode == NULL) {
+ /* File system cannot find ".." ... sad but possible */
+ dput(tdentry);
+ pdentry = ERR_PTR(-EINVAL);
+ }
But it would not prevent the code path 2.4.3-xfs hit.
pdentry is !=NULL and tdentry->d_inode is always NULL after d_alloc():611
Does xfs' child->d_inode->i_op->lookup(child->d_inode, tdentry);
change the contents of tdentry if the lookup("..") succeeds?
Then I suggest:
! if (!pdentry || tdentry->d_inode == NULL) {
What do you think?
BTW, when does a lookup("..") fail? Even in "/", lookup("..") returns "."
> > struct dentry *nfsd_findparent(struct dentry *child)
> > {
> > struct dentry *tdentry, *pdentry;
> > tdentry = d_alloc(child, &(const struct qstr) {"..", 2, 0});
> > if (!tdentry)
> > return ERR_PTR(-ENOMEM);
> >
> > /* I'm going to assume that if the returned dentry is different, then
> > * it is well connected. But nobody returns different dentrys do they?
> > */
> > pdentry = child->d_inode->i_op->lookup(child->d_inode, tdentry);
> > d_drop(tdentry); /* we never want ".." hashed */
+ if (!pdentry && tdentry->d_inode == NULL) {
+ /* File system cannot find ".." ... sad but possible */
+ dput(tdentry);
+ pdentry = ERR_PTR(-EINVAL);
+ }
> > if (!pdentry) {
> > /* I don't want to return a ".." dentry.
> > * I would prefer to return an unconnected "IS_ROOT" dentry,
> > * though a properly connected dentry is even better
> > */
> > /* if first or last of alias list is not tdentry, use that
> > * else make a root dentry
> > */
> > struct list_head *aliases = &tdentry->d_inode->i_dentry;
> > spin_lock(&dcache_lock);
> > if (aliases->next != aliases) { <=========== CRASH
> > pdentry = list_entry(aliases->next, struct dentry, d_
Peter W?chtler wrote:
>
> Steve Lord wrote:
> >
> > Where did you get your kernel (the 2.4.9 version that is) this problem
> > sounds familiar, but I am pretty sure we fixed this case in XFS somewhere
> > between 2.4.3 and 2.4.9.
> >
>
> The following diff was made in 2.4.4.
>
> diff -u --recursive --new-file v2.4.4/linux/fs/nfsd/nfsfh.c linux/fs/nfsd/nfsfh.c
> --- v2.4.4/linux/fs/nfsd/nfsfh.c Fri Feb 9 11:29:44 2001
> +++ linux/fs/nfsd/nfsfh.c Sat May 19 17:47:55 2001
> @@ -244,6 +245,11 @@
> */
> pdentry = child->d_inode->i_op->lookup(child->d_inode, tdentry);
> d_drop(tdentry); /* we never want ".." hashed */
> + if (!pdentry && tdentry->d_inode == NULL) {
> + /* File system cannot find ".." ... sad but possible */
> + dput(tdentry);
> + pdentry = ERR_PTR(-EINVAL);
> + }
>
> But it would not prevent the code path 2.4.3-xfs hit.
> pdentry is !=NULL and tdentry->d_inode is always NULL after d_alloc():611
>
Damn. pdentry IS NULL.
Sorry, the patch would prevent the crash.
> Peter Wdchtler wrote:
> >
> > Steve Lord wrote:
> > >
> > > Where did you get your kernel (the 2.4.9 version that is) this problem
> > > sounds familiar, but I am pretty sure we fixed this case in XFS somewhere
> > > between 2.4.3 and 2.4.9.
> > >
> >
> > The following diff was made in 2.4.4.
> >
> > diff -u --recursive --new-file v2.4.4/linux/fs/nfsd/nfsfh.c linux/fs/nfsd/n
> fsfh.c
> > --- v2.4.4/linux/fs/nfsd/nfsfh.c Fri Feb 9 11:29:44 2001
> > +++ linux/fs/nfsd/nfsfh.c Sat May 19 17:47:55 2001
> > @@ -244,6 +245,11 @@
> > */
> > pdentry = child->d_inode->i_op->lookup(child->d_inode, tdentry);
> > d_drop(tdentry); /* we never want ".." hashed */
> > + if (!pdentry && tdentry->d_inode == NULL) {
> > + /* File system cannot find ".." ... sad but possible */
> > + dput(tdentry);
> > + pdentry = ERR_PTR(-EINVAL);
> > + }
> >
> > But it would not prevent the code path 2.4.3-xfs hit.
> > pdentry is !=NULL and tdentry->d_inode is always NULL after d_alloc():611
> >
>
> Damn. pdentry IS NULL.
> Sorry, the patch would prevent the crash.
OK, I am back again....
I went and looked through xfs code changes, and the 2.4.9 patch should
have some changes in it related to this type of issue. However, you would
have to be using xfsdump to trigger them I think, or possibly the xfs_fsr
program. The combination of xfsdump and an nfs server did have some problems.
So, does this tie in with your configuation? It does not however explain
problems in 2.4.9. There was just a recent code cleanup of the code paths
used by xfsdump (post 2.4.12 patch), but I am not aware of any other XFS
changes in this area.
XFS does work OK with the new VM, so you can try later kernels if you want,
I am not aware of anyone else having problems with XFS which appear dcache
related.
Steve (slightly stumped and in need of coffee)
> -
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to [email protected]
> More majordomo info at http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at http://www.tux.org/lkml/
>