Hello Ana,
I have done some more testing.
Kindly look into it.
Setup
NFS Server IP:192.168.122.127
NFS Client IP:192.168.122.125
1- Transport Viewing
# ss
Netid State Recv-Q Send-Q Local
Address:Port Peer Address:Port Process
tcp ESTAB 0 0
192.168.122.125:872 192.168.122.127:nfs
[root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py xprt
xprt 1: tcp, 192.168.122.127, port 0, state <CONNECTED,BOUND>, main
Source: (enoent), port 872, Requests: 2
Congestion: cur 0, win 256, Slots: min 2, max 65536
Queues: binding 0, sending 0, pending 0, backlog 0, tasks 0
Here port 0 is seen for Remote which is wrong. It should be nfs(2049).
And I guess the name is also wrong. it should not be enoent. It should be
ens3.
2- I made the NIC down on the Server. And can see call traces as in the
attached image. (taken from console so can't paste here)
3- By client I understand RPC Client and if I tune the value of
tcp_slot_table_entries I should see an increase in number of RPC Client as
I would increase parallel connection of RPC Clients.
# ./tools/rpcctl/rpcctl.py client
client 0: switch 0, xprts 1, active 1, queue 0
xprt 1: tcp, 192.168.122.127 [main]
client 3: switch 0, xprts 1, active 1, queue 0
xprt 1: tcp, 192.168.122.127 [main]
# sysctl -a | grep tcp_slot_table_entries
sunrpc.tcp_slot_table_entries = 2
[root@rrathore-upstream-sysfs nfs-utils]# sysctl -w
sunrpc.tcp_slot_table_entries=8
sunrpc.tcp_slot_table_entries = 8
[root@rrathore-upstream-sysfs nfs-utils]#
[root@rrathore-upstream-sysfs nfs-utils]# sysctl -a | grep
tcp_slot_table_entries
sunrpc.tcp_slot_table_entries = 8
[root@rrathore-upstream-sysfs nfs-utils]#
[root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py client
<----------------------- Change in value of tcp_slot_table_entries doesn't
seem to have an effect
client 0: switch 0, xprts 1, active 1, queue 0
xprt 1: tcp, 192.168.122.127 [main]
client 3: switch 0, xprts 1, active 1, queue 0
xprt 1: tcp, 192.168.122.127 [main]
Later I started nfs and used this server as an NFS Server, then I could see
an increase in number.
# ./tools/rpcctl/rpcctl.py client
client 0: switch 0, xprts 1, active 1, queue 0
xprt 1: tcp, 192.168.122.127 [main]
client 1: switch 1, xprts 1, active 1, queue 0
xprt 0: local, /var/run/rpcbind.sock [main]
client 2: switch 1, xprts 1, active 1, queue 0
xprt 0: local, /var/run/rpcbind.sock [main]
client 3: switch 0, xprts 1, active 1, queue 0
xprt 1: tcp, 192.168.122.127 [main]
client 4: switch 2, xprts 1, active 1, queue 0
xprt 2: local, /var/run/gssproxy.sock [main]
client 5: switch 3, xprts 1, active 1, queue 0
xprt 3: tcp, 192.168.122.29 [main]
4- I am not sure if I am making a mistake or if it's the error due to which
value is not getting set.
[root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py xprt
xprt 0: local, /var/run/rpcbind.sock, port 0, state <CONNECTED,BOUND>, main
Source: (enoent), port 0, Requests: 8
Congestion: cur 0, win 256, Slots: min 8, max 65536
Queues: binding 0, sending 0, pending 0, backlog 0, tasks 0
xprt 1: tcp, 192.168.122.127, port 0, state <CONNECTED,BOUND>, main
Source: (enoent), port 813, Requests: 2
Congestion: cur 0, win 256, Slots: min 2, max 65536
Queues: binding 0, sending 0, pending 0, backlog 0, tasks 0
xprt 2: local, /var/run/gssproxy.sock, port 0, state <CONNECTED,BOUND>, main
Source: (enoent), port 0, Requests: 8
Congestion: cur 0, win 256, Slots: min 8, max 65536
Queues: binding 0, sending 0, pending 0, backlog 0, tasks 0
xprt 3: tcp, 192.168.122.29, port 0, state <CONNECTED,BOUND>, main
Source: (enoent), port 0, Requests: 8
Congestion: cur 0, win 256, Slots: min 8, max 8
Queues: binding 0, sending 0, pending 0, backlog 0, tasks 0
*None of the operation work*
[root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py xprt set
-h
usage: rpcctl.py xprt set [-h] --id ID [--dstaddr dstaddr] [--offline]
[--online] [--remove]
options:
-h, --help show this help message and exit
--id ID Id of a specific xprt to modify
--dstaddr dstaddr New dstaddr to set
--offline Set an xprt offline
--online Set an offline xprt back online
--remove Remove an xprt
[root@rrathore-upstream-sysfs nfs-utils]#
[root@rrathore-upstream-sysfs nfs-utils]#
[root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py xprt set
--id 3 --offline
[Errno 22] Invalid argument
[root@rrathore-upstream-sysfs nfs-utils]#
[root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py xprt set
--id 3 192.168.122.29 --offline
usage: rpcctl.py [-h] {client,switch,xprt} ...
rpcctl.py: error: unrecognized arguments: 192.168.122.29
[root@rrathore-upstream-sysfs nfs-utils]#
[root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py xprt set
--id 3 --dstaddr 192.168.122.29 --offline
[Errno 95] Operation not supported
[root@rrathore-upstream-sysfs nfs-utils]#
[root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py xprt set
--id 3 --dstaddr 192.168.122.29 --online
[Errno 95] Operation not supported
[root@rrathore-upstream-sysfs nfs-utils]#
[root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py xprt set
--id 3 --dstaddr 192.168.122.29 --remove
[Errno 95] Operation not supported
[root@rrathore-upstream-sysfs nfs-utils]#
[root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py xprt set
--id 1 --dstaddr 192.168.122.127 --offline
[Errno 22] Invalid argument
[root@rrathore-upstream-sysfs nfs-utils]#
[root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py xprt set
--id 1 --offline
[Errno 22] Invalid argument
5-* And it's similar If I do with switch*
[root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py switch
switch 0: xprts 1, active 1, queue 0
xprt 1: tcp, 192.168.122.127 [main]
switch 1: xprts 1, active 1, queue 0
xprt 0: local, /var/run/rpcbind.sock [main]
switch 2: xprts 1, active 1, queue 0
xprt 2: local, /var/run/gssproxy.sock [main]
switch 3: xprts 1, active 1, queue 0
xprt 3: tcp, 192.168.122.29 [main]
[root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py switch
set --id 3 --dstaddr 192.168.122.30
[Errno 95] Operation not supported
[root@rrathore-upstream-sysfs nfs-utils]#
[root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py switch
set --id 3 --dstaddr 192.168.122.29
[Errno 95] Operation not supported
Regards,
Rahul
Hi Rahul,
Thanks for testing!
On Sat, Feb 5, 2022 at 7:19 AM Rahul Rathore <[email protected]> wrote:
>
> Hello Ana,
>
> I have done some more testing.
>
> Kindly look into it.
>
> Setup
>
> NFS Server IP:192.168.122.127
> NFS Client IP:192.168.122.125
>
>
> 1- Transport Viewing
>
> # ss
> Netid State Recv-Q Send-Q Local
> Address:Port Peer Address:Port Process
> tcp ESTAB 0 0
> 192.168.122.125:872 192.168.122.127:nfs
>
>
> [root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py xprt
> xprt 1: tcp, 192.168.122.127, port 0, state <CONNECTED,BOUND>, main
> Source: (enoent), port 872, Requests: 2
> Congestion: cur 0, win 256, Slots: min 2, max 65536
> Queues: binding 0, sending 0, pending 0, backlog 0, tasks 0
>
>
>
> Here port 0 is seen for Remote which is wrong. It should be nfs(2049).
Can I ask what kernel you are running? This was a kernel-side issue
that was fixed in v5.15 with this patch:
commit 5d46dd04cb68771f77ba66dbf6fd323a4a2ce00d
Author: Anna Schumaker <[email protected]>
Date: Tue Jul 20 16:04:42 2021 -0400
sunrpc: Fix return value of get_srcport()
Since bc1c56e9bbe9 transport->srcport may by unset, causing
get_srcport() to return 0 when called. Fix this by querying the port
from the underlying socket instead of the transport.
Fixes: bc1c56e9bbe9 (SUNRPC: prevent port reuse on transports
which don't request it)
Signed-off-by: Anna Schumaker <[email protected]>
>
> And I guess the name is also wrong. it should not be enoent. It should be
> ens3.
enoent in this case means the file it was trying to read doesn't
exist. Probably you need this patch (also in v5.15):
commit e44773daf851dc2755144355723c1c305e7246a1
Author: Anna Schumaker <[email protected]>
Date: Thu Jul 29 16:45:23 2021 -0400
SUNRPC: Add srcaddr as a file in sysfs
I don't support changing it right now, but it could be useful
information for clients with multiple network cards.
Signed-off-by: Anna Schumaker <[email protected]>
>
>
> 2- I made the NIC down on the Server. And can see call traces as in the
> attached image. (taken from console so can't paste here)
I think this was the same bug that was fixed with this patch (which
went into v5.15-rc3):
commit 17f09d3f619a7ad2d2b021b4e5246f08225b1b0f
Author: Anna Schumaker <[email protected]>
Date: Thu Oct 28 15:17:41 2021 -0400
SUNRPC: Check if the xprt is connected before handling sysfs reads
xprts don't immediately reconnect when changing the "dstaddr" property,
instead this gets handled the next time an operation uses the transport.
This could lead to NULL pointer dereferences when trying to read sysfs
files between the disconnect and reconnect operations. Fix this by
returning an error if the xprt is not connected.
Signed-off-by: Anna Schumaker <[email protected]>
Signed-off-by: Trond Myklebust <[email protected]>
>
> 3- By client I understand RPC Client and if I tune the value of
> tcp_slot_table_entries I should see an increase in number of RPC Client as
> I would increase parallel connection of RPC Clients.
I don't remember offhand how using the tcp_slot_table_entries works
for increasing RPC clients, so there might be some other trigger
you're missing. Try using the nconnect=N mount option instead, since
extra connections are set up at mount time.
>
> # ./tools/rpcctl/rpcctl.py client
> client 0: switch 0, xprts 1, active 1, queue 0
> xprt 1: tcp, 192.168.122.127 [main]
> client 3: switch 0, xprts 1, active 1, queue 0
> xprt 1: tcp, 192.168.122.127 [main]
> # sysctl -a | grep tcp_slot_table_entries
> sunrpc.tcp_slot_table_entries = 2
>
> [root@rrathore-upstream-sysfs nfs-utils]# sysctl -w
> sunrpc.tcp_slot_table_entries=8
> sunrpc.tcp_slot_table_entries = 8
> [root@rrathore-upstream-sysfs nfs-utils]#
> [root@rrathore-upstream-sysfs nfs-utils]# sysctl -a | grep
> tcp_slot_table_entries
> sunrpc.tcp_slot_table_entries = 8
> [root@rrathore-upstream-sysfs nfs-utils]#
> [root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py client
> <----------------------- Change in value of tcp_slot_table_entries doesn't
> seem to have an effect
> client 0: switch 0, xprts 1, active 1, queue 0
> xprt 1: tcp, 192.168.122.127 [main]
> client 3: switch 0, xprts 1, active 1, queue 0
> xprt 1: tcp, 192.168.122.127 [main]
>
> Later I started nfs and used this server as an NFS Server, then I could see
> an increase in number.
>
> # ./tools/rpcctl/rpcctl.py client
> client 0: switch 0, xprts 1, active 1, queue 0
> xprt 1: tcp, 192.168.122.127 [main]
> client 1: switch 1, xprts 1, active 1, queue 0
> xprt 0: local, /var/run/rpcbind.sock [main]
> client 2: switch 1, xprts 1, active 1, queue 0
> xprt 0: local, /var/run/rpcbind.sock [main]
> client 3: switch 0, xprts 1, active 1, queue 0
> xprt 1: tcp, 192.168.122.127 [main]
> client 4: switch 2, xprts 1, active 1, queue 0
> xprt 2: local, /var/run/gssproxy.sock [main]
> client 5: switch 3, xprts 1, active 1, queue 0
> xprt 3: tcp, 192.168.122.29 [main]
>
>
> 4- I am not sure if I am making a mistake or if it's the error due to which
> value is not getting set.
>
> [root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py xprt
> xprt 0: local, /var/run/rpcbind.sock, port 0, state <CONNECTED,BOUND>, main
> Source: (enoent), port 0, Requests: 8
> Congestion: cur 0, win 256, Slots: min 8, max 65536
> Queues: binding 0, sending 0, pending 0, backlog 0, tasks 0
> xprt 1: tcp, 192.168.122.127, port 0, state <CONNECTED,BOUND>, main
> Source: (enoent), port 813, Requests: 2
> Congestion: cur 0, win 256, Slots: min 2, max 65536
> Queues: binding 0, sending 0, pending 0, backlog 0, tasks 0
> xprt 2: local, /var/run/gssproxy.sock, port 0, state <CONNECTED,BOUND>, main
> Source: (enoent), port 0, Requests: 8
> Congestion: cur 0, win 256, Slots: min 8, max 65536
> Queues: binding 0, sending 0, pending 0, backlog 0, tasks 0
> xprt 3: tcp, 192.168.122.29, port 0, state <CONNECTED,BOUND>, main
> Source: (enoent), port 0, Requests: 8
> Congestion: cur 0, win 256, Slots: min 8, max 8
> Queues: binding 0, sending 0, pending 0, backlog 0, tasks 0
>
> *None of the operation work*
>
> [root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py xprt set
> -h
> usage: rpcctl.py xprt set [-h] --id ID [--dstaddr dstaddr] [--offline]
> [--online] [--remove]
>
> options:
> -h, --help show this help message and exit
> --id ID Id of a specific xprt to modify
> --dstaddr dstaddr New dstaddr to set
> --offline Set an xprt offline
> --online Set an offline xprt back online
> --remove Remove an xprt
> [root@rrathore-upstream-sysfs nfs-utils]#
> [root@rrathore-upstream-sysfs nfs-utils]#
> [root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py xprt set
> --id 3 --offline
> [Errno 22] Invalid argument
> [root@rrathore-upstream-sysfs nfs-utils]#
> [root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py xprt set
> --id 3 192.168.122.29 --offline
> usage: rpcctl.py [-h] {client,switch,xprt} ...
> rpcctl.py: error: unrecognized arguments: 192.168.122.29
> [root@rrathore-upstream-sysfs nfs-utils]#
> [root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py xprt set
> --id 3 --dstaddr 192.168.122.29 --offline
> [Errno 95] Operation not supported
> [root@rrathore-upstream-sysfs nfs-utils]#
> [root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py xprt set
> --id 3 --dstaddr 192.168.122.29 --online
> [Errno 95] Operation not supported
> [root@rrathore-upstream-sysfs nfs-utils]#
> [root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py xprt set
> --id 3 --dstaddr 192.168.122.29 --remove
> [Errno 95] Operation not supported
> [root@rrathore-upstream-sysfs nfs-utils]#
> [root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py xprt set
> --id 1 --dstaddr 192.168.122.127 --offline
> [Errno 22] Invalid argument
> [root@rrathore-upstream-sysfs nfs-utils]#
> [root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py xprt set
> --id 1 --offline
> [Errno 22] Invalid argument
The commands themselves look okay. Can you update to a kernel that has
the other fixes and let me know if it's still a problem?
Thanks,
Anna
>
>
> 5-* And it's similar If I do with switch*
>
> [root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py switch
> switch 0: xprts 1, active 1, queue 0
> xprt 1: tcp, 192.168.122.127 [main]
> switch 1: xprts 1, active 1, queue 0
> xprt 0: local, /var/run/rpcbind.sock [main]
> switch 2: xprts 1, active 1, queue 0
> xprt 2: local, /var/run/gssproxy.sock [main]
> switch 3: xprts 1, active 1, queue 0
> xprt 3: tcp, 192.168.122.29 [main]
>
> [root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py switch
> set --id 3 --dstaddr 192.168.122.30
> [Errno 95] Operation not supported
> [root@rrathore-upstream-sysfs nfs-utils]#
> [root@rrathore-upstream-sysfs nfs-utils]# ./tools/rpcctl/rpcctl.py switch
> set --id 3 --dstaddr 192.168.122.29
> [Errno 95] Operation not supported
>
> Regards,
> Rahul