From: Andy Adamson <[email protected]>
Here is what I've been using to test DS recovery and MDSThreshold
-->Andy
Andy Adamson (3):
PYNFS use requested iomode on layoutget
PYNFS: add mdsthreshold to file layout server
PYNFS implement FREE_STATEID and TEST_STATEID operations
Fred Isaman (1):
PYNFS rpc drop fix
Weston Andros Adamson (2):
PYNFS fix rpcgen
PYNFS use loopback addresses for MDS<->DS communication
nfs4.1/dataserver.py | 28 ++++++++++++++++++++++------
nfs4.1/fs.py | 12 ++++++++++--
nfs4.1/nfs4lib.py | 2 +-
nfs4.1/nfs4server.py | 33 +++++++++++++++++++++++++++++++++
nfs4.1/nfs4state.py | 11 ++++++-----
rpc/rpc.py | 2 +-
xdr/xdrgen.py | 2 ++
7 files changed, 75 insertions(+), 15 deletions(-)
--
1.7.7.6
From: Andy Adamson <[email protected]>
Signed-off-by: Andy Adamson <[email protected]>
---
nfs4.1/nfs4server.py | 33 +++++++++++++++++++++++++++++++++
nfs4.1/nfs4state.py | 11 ++++++-----
2 files changed, 39 insertions(+), 5 deletions(-)
diff --git a/nfs4.1/nfs4server.py b/nfs4.1/nfs4server.py
index b279292..8d0f112 100755
--- a/nfs4.1/nfs4server.py
+++ b/nfs4.1/nfs4server.py
@@ -1296,6 +1296,39 @@ class NFS4Server(rpc.Server):
res = READ4resok(eof, data)
return encode_status(NFS4_OK, res)
+ def op_free_stateid(self, arg, env):
+ check_session(env)
+ status = NFS4_OK
+ with find_state(env, arg.fsa_stateid, no_fh=True) as state:
+ try:
+ log_41.info("free_stateid found state DELETING state")
+ state.delete()
+ except NFS4Error, e:
+ log_41.info("free_stateid %s error: %d") % (arg.fsa_stateid, e.status)
+ status = e.status
+
+ res = FREE_STATEID4res(fsr_status = status)
+ return encode_status(NFS4_OK)
+
+ def op_test_stateid(self, arg, env):
+ check_session(env)
+ out = []
+ for stateid in arg.ts_stateids:
+ status = NFS4_OK
+ with find_state(env, stateid, no_fh=True) as state:
+ try:
+ log_41.info("test_stateid found state")
+ self.check_opsconfig(env, "test_stateid")
+ except NFS4Error, e:
+ if (e.status == NFS4ERR_BAD_STATEID or e.status == NFS4ERR_ADMIN_REVOKED or e.status == NFS4ERR_OPENMODE):
+ state.delete()
+ status = e.status
+ finally:
+ out.append(status)
+
+ res = TEST_STATEID4resok(tsr_status_codes = out)
+ return encode_status(NFS4_OK, res)
+
def op_open(self, arg, env):
self.check_opsconfig(env, "open")
check_session(env)
diff --git a/nfs4.1/nfs4state.py b/nfs4.1/nfs4state.py
index e8445e3..a6b21f4 100644
--- a/nfs4.1/nfs4state.py
+++ b/nfs4.1/nfs4state.py
@@ -23,7 +23,7 @@ NORMAL, CB_INIT, CB_SENT, CB_RECEIVED, INVALID = range(5) # delegation/layout st
DS_MAGIC = "\xa5" # STUB part of HACK code to ignore DS stateid
@contextmanager
-def find_state(env, stateid, allow_0=True, allow_bypass=False):
+def find_state(env, stateid, allow_0=True, allow_bypass=False, no_fh=False):
"""Find the matching StateTableEntry, and manage its lock."""
anon = False
if env.is_ds:
@@ -57,10 +57,11 @@ def find_state(env, stateid, allow_0=True, allow_bypass=False):
state = env.session.client.state.get(stateid.other, None)
if state is None:
raise NFS4Error(NFS4ERR_BAD_STATEID, tag="stateid not known")
- if state.file != env.cfh:
- raise NFS4Error(NFS4ERR_BAD_STATEID,
- tag="cfh %r does not match stateid %r" %
- (state.file.fh, env.cfh.fh))
+ if no_fh == False:
+ if state.file != env.cfh:
+ raise NFS4Error(NFS4ERR_BAD_STATEID,
+ tag="cfh %r does not match stateid %r" %
+ (state.file.fh, env.cfh.fh))
state.lock.acquire()
# It is possible that while waiting to get the lock, the state has been
# removed. In that case, the removal sets the invalid flag.
--
1.7.7.6
From: Weston Andros Adamson <[email protected]>
if a loopback addresses is specified for a DS, use that address for MDS to DS
communication and remove it from the advertised addresses in getdeviceinfo
(it obviously would never work).
this is useful for testing network partitions - you can bring down / firewall
off an interface without breaking MDS<->DS communications.
an example dataservers.conf:
192.168.55.1:12345,127.0.0.1:12345/pynfs_mds
192.168.55.1:12346,127.0.0.1:12346/pynfs_mds
Signed-off-by: Weston Andros Adamson <[email protected]>
---
nfs4.1/dataserver.py | 28 ++++++++++++++++++++++------
1 files changed, 22 insertions(+), 6 deletions(-)
diff --git a/nfs4.1/dataserver.py b/nfs4.1/dataserver.py
index 65e4f55..dd2a535 100644
--- a/nfs4.1/dataserver.py
+++ b/nfs4.1/dataserver.py
@@ -187,9 +187,27 @@ class DSDevice(object):
log.critical("Could not parse line: %r" % line)
sys.exit(1)
- # for now, just use the last path for local connections
- server, port = server_list[-1]
- server_list = server_list[:-1]
+ # if there is a (IPv4) loopback address in the multipath list,
+ # use that as the connection to the DS, but dont advertise it
+ loopback_idx = -1
+ for i, serverport in enumerate(server_list):
+ if serverport[0] == '127.0.0.1':
+ loopback_idx = i
+ break
+
+ if loopback_idx >= 0:
+ print "Using loopback address for MDS->DS communication only"
+ server_list = list(server_list)
+ server, port = server_list.pop(loopback_idx)
+ else:
+ # just use the last path for local connections
+ server, port = server_list[-1]
+
+ if not server_list:
+ log.critical("At least one non-loopback address must " +
+ "be specified for each DS")
+ sys.exit(1)
+
print server, port, path
try:
log.info("Adding dataserver ip:%s port:%s path:%s" %
@@ -210,9 +228,7 @@ class DSDevice(object):
index = 0
for d in self.list:
if d.active:
- multipath = []
- multipath.extend(d.get_multipath_netaddr4s())
- multipath.append(d.get_netaddr4())
+ multipath = d.get_multipath_netaddr4s()
netaddrs.append(multipath)
stripe_indices.append(index)
index = index + 1
--
1.7.7.6
I think I gave this to Fred some time ago. This patch is a hack that only works when the MDS and all data servers are run on the same host.
It's easy enough to use aliases on an interface and bring those down, but the ordering problem remains:
The pynfs MDS <-> DS backchannel uses the first address in the address list and most clients (including linux) will use the first address that works when communicating with data servers, so bringing down the alias that the DS is communicating on will also break the pynfs backchannel.
I'll rework the patch to allow this testing on both localhost and across hosts.
-dros
On May 29, 2012, at 5:57 PM, <[email protected]>
<[email protected]> wrote:
> From: Weston Andros Adamson <[email protected]>
>
> if a loopback addresses is specified for a DS, use that address for MDS to DS
> communication and remove it from the advertised addresses in getdeviceinfo
> (it obviously would never work).
>
> this is useful for testing network partitions - you can bring down / firewall
> off an interface without breaking MDS<->DS communications.
>
> an example dataservers.conf:
>
> 192.168.55.1:12345,127.0.0.1:12345/pynfs_mds
> 192.168.55.1:12346,127.0.0.1:12346/pynfs_mds
>
> Signed-off-by: Weston Andros Adamson <[email protected]>
> ---
> nfs4.1/dataserver.py | 28 ++++++++++++++++++++++------
> 1 files changed, 22 insertions(+), 6 deletions(-)
>
> diff --git a/nfs4.1/dataserver.py b/nfs4.1/dataserver.py
> index 65e4f55..dd2a535 100644
> --- a/nfs4.1/dataserver.py
> +++ b/nfs4.1/dataserver.py
> @@ -187,9 +187,27 @@ class DSDevice(object):
> log.critical("Could not parse line: %r" % line)
> sys.exit(1)
>
> - # for now, just use the last path for local connections
> - server, port = server_list[-1]
> - server_list = server_list[:-1]
> + # if there is a (IPv4) loopback address in the multipath list,
> + # use that as the connection to the DS, but dont advertise it
> + loopback_idx = -1
> + for i, serverport in enumerate(server_list):
> + if serverport[0] == '127.0.0.1':
> + loopback_idx = i
> + break
> +
> + if loopback_idx >= 0:
> + print "Using loopback address for MDS->DS communication only"
> + server_list = list(server_list)
> + server, port = server_list.pop(loopback_idx)
> + else:
> + # just use the last path for local connections
> + server, port = server_list[-1]
> +
> + if not server_list:
> + log.critical("At least one non-loopback address must " +
> + "be specified for each DS")
> + sys.exit(1)
> +
> print server, port, path
> try:
> log.info("Adding dataserver ip:%s port:%s path:%s" %
> @@ -210,9 +228,7 @@ class DSDevice(object):
> index = 0
> for d in self.list:
> if d.active:
> - multipath = []
> - multipath.extend(d.get_multipath_netaddr4s())
> - multipath.append(d.get_netaddr4())
> + multipath = d.get_multipath_netaddr4s()
> netaddrs.append(multipath)
> stripe_indices.append(index)
> index = index + 1
> --
> 1.7.7.6
>
From: Fred Isaman <[email protected]>
Signed-off-by: Fred Isaman <[email protected]>
---
rpc/rpc.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/rpc/rpc.py b/rpc/rpc.py
index 38efcd8..3166632 100644
--- a/rpc/rpc.py
+++ b/rpc/rpc.py
@@ -514,7 +514,7 @@ class ConnectionHandler(object):
data = self.sockets[fd].recv_records(self.rsize)
except socket.error:
data = None
- if data:
+ if data is not None:
self._event_read(data, fd)
else:
self._event_close(fd)
--
1.7.7.6
From: Weston Andros Adamson <[email protected]>
Signed-off-by: Weston Andros Adamson <[email protected]>
---
xdr/xdrgen.py | 2 ++
1 files changed, 2 insertions(+), 0 deletions(-)
diff --git a/xdr/xdrgen.py b/xdr/xdrgen.py
index 0148f96..72954d1 100755
--- a/xdr/xdrgen.py
+++ b/xdr/xdrgen.py
@@ -786,6 +786,8 @@ class Info(object):
if t.type in name_dict:
p = name_dict[t.type]
if p.parent and p.type == 'enum':
+ if t.array:
+ return "','.join([ const.%s.get(x, x) for x in self.%s ])" % (p.id, t.id)
return "const.%s.get(self.%s, self.%s)" % (p.id, t.id, t.id)
return "repr(self.%s)" % t.id
indent2 = prefix + indent
--
1.7.7.6
From: Andy Adamson <[email protected]>
Signed-off-by: Andy Adamson <[email protected]>
---
nfs4.1/fs.py | 3 ++-
1 files changed, 2 insertions(+), 1 deletions(-)
diff --git a/nfs4.1/fs.py b/nfs4.1/fs.py
index adaa2d5..2d92d77 100644
--- a/nfs4.1/fs.py
+++ b/nfs4.1/fs.py
@@ -1382,7 +1382,8 @@ class FSLayoutFSObj(FSObject):
# as it facilitates commits, returns, recalls etc.
l_offset = 0
l_len = NFS4_UINT64_MAX
- l_mode = LAYOUTIOMODE4_RW
+ # use requested iomode
+ l_mode = arg.loga_iomode
l_type = LAYOUT4_NFSV4_1_FILES
self.current_layout = (l_type, l_offset, l_len, l_mode)
return layout4(l_offset, l_len, l_mode,
--
1.7.7.6
From: Andy Adamson <[email protected]>
NOTE: values are hard coded and apply to all files. Edit nfs4server.py to
change the values.
Signed-off-by: Andy Adamson <[email protected]>
---
nfs4.1/fs.py | 9 ++++++++-
nfs4.1/nfs4lib.py | 2 +-
2 files changed, 9 insertions(+), 2 deletions(-)
diff --git a/nfs4.1/fs.py b/nfs4.1/fs.py
index 2d92d77..c0f18fa 100644
--- a/nfs4.1/fs.py
+++ b/nfs4.1/fs.py
@@ -1,6 +1,6 @@
from nfs4state import FileState
from nfs4_const import *
-from nfs4_type import fsid4, layout4, layout_content4, nfsv4_1_file_layout4
+from nfs4_type import fsid4, layout4, layout_content4, nfsv4_1_file_layout4, mdsthreshold4, threshold_item4
import nfs4lib
from nfs4lib import NFS4Error
import struct
@@ -1429,6 +1429,13 @@ class FileLayoutFS(FileSystem):
self.fattr4_maxread = 8192
self.fattr4_supported_attrs |= 1 << FATTR4_MAXWRITE
self.fattr4_supported_attrs |= 1 << FATTR4_MAXREAD
+ threshold_item4.thi_layout_type = LAYOUT4_NFSV4_1_FILES
+ # set mdsthreashold read_size, read_iosize and write_iosize
+ threshold_item4.thi_hintset = nfs4lib.list2bitmap([0, 2, 3])
+ threshold_item4.thi_hintlist = struct.pack('!QQQ',512,512,512)
+ mdsthreshold4.mth_hints = [threshold_item4]
+ self.fattr4_mdsthreshold = mdsthreshold4
+ self.fattr4_supported_attrs |= 1 << FATTR4_MDSTHRESHOLD
self.sync(self.root, FILE_SYNC4)
def attach_to_server(self, server):
diff --git a/nfs4.1/nfs4lib.py b/nfs4.1/nfs4lib.py
index 82045a7..a380cce 100644
--- a/nfs4.1/nfs4lib.py
+++ b/nfs4.1/nfs4lib.py
@@ -676,7 +676,7 @@ attr_info = { FATTR4_SUPPORTED_ATTRS : A("r", "fs"),
FATTR4_LAYOUT_BLKSIZE : A("r", "fs"),
FATTR4_LAYOUT_ALIGNMENT : A("r", "obj"),
FATTR4_FS_LOCATIONS_INFO : A("r", "fs"),
- FATTR4_MDSTHRESHOLD : A("r", "obj"),
+ FATTR4_MDSTHRESHOLD : A("r", "fs"),
FATTR4_RETENTION_GET : A("r", "obj"),
FATTR4_RETENTION_SET : A("w", "obj"),
FATTR4_RETENTEVT_GET : A("r", "obj"),
--
1.7.7.6