.. and an immediate new version because I had the wrong patch in 0/8 last
time. Sorry for the noise.
Here's another round of sysfs entries for each nfs_server, this time with a
single use-case: a "shutdown" toggle that causes the basic rpc_clnt(s) to
immediately fail tasks with -EIO. It works well for the non pNFS cases to
allow an unmount of a filesystem when the NFS server has gone away.
I'm posting to gain potential NACKing, or to be redirected, or to serve as
fodder for discussion at LSF.
I'm thinking I'd like to toggle v4.2 things like READ_PLUS in here next, or
other module-level options that maybe would be useful per-mount.
Benjamin Coddington (9):
NFS: rename nfs_client_kset to nfs_kset
NFS: rename nfs_client_kobj to nfs_net_kobj
NFS: add superblock sysfs entries
NFS: Add sysfs links to sunrpc clients for nfs_clients
NFS: add a sysfs link to the lockd rpc_client
NFS: add a sysfs link to the acl rpc_client
NFS: add sysfs shutdown knob
NFS: Cancel all existing RPC tasks when shutdown
NFSv4: Clean up some shutdown loops
fs/lockd/clntlock.c | 6 ++
fs/nfs/client.c | 21 +++++
fs/nfs/nfs3client.c | 4 +
fs/nfs/nfs4client.c | 2 +
fs/nfs/nfs4proc.c | 2 +-
fs/nfs/nfs4state.c | 3 +
fs/nfs/super.c | 6 +-
fs/nfs/sysfs.c | 159 +++++++++++++++++++++++++++++++++---
fs/nfs/sysfs.h | 9 +-
include/linux/lockd/bind.h | 2 +
include/linux/nfs_fs_sb.h | 3 +
include/linux/sunrpc/clnt.h | 11 ++-
net/sunrpc/clnt.c | 5 ++
net/sunrpc/sysfs.h | 7 --
14 files changed, 216 insertions(+), 24 deletions(-)
--
2.39.2
Within each nfs_server sysfs tree, add an entry named "shutdown". Writing
1 to this file will set the cl_shutdown bit on the rpc_clnt structs
associated with that mount. If cl_shutdown is set, the task scheduler
immediately returns -EIO for new tasks.
Signed-off-by: Benjamin Coddington <[email protected]>
---
fs/nfs/sysfs.c | 46 +++++++++++++++++++++++++++++++++++++
include/linux/nfs_fs_sb.h | 1 +
include/linux/sunrpc/clnt.h | 3 ++-
net/sunrpc/clnt.c | 5 ++++
4 files changed, 54 insertions(+), 1 deletion(-)
diff --git a/fs/nfs/sysfs.c b/fs/nfs/sysfs.c
index e1b5df7a3df5..09fde2719495 100644
--- a/fs/nfs/sysfs.c
+++ b/fs/nfs/sysfs.c
@@ -12,6 +12,7 @@
#include <linux/string.h>
#include <linux/nfs_fs.h>
#include <linux/rcupdate.h>
+#include <linux/lockd/lockd.h>
#include "nfs4_fs.h"
#include "netns.h"
@@ -191,6 +192,50 @@ void nfs_netns_sysfs_destroy(struct nfs_net *netns)
}
}
+static ssize_t
+shutdown_show(struct kobject *kobj, struct kobj_attribute *attr,
+ char *buf)
+{
+ struct nfs_server *server = container_of(kobj, struct nfs_server, kobj);
+ bool shutdown = server->flags & NFS_MOUNT_SHUTDOWN;
+ return sysfs_emit(buf, "%d\n", shutdown);
+}
+
+static ssize_t
+shutdown_store(struct kobject *kobj, struct kobj_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct nfs_server *server;
+ int ret, val;
+
+ server = container_of(kobj, struct nfs_server, kobj);
+
+ ret = kstrtoint(buf, 0, &val);
+ if (ret)
+ return ret;
+
+ if (val != 1)
+ return -EINVAL;
+
+ /* already shut down? */
+ if (server->flags & NFS_MOUNT_SHUTDOWN)
+ goto out;
+
+ server->flags |= NFS_MOUNT_SHUTDOWN;
+ server->client->cl_shutdown = 1;
+ server->nfs_client->cl_rpcclient->cl_shutdown = 1;
+
+ if (!IS_ERR(server->client_acl))
+ server->client_acl->cl_shutdown = 1;
+
+ if (server->nlm_host)
+ server->nlm_host->h_rpcclnt->cl_shutdown = 1;
+out:
+ return count;
+}
+
+static struct kobj_attribute nfs_sysfs_attr_shutdown = __ATTR_RW(shutdown);
+
#define RPC_CLIENT_NAME_SIZE 64
void nfs_sysfs_link_rpc_client(struct nfs_server *server,
@@ -217,6 +262,7 @@ static void nfs_sysfs_sb_release(struct kobject *kobj)
}
static struct attribute *nfs_mp_attrs[] = {
+ &nfs_sysfs_attr_shutdown.attr,
NULL,
};
diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h
index dee1664abca8..bc89496384ff 100644
--- a/include/linux/nfs_fs_sb.h
+++ b/include/linux/nfs_fs_sb.h
@@ -153,6 +153,7 @@ struct nfs_server {
#define NFS_MOUNT_WRITE_EAGER 0x01000000
#define NFS_MOUNT_WRITE_WAIT 0x02000000
#define NFS_MOUNT_TRUNK_DISCOVERY 0x04000000
+#define NFS_MOUNT_SHUTDOWN 0x08000000
unsigned int fattr_valid; /* Valid attributes */
unsigned int caps; /* server capabilities */
diff --git a/include/linux/sunrpc/clnt.h b/include/linux/sunrpc/clnt.h
index 4ec718aa91f5..9f72c75a2056 100644
--- a/include/linux/sunrpc/clnt.h
+++ b/include/linux/sunrpc/clnt.h
@@ -63,7 +63,8 @@ struct rpc_clnt {
cl_discrtry : 1,/* disconnect before retry */
cl_noretranstimeo: 1,/* No retransmit timeouts */
cl_autobind : 1,/* use getport() */
- cl_chatty : 1;/* be verbose */
+ cl_chatty : 1,/* be verbose */
+ cl_shutdown : 1;/* rpc immediate -EIO */
struct rpc_rtt * cl_rtt; /* RTO estimator data */
const struct rpc_timeout *cl_timeout; /* Timeout strategy */
diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
index 0b0b9f1eed46..3c5dca88a918 100644
--- a/net/sunrpc/clnt.c
+++ b/net/sunrpc/clnt.c
@@ -1717,6 +1717,11 @@ call_start(struct rpc_task *task)
trace_rpc_request(task);
+ if (task->tk_client->cl_shutdown) {
+ rpc_call_rpcerror(task, -EIO);
+ return;
+ }
+
/* Increment call count (version might not be valid for ping) */
if (clnt->cl_program->version[clnt->cl_vers])
clnt->cl_program->version[clnt->cl_vers]->counts[idx]++;
--
2.39.2
For the general and state management nfs_client under each mount, create
symlinks to their respective rpc_client sysfs entries.
Signed-off-by: Benjamin Coddington <[email protected]>
---
fs/nfs/client.c | 5 +++++
fs/nfs/nfs4client.c | 2 ++
fs/nfs/sysfs.c | 20 ++++++++++++++++++++
fs/nfs/sysfs.h | 2 ++
include/linux/sunrpc/clnt.h | 8 +++++++-
net/sunrpc/sysfs.h | 7 -------
6 files changed, 36 insertions(+), 8 deletions(-)
diff --git a/fs/nfs/client.c b/fs/nfs/client.c
index 72da715fc617..de275f1fde92 100644
--- a/fs/nfs/client.c
+++ b/fs/nfs/client.c
@@ -621,6 +621,7 @@ int nfs_init_server_rpcclient(struct nfs_server *server,
if (server->flags & NFS_MOUNT_SOFT)
server->client->cl_softrtry = 1;
+ nfs_sysfs_link_rpc_client(server, server->client, NULL);
return 0;
}
EXPORT_SYMBOL_GPL(nfs_init_server_rpcclient);
@@ -690,6 +691,7 @@ static int nfs_init_server(struct nfs_server *server,
return PTR_ERR(clp);
server->nfs_client = clp;
+ nfs_sysfs_link_rpc_client(server, clp->cl_rpcclient, "_state");
/* Initialise the client representation from the mount data */
server->flags = ctx->flags;
@@ -1116,6 +1118,9 @@ struct nfs_server *nfs_clone_server(struct nfs_server *source,
server->fsid = fattr->fsid;
+ nfs_sysfs_link_rpc_client(server,
+ server->nfs_client->cl_rpcclient, "_state");
+
error = nfs_init_server_rpcclient(server,
source->client->cl_timeout,
flavor);
diff --git a/fs/nfs/nfs4client.c b/fs/nfs/nfs4client.c
index d3051b051a56..c7012f22a009 100644
--- a/fs/nfs/nfs4client.c
+++ b/fs/nfs/nfs4client.c
@@ -18,6 +18,7 @@
#include "nfs4idmap.h"
#include "pnfs.h"
#include "netns.h"
+#include "sysfs.h"
#define NFSDBG_FACILITY NFSDBG_CLIENT
@@ -947,6 +948,7 @@ static int nfs4_set_client(struct nfs_server *server,
set_bit(NFS_CS_CHECK_LEASE_TIME, &clp->cl_res_state);
server->nfs_client = clp;
+ nfs_sysfs_link_rpc_client(server, clp->cl_rpcclient, "_state");
return 0;
}
diff --git a/fs/nfs/sysfs.c b/fs/nfs/sysfs.c
index 534e2293a698..e1b5df7a3df5 100644
--- a/fs/nfs/sysfs.c
+++ b/fs/nfs/sysfs.c
@@ -191,6 +191,26 @@ void nfs_netns_sysfs_destroy(struct nfs_net *netns)
}
}
+#define RPC_CLIENT_NAME_SIZE 64
+
+void nfs_sysfs_link_rpc_client(struct nfs_server *server,
+ struct rpc_clnt *clnt, const char *uniq)
+{
+ char name[RPC_CLIENT_NAME_SIZE];
+ int ret;
+
+ strcpy(name, clnt->cl_program->name);
+ strcat(name, uniq ? uniq : "");
+ strcat(name, "_client");
+
+ ret = sysfs_create_link_nowarn(&server->kobj,
+ &clnt->cl_sysfs->kobject, name);
+ if (ret < 0)
+ pr_warn("NFS: can't create link to %s in sysfs (%d)\n",
+ name, ret);
+}
+EXPORT_SYMBOL_GPL(nfs_sysfs_link_rpc_client);
+
static void nfs_sysfs_sb_release(struct kobject *kobj)
{
/* no-op: why? see lib/kobject.c kobject_cleanup() */
diff --git a/fs/nfs/sysfs.h b/fs/nfs/sysfs.h
index 0fc80fb55b3e..51e0973f22d1 100644
--- a/fs/nfs/sysfs.h
+++ b/fs/nfs/sysfs.h
@@ -22,6 +22,8 @@ extern void nfs_sysfs_exit(void);
void nfs_netns_sysfs_setup(struct nfs_net *netns, struct net *net);
void nfs_netns_sysfs_destroy(struct nfs_net *netns);
+void nfs_sysfs_link_rpc_client(struct nfs_server *server,
+ struct rpc_clnt *clnt, const char *sysfs_prefix);
void nfs_sysfs_add_server(struct nfs_server *s);
void nfs_sysfs_move_server_to_sb(struct super_block *s);
void nfs_sysfs_move_sb_to_server(struct nfs_server *s);
diff --git a/include/linux/sunrpc/clnt.h b/include/linux/sunrpc/clnt.h
index 770ef2cb5775..4ec718aa91f5 100644
--- a/include/linux/sunrpc/clnt.h
+++ b/include/linux/sunrpc/clnt.h
@@ -30,7 +30,13 @@
#include <linux/sunrpc/xprtmultipath.h>
struct rpc_inode;
-struct rpc_sysfs_client;
+struct rpc_sysfs_client {
+ struct kobject kobject;
+ struct net *net;
+ struct rpc_clnt *clnt;
+ struct rpc_xprt_switch *xprt_switch;
+};
+
/*
* The high-level client handle
diff --git a/net/sunrpc/sysfs.h b/net/sunrpc/sysfs.h
index 6620cebd1037..d2dd77a0a0e9 100644
--- a/net/sunrpc/sysfs.h
+++ b/net/sunrpc/sysfs.h
@@ -5,13 +5,6 @@
#ifndef __SUNRPC_SYSFS_H
#define __SUNRPC_SYSFS_H
-struct rpc_sysfs_client {
- struct kobject kobject;
- struct net *net;
- struct rpc_clnt *clnt;
- struct rpc_xprt_switch *xprt_switch;
-};
-
struct rpc_sysfs_xprt_switch {
struct kobject kobject;
struct net *net;
--
2.39.2
After lockd is started, add a symlink for lockd's rpc_client under
NFS' superblock sysfs.
Signed-off-by: Benjamin Coddington <[email protected]>
---
fs/lockd/clntlock.c | 6 ++++++
fs/nfs/client.c | 1 +
include/linux/lockd/bind.h | 2 ++
3 files changed, 9 insertions(+)
diff --git a/fs/lockd/clntlock.c b/fs/lockd/clntlock.c
index a5bb3f721a9d..0340e10b5715 100644
--- a/fs/lockd/clntlock.c
+++ b/fs/lockd/clntlock.c
@@ -94,6 +94,12 @@ void nlmclnt_done(struct nlm_host *host)
}
EXPORT_SYMBOL_GPL(nlmclnt_done);
+struct rpc_clnt *nlmclnt_rpc_clnt(struct nlm_host *host)
+{
+ return host->h_rpcclnt;
+}
+EXPORT_SYMBOL_GPL(nlmclnt_rpc_clnt);
+
/*
* Queue up a lock for blocking so that the GRANTED request can see it
*/
diff --git a/fs/nfs/client.c b/fs/nfs/client.c
index de275f1fde92..eede8c28a56b 100644
--- a/fs/nfs/client.c
+++ b/fs/nfs/client.c
@@ -592,6 +592,7 @@ static int nfs_start_lockd(struct nfs_server *server)
server->nlm_host = host;
server->destroy = nfs_destroy_server;
+ nfs_sysfs_link_rpc_client(server, nlmclnt_rpc_clnt(host), NULL);
return 0;
}
diff --git a/include/linux/lockd/bind.h b/include/linux/lockd/bind.h
index 3bc9f7410e21..c53c81242e72 100644
--- a/include/linux/lockd/bind.h
+++ b/include/linux/lockd/bind.h
@@ -20,6 +20,7 @@
/* Dummy declarations */
struct svc_rqst;
struct rpc_task;
+struct rpc_clnt;
/*
* This is the set of functions for lockd->nfsd communication
@@ -56,6 +57,7 @@ struct nlmclnt_initdata {
extern struct nlm_host *nlmclnt_init(const struct nlmclnt_initdata *nlm_init);
extern void nlmclnt_done(struct nlm_host *host);
+extern struct rpc_clnt *nlmclnt_rpc_clnt(struct nlm_host *host);
/*
* NLM client operations provide a means to modify RPC processing of NLM
--
2.39.2
Walk existing RPC tasks and cancel them with -EIO when the client is
shutdown.
Signed-off-by: Benjamin Coddington <[email protected]>
---
fs/nfs/sysfs.c | 19 +++++++++++++++----
1 file changed, 15 insertions(+), 4 deletions(-)
diff --git a/fs/nfs/sysfs.c b/fs/nfs/sysfs.c
index 09fde2719495..70cd211d870c 100644
--- a/fs/nfs/sysfs.c
+++ b/fs/nfs/sysfs.c
@@ -192,6 +192,17 @@ void nfs_netns_sysfs_destroy(struct nfs_net *netns)
}
}
+static bool shutdown_match_client(const struct rpc_task *task, const void *data)
+{
+ return true;
+}
+
+static void shutdown_client(struct rpc_clnt *clnt)
+{
+ clnt->cl_shutdown = 1;
+ rpc_cancel_tasks(clnt, -EIO, shutdown_match_client, NULL);
+}
+
static ssize_t
shutdown_show(struct kobject *kobj, struct kobj_attribute *attr,
char *buf)
@@ -222,14 +233,14 @@ shutdown_store(struct kobject *kobj, struct kobj_attribute *attr,
goto out;
server->flags |= NFS_MOUNT_SHUTDOWN;
- server->client->cl_shutdown = 1;
- server->nfs_client->cl_rpcclient->cl_shutdown = 1;
+ shutdown_client(server->client);
+ shutdown_client(server->nfs_client->cl_rpcclient);
if (!IS_ERR(server->client_acl))
- server->client_acl->cl_shutdown = 1;
+ shutdown_client(server->client_acl);
if (server->nlm_host)
- server->nlm_host->h_rpcclnt->cl_shutdown = 1;
+ shutdown_client(server->nlm_host->h_rpcclnt);
out:
return count;
}
--
2.39.2
Be brief and match the subsystem name. There's no need to distinguish this
kset variable from the server.
Signed-off-by: Benjamin Coddington <[email protected]>
---
fs/nfs/sysfs.c | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/fs/nfs/sysfs.c b/fs/nfs/sysfs.c
index 0cbcd2dfa732..81d98727b79f 100644
--- a/fs/nfs/sysfs.c
+++ b/fs/nfs/sysfs.c
@@ -18,7 +18,7 @@
#include "sysfs.h"
struct kobject *nfs_client_kobj;
-static struct kset *nfs_client_kset;
+static struct kset *nfs_kset;
static void nfs_netns_object_release(struct kobject *kobj)
{
@@ -55,13 +55,13 @@ static struct kobject *nfs_netns_object_alloc(const char *name,
int nfs_sysfs_init(void)
{
- nfs_client_kset = kset_create_and_add("nfs", NULL, fs_kobj);
- if (!nfs_client_kset)
+ nfs_kset = kset_create_and_add("nfs", NULL, fs_kobj);
+ if (!nfs_kset)
return -ENOMEM;
- nfs_client_kobj = nfs_netns_object_alloc("net", nfs_client_kset, NULL);
+ nfs_client_kobj = nfs_netns_object_alloc("net", nfs_kset, NULL);
if (!nfs_client_kobj) {
- kset_unregister(nfs_client_kset);
- nfs_client_kset = NULL;
+ kset_unregister(nfs_kset);
+ nfs_kset = NULL;
return -ENOMEM;
}
return 0;
@@ -70,7 +70,7 @@ int nfs_sysfs_init(void)
void nfs_sysfs_exit(void)
{
kobject_put(nfs_client_kobj);
- kset_unregister(nfs_client_kset);
+ kset_unregister(nfs_kset);
}
static ssize_t nfs_netns_identifier_show(struct kobject *kobj,
@@ -159,7 +159,7 @@ static struct nfs_netns_client *nfs_netns_client_alloc(struct kobject *parent,
p = kzalloc(sizeof(*p), GFP_KERNEL);
if (p) {
p->net = net;
- p->kobject.kset = nfs_client_kset;
+ p->kobject.kset = nfs_kset;
if (kobject_init_and_add(&p->kobject, &nfs_netns_client_type,
parent, "nfs_client") == 0)
return p;
--
2.39.2
Match the variable names to the sysfs structure.
Signed-off-by: Benjamin Coddington <[email protected]>
---
fs/nfs/sysfs.c | 10 +++++-----
fs/nfs/sysfs.h | 2 +-
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/fs/nfs/sysfs.c b/fs/nfs/sysfs.c
index 81d98727b79f..a496e26fcfb7 100644
--- a/fs/nfs/sysfs.c
+++ b/fs/nfs/sysfs.c
@@ -17,7 +17,7 @@
#include "netns.h"
#include "sysfs.h"
-struct kobject *nfs_client_kobj;
+struct kobject *nfs_net_kobj;
static struct kset *nfs_kset;
static void nfs_netns_object_release(struct kobject *kobj)
@@ -58,8 +58,8 @@ int nfs_sysfs_init(void)
nfs_kset = kset_create_and_add("nfs", NULL, fs_kobj);
if (!nfs_kset)
return -ENOMEM;
- nfs_client_kobj = nfs_netns_object_alloc("net", nfs_kset, NULL);
- if (!nfs_client_kobj) {
+ nfs_net_kobj = nfs_netns_object_alloc("net", nfs_kset, NULL);
+ if (!nfs_net_kobj) {
kset_unregister(nfs_kset);
nfs_kset = NULL;
return -ENOMEM;
@@ -69,7 +69,7 @@ int nfs_sysfs_init(void)
void nfs_sysfs_exit(void)
{
- kobject_put(nfs_client_kobj);
+ kobject_put(nfs_net_kobj);
kset_unregister(nfs_kset);
}
@@ -172,7 +172,7 @@ void nfs_netns_sysfs_setup(struct nfs_net *netns, struct net *net)
{
struct nfs_netns_client *clp;
- clp = nfs_netns_client_alloc(nfs_client_kobj, net);
+ clp = nfs_netns_client_alloc(nfs_net_kobj, net);
if (clp) {
netns->nfs_client = clp;
kobject_uevent(&clp->kobject, KOBJ_ADD);
diff --git a/fs/nfs/sysfs.h b/fs/nfs/sysfs.h
index 5501ef573c32..0423aaf388c9 100644
--- a/fs/nfs/sysfs.h
+++ b/fs/nfs/sysfs.h
@@ -14,7 +14,7 @@ struct nfs_netns_client {
const char __rcu *identifier;
};
-extern struct kobject *nfs_client_kobj;
+extern struct kobject *nfs_net_kobj;
extern int nfs_sysfs_init(void);
extern void nfs_sysfs_exit(void);
--
2.39.2
If a SEQUENCE call receives -EIO for a shutdown client, it will retry the
RPC call. Instead of doing that for a shutdown client, just bail out.
Likewise, if the state manager decides to perform recovery for a shutdown
client, it will continuously retry. As above, just bail out.
Signed-off-by: Benjamin Coddington <[email protected]>
---
fs/nfs/nfs4proc.c | 2 +-
fs/nfs/nfs4state.c | 3 +++
2 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index 40d749f29ed3..a36e35d885c3 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -9357,7 +9357,7 @@ static void nfs41_sequence_call_done(struct rpc_task *task, void *data)
return;
trace_nfs4_sequence(clp, task->tk_status);
- if (task->tk_status < 0) {
+ if (task->tk_status < 0 && !task->tk_client->cl_shutdown) {
dprintk("%s ERROR %d\n", __func__, task->tk_status);
if (refcount_read(&clp->cl_count) == 1)
return;
diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c
index 2a0ca5c7f082..ead60d7ed4e9 100644
--- a/fs/nfs/nfs4state.c
+++ b/fs/nfs/nfs4state.c
@@ -1210,6 +1210,9 @@ void nfs4_schedule_state_manager(struct nfs_client *clp)
while (cl != cl->cl_parent)
cl = cl->cl_parent;
+ if (clp->cl_rpcclient->cl_shutdown)
+ return;
+
set_bit(NFS4CLNT_RUN_MANAGER, &clp->cl_state);
if (test_and_set_bit(NFS4CLNT_MANAGER_AVAILABLE, &clp->cl_state) != 0) {
wake_up_var(&clp->cl_state);
--
2.39.2