The following patches implement RECLAIM_COMPLETE functionality to
indicate completion of _global_ reclaims for the client. They do
not yet implement RECLAIM_COMPLETE for file system migration.
[PATCH 1/4] Cleanup some NFSv4 XDR decode comments
[PATCH 2/4] nfs41: RECLAIM_COMPLETE XDR functionality
[PATCH 3/4] nfs41: RECLAIM_COMPLETE functionality
[PATCH 4/4] nfs41: Invoke RECLAIM_COMPLETE
A separate patch will be mailed to Bruce to integrate skeleton
functionality on the server side.
The patches apply to Trond's master branch. They were tested on
top of an old version of Andy's session draining patches, and
the previous state reclaim bug fix series.
- ricardo
Signed-off-by: Ricardo Labiaga <[email protected]>
---
fs/nfs/nfs4xdr.c | 26 +++++++++++++-------------
1 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/fs/nfs/nfs4xdr.c b/fs/nfs/nfs4xdr.c
index 20b4e30..1969ba4 100644
--- a/fs/nfs/nfs4xdr.c
+++ b/fs/nfs/nfs4xdr.c
@@ -5309,7 +5309,7 @@ out:
}
/*
- * FSINFO request
+ * Decode FSINFO response
*/
static int nfs4_xdr_dec_fsinfo(struct rpc_rqst *req, __be32 *p,
struct nfs4_fsinfo_res *res)
@@ -5330,7 +5330,7 @@ static int nfs4_xdr_dec_fsinfo(struct rpc_rqst *req, __be32 *p,
}
/*
- * PATHCONF request
+ * Decode PATHCONF response
*/
static int nfs4_xdr_dec_pathconf(struct rpc_rqst *req, __be32 *p,
struct nfs4_pathconf_res *res)
@@ -5351,7 +5351,7 @@ static int nfs4_xdr_dec_pathconf(struct rpc_rqst *req, __be32 *p,
}
/*
- * STATFS request
+ * Decode STATFS response
*/
static int nfs4_xdr_dec_statfs(struct rpc_rqst *req, __be32 *p,
struct nfs4_statfs_res *res)
@@ -5372,7 +5372,7 @@ static int nfs4_xdr_dec_statfs(struct rpc_rqst *req, __be32 *p,
}
/*
- * GETATTR_BITMAP request
+ * Decode GETATTR_BITMAP response
*/
static int nfs4_xdr_dec_server_caps(struct rpc_rqst *req, __be32 *p, struct nfs4_server_caps_res *res)
{
@@ -5411,7 +5411,7 @@ static int nfs4_xdr_dec_renew(struct rpc_rqst *rqstp, __be32 *p, void *dummy)
}
/*
- * a SETCLIENTID request
+ * Decode SETCLIENTID response
*/
static int nfs4_xdr_dec_setclientid(struct rpc_rqst *req, __be32 *p,
struct nfs_client *clp)
@@ -5428,7 +5428,7 @@ static int nfs4_xdr_dec_setclientid(struct rpc_rqst *req, __be32 *p,
}
/*
- * a SETCLIENTID_CONFIRM request
+ * Decode SETCLIENTID_CONFIRM response
*/
static int nfs4_xdr_dec_setclientid_confirm(struct rpc_rqst *req, __be32 *p, struct nfs_fsinfo *fsinfo)
{
@@ -5448,7 +5448,7 @@ static int nfs4_xdr_dec_setclientid_confirm(struct rpc_rqst *req, __be32 *p, str
}
/*
- * DELEGRETURN request
+ * Decode DELEGRETURN response
*/
static int nfs4_xdr_dec_delegreturn(struct rpc_rqst *rqstp, __be32 *p, struct nfs4_delegreturnres *res)
{
@@ -5474,7 +5474,7 @@ out:
}
/*
- * FS_LOCATIONS request
+ * Decode FS_LOCATIONS response
*/
static int nfs4_xdr_dec_fs_locations(struct rpc_rqst *req, __be32 *p,
struct nfs4_fs_locations_res *res)
@@ -5504,7 +5504,7 @@ out:
#if defined(CONFIG_NFS_V4_1)
/*
- * EXCHANGE_ID request
+ * Decode EXCHANGE_ID response
*/
static int nfs4_xdr_dec_exchange_id(struct rpc_rqst *rqstp, uint32_t *p,
void *res)
@@ -5521,7 +5521,7 @@ static int nfs4_xdr_dec_exchange_id(struct rpc_rqst *rqstp, uint32_t *p,
}
/*
- * a CREATE_SESSION request
+ * Decode CREATE_SESSION response
*/
static int nfs4_xdr_dec_create_session(struct rpc_rqst *rqstp, uint32_t *p,
struct nfs41_create_session_res *res)
@@ -5538,7 +5538,7 @@ static int nfs4_xdr_dec_create_session(struct rpc_rqst *rqstp, uint32_t *p,
}
/*
- * a DESTROY_SESSION request
+ * Decode DESTROY_SESSION response
*/
static int nfs4_xdr_dec_destroy_session(struct rpc_rqst *rqstp, uint32_t *p,
void *dummy)
@@ -5555,7 +5555,7 @@ static int nfs4_xdr_dec_destroy_session(struct rpc_rqst *rqstp, uint32_t *p,
}
/*
- * a SEQUENCE request
+ * Decode SEQUENCE response
*/
static int nfs4_xdr_dec_sequence(struct rpc_rqst *rqstp, uint32_t *p,
struct nfs4_sequence_res *res)
@@ -5572,7 +5572,7 @@ static int nfs4_xdr_dec_sequence(struct rpc_rqst *rqstp, uint32_t *p,
}
/*
- * a GET_LEASE_TIME request
+ * Decode GET_LEASE_TIME response
*/
static int nfs4_xdr_dec_get_lease_time(struct rpc_rqst *rqstp, uint32_t *p,
struct nfs4_get_lease_time_res *res)
--
1.5.4.3
XDR encoding and decoding for RECLAIM_COMPLETE. Implements the necessary
encoding to indicate reclaim complete for the entire client. In the future,
it can be extended to provide reclaim complete functionality for a single
file system after migration.
Signed-off-by: Ricardo Labiaga <[email protected]>
---
fs/nfs/nfs4xdr.c | 66 +++++++++++++++++++++++++++++++++++++++++++++++
include/linux/nfs4.h | 1 +
include/linux/nfs_xdr.h | 10 +++++++
3 files changed, 77 insertions(+), 0 deletions(-)
diff --git a/fs/nfs/nfs4xdr.c b/fs/nfs/nfs4xdr.c
index 1969ba4..ee0a15b 100644
--- a/fs/nfs/nfs4xdr.c
+++ b/fs/nfs/nfs4xdr.c
@@ -299,6 +299,8 @@ static int nfs4_stat_to_errno(int);
XDR_QUADLEN(NFS4_MAX_SESSIONID_LEN) + 4)
#define decode_sequence_maxsz (op_decode_hdr_maxsz + \
XDR_QUADLEN(NFS4_MAX_SESSIONID_LEN) + 5)
+#define encode_reclaim_complete_maxsz (op_encode_hdr_maxsz + 4)
+#define decode_reclaim_complete_maxsz (op_decode_hdr_maxsz + 4)
#else /* CONFIG_NFS_V4_1 */
#define encode_sequence_maxsz 0
#define decode_sequence_maxsz 0
@@ -676,6 +678,12 @@ static int nfs4_stat_to_errno(int);
decode_sequence_maxsz + \
decode_putrootfh_maxsz + \
decode_fsinfo_maxsz)
+#define NFS4_enc_reclaim_complete_sz (compound_encode_hdr_maxsz + \
+ encode_sequence_maxsz + \
+ encode_reclaim_complete_maxsz)
+#define NFS4_dec_reclaim_complete_sz (compound_decode_hdr_maxsz + \
+ decode_sequence_maxsz + \
+ decode_reclaim_complete_maxsz)
#endif /* CONFIG_NFS_V4_1 */
static const umode_t nfs_type2fmt[] = {
@@ -1592,6 +1600,19 @@ static void encode_destroy_session(struct xdr_stream *xdr,
hdr->nops++;
hdr->replen += decode_destroy_session_maxsz;
}
+
+static void encode_reclaim_complete(struct xdr_stream *xdr,
+ struct nfs41_reclaim_complete_args *args,
+ struct compound_hdr *hdr)
+{
+ __be32 *p;
+
+ p = reserve_space(xdr, 8);
+ *p++ = cpu_to_be32(OP_RECLAIM_COMPLETE);
+ *p++ = cpu_to_be32(args->one_fs);
+ hdr->nops++;
+ hdr->replen += decode_reclaim_complete_maxsz;
+}
#endif /* CONFIG_NFS_V4_1 */
static void encode_sequence(struct xdr_stream *xdr,
@@ -2420,6 +2441,26 @@ static int nfs4_xdr_enc_get_lease_time(struct rpc_rqst *req, uint32_t *p,
encode_nops(&hdr);
return 0;
}
+
+/*
+ * a RECLAIM_COMPLETE request
+ */
+static int nfs4_xdr_enc_reclaim_complete(struct rpc_rqst *req, uint32_t *p,
+ struct nfs41_reclaim_complete_args *args)
+{
+ struct xdr_stream xdr;
+ struct compound_hdr hdr = {
+ .minorversion = nfs4_xdr_minorversion(&args->seq_args)
+ };
+
+ xdr_init_encode(&xdr, &req->rq_snd_buf, p);
+ encode_compound_hdr(&xdr, req, &hdr);
+ encode_sequence(&xdr, &args->seq_args, &hdr);
+ encode_reclaim_complete(&xdr, args, &hdr);
+ encode_nops(&hdr);
+ return 0;
+}
+
#endif /* CONFIG_NFS_V4_1 */
static void print_overflow_msg(const char *func, const struct xdr_stream *xdr)
@@ -4528,6 +4569,11 @@ static int decode_destroy_session(struct xdr_stream *xdr, void *dummy)
{
return decode_op_hdr(xdr, OP_DESTROY_SESSION);
}
+
+static int decode_reclaim_complete(struct xdr_stream *xdr, void *dummy)
+{
+ return decode_op_hdr(xdr, OP_RECLAIM_COMPLETE);
+}
#endif /* CONFIG_NFS_V4_1 */
static int decode_sequence(struct xdr_stream *xdr,
@@ -5591,6 +5637,25 @@ static int nfs4_xdr_dec_get_lease_time(struct rpc_rqst *rqstp, uint32_t *p,
status = decode_fsinfo(&xdr, res->lr_fsinfo);
return status;
}
+
+/*
+ * Decode RECLAIM_COMPLETE response
+ */
+static int nfs4_xdr_dec_reclaim_complete(struct rpc_rqst *rqstp, uint32_t *p,
+ struct nfs41_reclaim_complete_res *res)
+{
+ struct xdr_stream xdr;
+ struct compound_hdr hdr;
+ int status;
+
+ xdr_init_decode(&xdr, &rqstp->rq_rcv_buf, p);
+ status = decode_compound_hdr(&xdr, &hdr);
+ if (!status)
+ status = decode_sequence(&xdr, &res->seq_res, rqstp);
+ if (!status)
+ status = decode_reclaim_complete(&xdr, (void *)NULL);
+ return status;
+}
#endif /* CONFIG_NFS_V4_1 */
__be32 *nfs4_decode_dirent(__be32 *p, struct nfs_entry *entry, int plus)
@@ -5767,6 +5832,7 @@ struct rpc_procinfo nfs4_procedures[] = {
PROC(DESTROY_SESSION, enc_destroy_session, dec_destroy_session),
PROC(SEQUENCE, enc_sequence, dec_sequence),
PROC(GET_LEASE_TIME, enc_get_lease_time, dec_get_lease_time),
+ PROC(RECLAIM_COMPLETE, enc_reclaim_complete, dec_reclaim_complete),
#endif /* CONFIG_NFS_V4_1 */
};
diff --git a/include/linux/nfs4.h b/include/linux/nfs4.h
index c4c0602..f7bb8e0 100644
--- a/include/linux/nfs4.h
+++ b/include/linux/nfs4.h
@@ -528,6 +528,7 @@ enum {
NFSPROC4_CLNT_DESTROY_SESSION,
NFSPROC4_CLNT_SEQUENCE,
NFSPROC4_CLNT_GET_LEASE_TIME,
+ NFSPROC4_CLNT_RECLAIM_COMPLETE,
};
/* nfs41 types */
diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h
index 62f63fb..1810d1f 100644
--- a/include/linux/nfs_xdr.h
+++ b/include/linux/nfs_xdr.h
@@ -938,6 +938,16 @@ struct nfs41_create_session_args {
struct nfs41_create_session_res {
struct nfs_client *client;
};
+
+struct nfs41_reclaim_complete_args {
+ /* In the future extend to include curr_fh for use with migration */
+ unsigned char one_fs:1;
+ struct nfs4_sequence_args seq_args;
+};
+
+struct nfs41_reclaim_complete_res {
+ struct nfs4_sequence_res seq_res;
+};
#endif /* CONFIG_NFS_V4_1 */
struct nfs_page;
--
1.5.4.3
Implements RECLAIM_COMPLETE as an asynchronous RPC.
NFS4ERR_DELAY is retried, NFS4ERR_DEADSESSION invokes the error handling
but does not result in a retry, since we don't want to have a lingering
RECLAIM_COMPLETE call sent in the middle of a possible new state recovery
cycle. If a session reset occurs, a new wave of reclaim operations will
follow, containing their own RECLAIM_COMPLETE call. We don't want a
retry to get on the way of recovery by incorrectly indicating to the
server that we're done reclaiming state.
A subsequent patch invokes the functionality.
Signed-off-by: Ricardo Labiaga <[email protected]>
---
fs/nfs/nfs4_fs.h | 1 +
fs/nfs/nfs4proc.c | 105 ++++++++++++++++++++++++++++++++++++++++++++++++++++
fs/nfs/nfs4state.c | 8 ++++
3 files changed, 114 insertions(+), 0 deletions(-)
diff --git a/fs/nfs/nfs4_fs.h b/fs/nfs/nfs4_fs.h
index e8604af..1aac065 100644
--- a/fs/nfs/nfs4_fs.h
+++ b/fs/nfs/nfs4_fs.h
@@ -181,6 +181,7 @@ struct nfs4_state_recovery_ops {
int (*recover_lock)(struct nfs4_state *, struct file_lock *);
int (*establish_clid)(struct nfs_client *, struct rpc_cred *);
struct rpc_cred * (*get_clid_cred)(struct nfs_client *);
+ int (*reclaim_complete)(struct nfs_client *);
};
struct nfs4_state_maintenance_ops {
diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index 25f4180..990533a 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -4948,6 +4948,110 @@ static int nfs41_proc_async_sequence(struct nfs_client *clp,
&nfs41_sequence_ops, (void *)clp);
}
+struct nfs4_reclaim_complete_data {
+ struct nfs_client *clp;
+ struct nfs41_reclaim_complete_args arg;
+ struct nfs41_reclaim_complete_res res;
+};
+
+static void nfs4_reclaim_complete_prepare(struct rpc_task *task, void *data)
+{
+ struct nfs4_reclaim_complete_data *calldata = data;
+
+ if (nfs4_setup_sequence(calldata->clp, &calldata->arg.seq_args,
+ &calldata->res.seq_res, 0, task))
+ return;
+
+ rpc_call_start(task);
+}
+
+static void nfs4_reclaim_complete_done(struct rpc_task *task, void *data)
+{
+ struct nfs4_reclaim_complete_data *calldata = data;
+ struct nfs_client *clp = calldata->clp;
+ struct nfs4_sequence_res *res = &calldata->res.seq_res;
+
+ dprintk("--> %s\n", __func__);
+ nfs41_sequence_done(clp, res, task->tk_status);
+ switch (task->tk_status) {
+ case 0:
+ case -NFS4ERR_COMPLETE_ALREADY:
+ break;
+ case -NFS4ERR_BADSESSION:
+ case -NFS4ERR_DEADSESSION:
+ /*
+ * Handle the session error, but do not retry the operation, as
+ * we have no way of telling whether the clientid had to be
+ * reset before we got our reply. If reset, a new wave of
+ * reclaim operations will follow, containing their own reclaim
+ * complete. We don't want our retry to get on the way of
+ * recovery by incorrectly indicating to the server that we're
+ * done reclaiming state since the process had to be restarted.
+ */
+ _nfs4_async_handle_error(task, NULL, clp, NULL);
+ break;
+ default:
+ if (_nfs4_async_handle_error(
+ task, NULL, clp, NULL) == -EAGAIN) {
+ rpc_restart_call_prepare(task);
+ return;
+ }
+ }
+ nfs41_sequence_free_slot(clp, res);
+
+ dprintk("<-- %s\n", __func__);
+}
+
+static void nfs4_free_reclaim_complete_data(void *data)
+{
+ struct nfs4_reclaim_complete_data *calldata = data;
+
+ kfree(calldata);
+}
+
+static const struct rpc_call_ops nfs4_reclaim_complete_call_ops = {
+ .rpc_call_prepare = nfs4_reclaim_complete_prepare,
+ .rpc_call_done = nfs4_reclaim_complete_done,
+ .rpc_release = nfs4_free_reclaim_complete_data,
+};
+
+/*
+ * Issue a global reclaim complete.
+ */
+static int nfs41_proc_reclaim_complete(struct nfs_client *clp)
+{
+ struct nfs4_reclaim_complete_data *calldata;
+ struct rpc_task *task;
+ struct rpc_message msg = {
+ .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_RECLAIM_COMPLETE],
+ };
+ struct rpc_task_setup task_setup_data = {
+ .rpc_client = clp->cl_rpcclient,
+ .rpc_message = &msg,
+ .callback_ops = &nfs4_reclaim_complete_call_ops,
+ .flags = RPC_TASK_ASYNC,
+ };
+ int status = -ENOMEM;
+
+ dprintk("--> %s\n", __func__);
+ calldata = kzalloc(sizeof(*calldata), GFP_KERNEL);
+ if (calldata == NULL)
+ goto out;
+ calldata->clp = clp;
+ calldata->arg.one_fs = 0;
+ calldata->res.seq_res.sr_slotid = NFS4_MAX_SLOT_TABLE;
+
+ msg.rpc_argp = &calldata->arg;
+ msg.rpc_resp = &calldata->res;
+ task_setup_data.callback_data = calldata;
+ task = rpc_run_task(&task_setup_data);
+ if (IS_ERR(task))
+ status = PTR_ERR(task);
+ rpc_put_task(task);
+out:
+ dprintk("<-- %s status=%d\n", __func__, status);
+ return status;
+}
#endif /* CONFIG_NFS_V4_1 */
struct nfs4_state_recovery_ops nfs40_reboot_recovery_ops = {
@@ -4967,6 +5071,7 @@ struct nfs4_state_recovery_ops nfs41_reboot_recovery_ops = {
.recover_lock = nfs4_lock_reclaim,
.establish_clid = nfs4_proc_exchange_id,
.get_clid_cred = nfs4_get_exchange_id_cred,
+ .reclaim_complete = nfs41_proc_reclaim_complete,
};
#endif /* CONFIG_NFS_V4_1 */
diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c
index ae90df8..e325b64 100644
--- a/fs/nfs/nfs4state.c
+++ b/fs/nfs/nfs4state.c
@@ -1011,6 +1011,14 @@ static void nfs4_state_start_reclaim_reboot(struct nfs_client *clp)
nfs4_state_mark_reclaim_helper(clp, nfs4_state_mark_reclaim_reboot);
}
+static void nfs4_reclaim_complete(struct nfs_client *clp,
+ const struct nfs4_state_recovery_ops *ops)
+{
+ /* Notify the server we're done reclaiming our state */
+ if (ops->reclaim_complete)
+ (void)ops->reclaim_complete(clp);
+}
+
static void nfs4_state_end_reclaim_reboot(struct nfs_client *clp)
{
struct nfs4_state_owner *sp;
--
1.5.4.3
This patch invokes RECLAIM_COMPLETE after the client is done
reclaiming state.
There are interpretations of the spec that suggest that
RECLAIM_COMPLETE should also be issued after a new clientid
has been obtained from the server and even if there is no
state to reclaim. This tells the server that the client
has no state to reclaim even if the client isn't aware the
server may have rebooted.
Signed-off-by: Ricardo Labiaga <[email protected]>
---
fs/nfs/nfs4state.c | 3 +++
1 files changed, 3 insertions(+), 0 deletions(-)
diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c
index e325b64..24bde5e 100644
--- a/fs/nfs/nfs4state.c
+++ b/fs/nfs/nfs4state.c
@@ -1025,6 +1025,9 @@ static void nfs4_state_end_reclaim_reboot(struct nfs_client *clp)
struct rb_node *pos;
struct nfs4_state *state;
+ nfs4_reclaim_complete(clp,
+ nfs4_reboot_recovery_ops[clp->cl_minorversion]);
+
if (!test_and_clear_bit(NFS4CLNT_RECLAIM_REBOOT, &clp->cl_state))
return;
--
1.5.4.3
On Sat, 2009-12-05 at 12:20 -0800, Ricardo Labiaga wrote:
> The following patches implement RECLAIM_COMPLETE functionality to
> indicate completion of _global_ reclaims for the client. They do
> not yet implement RECLAIM_COMPLETE for file system migration.
>
> [PATCH 1/4] Cleanup some NFSv4 XDR decode comments
> [PATCH 2/4] nfs41: RECLAIM_COMPLETE XDR functionality
> [PATCH 3/4] nfs41: RECLAIM_COMPLETE functionality
> [PATCH 4/4] nfs41: Invoke RECLAIM_COMPLETE
>
> A separate patch will be mailed to Bruce to integrate skeleton
> functionality on the server side.
>
> The patches apply to Trond's master branch. They were tested on
> top of an old version of Andy's session draining patches, and
> the previous state reclaim bug fix series.
Applied. Thanks Ricardo...