The following patchset applies on top of my current linux-next branch,
and contains updates to allow the client to piggyback layoutstats in
the flexfiles layoutreturn (as per spec).
This is mainly useful when running workloads that do I/O on lots of
small or medium sized files, which are rarely kept open for long
enough to send a standalone layoutstat.
Trond Myklebust (10):
pNFS/flexfiles: Don't attempt to send layoutstats if there are no
entries
pNFS/flexfiles: Only send layoutstats updates for mirrors that were
updated
NFSv4: Add a generic structure for managing layout-private information
pNFS: Allow layout drivers to manage private data in struct
nfs4_layoutreturn
pNFS: Add a layoutreturn callback to performa layout-private setup
pNFS/flexfiles: Refactor encoding of the layoutreturn payload
pNFS/flexfiles: Clean up layoutstats
NFS: Fix up read of mirror stats
pNFS/flexfiles: Minor refactoring before adding iostats to
layoutreturn
pNFS/flexfiles: Support sending layoutstats in layoutreturn
fs/nfs/flexfilelayout/flexfilelayout.c | 236 ++++++++++++++++++++++--------
fs/nfs/flexfilelayout/flexfilelayout.h | 20 ++-
fs/nfs/flexfilelayout/flexfilelayoutdev.c | 78 ++++++++--
fs/nfs/nfs42proc.c | 9 +-
fs/nfs/nfs42xdr.c | 5 +-
fs/nfs/nfs4proc.c | 6 +
fs/nfs/nfs4xdr.c | 4 +-
fs/nfs/pnfs.c | 15 +-
fs/nfs/pnfs.h | 2 +-
include/linux/nfs_xdr.h | 19 ++-
10 files changed, 306 insertions(+), 88 deletions(-)
--
2.9.3
If the list of mirrors is empty, then don't send an RPC call.
Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/flexfilelayout/flexfilelayout.c | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c
index 90462a2a9237..a6264d6836dc 100644
--- a/fs/nfs/flexfilelayout/flexfilelayout.c
+++ b/fs/nfs/flexfilelayout/flexfilelayout.c
@@ -2250,6 +2250,11 @@ ff_layout_prepare_layoutstats(struct nfs42_layoutstat_args *args)
args->num_dev = ff_layout_mirror_prepare_stats(args,
&ff_layout->generic_hdr, dev_count);
spin_unlock(&args->inode->i_lock);
+ if (!args->num_dev) {
+ kfree(args->devinfo);
+ args->devinfo = NULL;
+ return -ENOENT;
+ }
return 0;
}
--
2.9.3
Signed-off-by: Trond Myklebust <[email protected]>
---
include/linux/nfs_xdr.h | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h
index bfbd0cace91b..331a3200eb01 100644
--- a/include/linux/nfs_xdr.h
+++ b/include/linux/nfs_xdr.h
@@ -216,6 +216,20 @@ struct nfs4_get_lease_time_res {
struct nfs_fsinfo *lr_fsinfo;
};
+struct xdr_stream;
+struct nfs4_xdr_opaque_data;
+
+struct nfs4_xdr_opaque_ops {
+ void (*encode)(struct xdr_stream *, const void *args,
+ const struct nfs4_xdr_opaque_data *);
+ void (*free)(struct nfs4_xdr_opaque_data *);
+};
+
+struct nfs4_xdr_opaque_data {
+ const struct nfs4_xdr_opaque_ops *ops;
+ void *data;
+};
+
#define PNFS_LAYOUT_MAXSIZE 4096
struct nfs4_layoutdriver_data {
--
2.9.3
If there have been no reads or writes to a given mirror since the last
layoutstats update, then don't resend the same data.
Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/flexfilelayout/flexfilelayout.c | 6 ++++++
fs/nfs/flexfilelayout/flexfilelayout.h | 3 +++
2 files changed, 9 insertions(+)
diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c
index a6264d6836dc..540813c30b00 100644
--- a/fs/nfs/flexfilelayout/flexfilelayout.c
+++ b/fs/nfs/flexfilelayout/flexfilelayout.c
@@ -705,6 +705,7 @@ nfs4_ff_layout_stat_io_start_read(struct inode *inode,
spin_lock(&mirror->lock);
report = nfs4_ff_layoutstat_start_io(mirror, &mirror->read_stat, now);
nfs4_ff_layout_stat_io_update_requested(&mirror->read_stat, requested);
+ set_bit(NFS4_FF_MIRROR_STAT_AVAIL, &mirror->flags);
spin_unlock(&mirror->lock);
if (report)
@@ -721,6 +722,7 @@ nfs4_ff_layout_stat_io_end_read(struct rpc_task *task,
nfs4_ff_layout_stat_io_update_completed(&mirror->read_stat,
requested, completed,
ktime_get(), task->tk_start);
+ set_bit(NFS4_FF_MIRROR_STAT_AVAIL, &mirror->flags);
spin_unlock(&mirror->lock);
}
@@ -734,6 +736,7 @@ nfs4_ff_layout_stat_io_start_write(struct inode *inode,
spin_lock(&mirror->lock);
report = nfs4_ff_layoutstat_start_io(mirror , &mirror->write_stat, now);
nfs4_ff_layout_stat_io_update_requested(&mirror->write_stat, requested);
+ set_bit(NFS4_FF_MIRROR_STAT_AVAIL, &mirror->flags);
spin_unlock(&mirror->lock);
if (report)
@@ -753,6 +756,7 @@ nfs4_ff_layout_stat_io_end_write(struct rpc_task *task,
spin_lock(&mirror->lock);
nfs4_ff_layout_stat_io_update_completed(&mirror->write_stat,
requested, completed, ktime_get(), task->tk_start);
+ set_bit(NFS4_FF_MIRROR_STAT_AVAIL, &mirror->flags);
spin_unlock(&mirror->lock);
}
@@ -2201,6 +2205,8 @@ ff_layout_mirror_prepare_stats(struct nfs42_layoutstat_args *args,
break;
if (!mirror->mirror_ds)
continue;
+ if (!test_and_clear_bit(NFS4_FF_MIRROR_STAT_AVAIL, &mirror->flags))
+ continue;
/* mirror refcount put in cleanup_layoutstats */
if (!atomic_inc_not_zero(&mirror->ref))
continue;
diff --git a/fs/nfs/flexfilelayout/flexfilelayout.h b/fs/nfs/flexfilelayout/flexfilelayout.h
index 3ee0c9fcea76..09f292e3a4ad 100644
--- a/fs/nfs/flexfilelayout/flexfilelayout.h
+++ b/fs/nfs/flexfilelayout/flexfilelayout.h
@@ -81,12 +81,15 @@ struct nfs4_ff_layout_mirror {
struct rpc_cred __rcu *rw_cred;
atomic_t ref;
spinlock_t lock;
+ unsigned long flags;
struct nfs4_ff_layoutstat read_stat;
struct nfs4_ff_layoutstat write_stat;
ktime_t start_time;
u32 report_interval;
};
+#define NFS4_FF_MIRROR_STAT_AVAIL (0)
+
struct nfs4_ff_layout_segment {
struct pnfs_layout_segment generic_hdr;
u64 stripe_unit;
--
2.9.3
Add a callback to allow the flexfiles layout driver to initialise the
layout private payload.
Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/pnfs.c | 14 +++++++++++++-
fs/nfs/pnfs.h | 1 +
2 files changed, 14 insertions(+), 1 deletion(-)
diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 4631d15227ae..2a3431314c23 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -1072,6 +1072,7 @@ pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, const nfs4_stateid *stateid,
enum pnfs_iomode iomode, bool sync)
{
struct inode *ino = lo->plh_inode;
+ struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld;
struct nfs4_layoutreturn *lrp;
int status = 0;
@@ -1089,6 +1090,8 @@ pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, const nfs4_stateid *stateid,
lrp->args.ld_private = &lrp->ld_private;
lrp->clp = NFS_SERVER(ino)->nfs_client;
lrp->cred = lo->plh_lc_cred;
+ if (ld->prepare_layoutreturn)
+ ld->prepare_layoutreturn(&lrp->args);
status = nfs4_proc_layoutreturn(lrp, sync);
out:
@@ -1310,9 +1313,15 @@ bool pnfs_roc(struct inode *ino,
out_noroc:
spin_unlock(&ino->i_lock);
pnfs_layoutcommit_inode(ino, true);
+ if (roc) {
+ struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld;
+ if (ld->prepare_layoutreturn)
+ ld->prepare_layoutreturn(args);
+ return true;
+ }
if (layoutreturn)
pnfs_send_layoutreturn(lo, &stateid, iomode, true);
- return roc;
+ return false;
}
void pnfs_roc_release(struct nfs4_layoutreturn_args *args,
@@ -1322,6 +1331,7 @@ void pnfs_roc_release(struct nfs4_layoutreturn_args *args,
struct pnfs_layout_hdr *lo = args->layout;
const nfs4_stateid *arg_stateid = NULL;
const nfs4_stateid *res_stateid = NULL;
+ struct nfs4_xdr_opaque_data *ld_private = args->ld_private;
if (ret == 0) {
arg_stateid = &args->stateid;
@@ -1330,6 +1340,8 @@ void pnfs_roc_release(struct nfs4_layoutreturn_args *args,
}
pnfs_layoutreturn_free_lsegs(lo, arg_stateid, &args->range,
res_stateid);
+ if (ld_private && ld_private->ops && ld_private->ops->free)
+ ld_private->ops->free(ld_private);
pnfs_put_layout_hdr(lo);
trace_nfs4_layoutreturn_on_close(args->inode, 0);
}
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index 459ac2c74af8..d1e028175cd1 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -172,6 +172,7 @@ struct pnfs_layoutdriver_type {
(struct nfs_server *server, struct pnfs_device *pdev,
gfp_t gfp_flags);
+ int (*prepare_layoutreturn) (struct nfs4_layoutreturn_args *);
void (*encode_layoutreturn) (struct xdr_stream *xdr,
const struct nfs4_layoutreturn_args *args);
--
2.9.3
Need to lock while reading in order to ensure 64-bit reads are correct.
Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/flexfilelayout/flexfilelayout.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c
index f61df1641567..c18afd5cc0bb 100644
--- a/fs/nfs/flexfilelayout/flexfilelayout.c
+++ b/fs/nfs/flexfilelayout/flexfilelayout.c
@@ -2250,10 +2250,12 @@ ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo,
memcpy(&devinfo->dev_id, &dev->deviceid, NFS4_DEVICEID4_SIZE);
devinfo->offset = 0;
devinfo->length = NFS4_MAX_UINT64;
+ spin_lock(&mirror->lock);
devinfo->read_count = mirror->read_stat.io_stat.ops_completed;
devinfo->read_bytes = mirror->read_stat.io_stat.bytes_completed;
devinfo->write_count = mirror->write_stat.io_stat.ops_completed;
devinfo->write_bytes = mirror->write_stat.io_stat.bytes_completed;
+ spin_unlock(&mirror->lock);
devinfo->layout_type = LAYOUT_FLEX_FILES;
devinfo->layoutstats_encode = ff_layout_encode_layoutstats;
devinfo->layout_private = mirror;
--
2.9.3
Add the layout error payload to the flexfiles layoutreturn private
data, and set up the encoding mechanisms. This is a refactoring in
preparation for adding the layout iostats payload.
Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/flexfilelayout/flexfilelayout.c | 63 +++++++++++++++++++------
fs/nfs/flexfilelayout/flexfilelayout.h | 14 ++++--
fs/nfs/flexfilelayout/flexfilelayoutdev.c | 78 +++++++++++++++++++++++++------
3 files changed, 124 insertions(+), 31 deletions(-)
diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c
index 540813c30b00..7f31c359a28a 100644
--- a/fs/nfs/flexfilelayout/flexfilelayout.c
+++ b/fs/nfs/flexfilelayout/flexfilelayout.c
@@ -25,6 +25,8 @@
#define NFSDBG_FACILITY NFSDBG_PNFS_LD
#define FF_LAYOUT_POLL_RETRY_MAX (15*HZ)
+#define FF_LAYOUTRETURN_MAXERR 20
+
static struct group_info *ff_zero_group;
@@ -1971,24 +1973,18 @@ ff_layout_free_deviceid_node(struct nfs4_deviceid_node *d)
static int ff_layout_encode_ioerr(struct nfs4_flexfile_layout *flo,
struct xdr_stream *xdr,
- const struct nfs4_layoutreturn_args *args)
+ const struct nfs4_layoutreturn_args *args,
+ const struct nfs4_flexfile_layoutreturn_args *ff_args)
{
- struct pnfs_layout_hdr *hdr = &flo->generic_hdr;
__be32 *start;
- int count = 0, ret = 0;
start = xdr_reserve_space(xdr, 4);
if (unlikely(!start))
return -E2BIG;
+ *start = cpu_to_be32(ff_args->num_errors);
/* This assume we always return _ALL_ layouts */
- spin_lock(&hdr->plh_inode->i_lock);
- ret = ff_layout_encode_ds_ioerr(flo, xdr, &count, &args->range);
- spin_unlock(&hdr->plh_inode->i_lock);
-
- *start = cpu_to_be32(count);
-
- return ret;
+ return ff_layout_encode_ds_ioerr(xdr, &ff_args->errors);
}
/* report nothing for now */
@@ -2017,8 +2013,10 @@ ff_layout_alloc_deviceid_node(struct nfs_server *server,
static void
ff_layout_encode_layoutreturn(struct xdr_stream *xdr,
- const struct nfs4_layoutreturn_args *args)
+ const void *voidargs,
+ const struct nfs4_xdr_opaque_data *ff_opaque)
{
+ const struct nfs4_layoutreturn_args *args = voidargs;
struct pnfs_layout_hdr *lo = args->layout;
struct nfs4_flexfile_layout *flo = FF_LAYOUT_FROM_HDR(lo);
__be32 *start;
@@ -2027,13 +2025,52 @@ ff_layout_encode_layoutreturn(struct xdr_stream *xdr,
start = xdr_reserve_space(xdr, 4);
BUG_ON(!start);
- ff_layout_encode_ioerr(flo, xdr, args);
+ ff_layout_encode_ioerr(flo, xdr, args, ff_opaque->data);
ff_layout_encode_iostats(flo, xdr, args);
*start = cpu_to_be32((xdr->p - start - 1) * 4);
dprintk("%s: Return\n", __func__);
}
+static void
+ff_layout_free_layoutreturn(struct nfs4_xdr_opaque_data *args)
+{
+ struct nfs4_flexfile_layoutreturn_args *ff_args;
+
+ if (!args->data)
+ return;
+ ff_args = args->data;
+ args->data = NULL;
+
+ ff_layout_free_ds_ioerr(&ff_args->errors);
+
+ kfree(ff_args);
+}
+
+const struct nfs4_xdr_opaque_ops layoutreturn_ops = {
+ .encode = ff_layout_encode_layoutreturn,
+ .free = ff_layout_free_layoutreturn,
+};
+
+static int
+ff_layout_prepare_layoutreturn(struct nfs4_layoutreturn_args *args)
+{
+ struct nfs4_flexfile_layoutreturn_args *ff_args;
+
+ ff_args = kmalloc(sizeof(*ff_args), GFP_KERNEL);
+ if (!ff_args)
+ return -ENOMEM;
+
+ INIT_LIST_HEAD(&ff_args->errors);
+ ff_args->num_errors = ff_layout_fetch_ds_ioerr(args->layout,
+ &args->range, &ff_args->errors,
+ FF_LAYOUTRETURN_MAXERR);
+
+ args->ld_private->ops = &layoutreturn_ops;
+ args->ld_private->data = ff_args;
+ return 0;
+}
+
static int
ff_layout_ntop4(const struct sockaddr *sap, char *buf, const size_t buflen)
{
@@ -2299,7 +2336,7 @@ static struct pnfs_layoutdriver_type flexfilelayout_type = {
.read_pagelist = ff_layout_read_pagelist,
.write_pagelist = ff_layout_write_pagelist,
.alloc_deviceid_node = ff_layout_alloc_deviceid_node,
- .encode_layoutreturn = ff_layout_encode_layoutreturn,
+ .prepare_layoutreturn = ff_layout_prepare_layoutreturn,
.sync = pnfs_nfs_generic_sync,
.prepare_layoutstats = ff_layout_prepare_layoutstats,
.cleanup_layoutstats = ff_layout_cleanup_layoutstats,
diff --git a/fs/nfs/flexfilelayout/flexfilelayout.h b/fs/nfs/flexfilelayout/flexfilelayout.h
index 09f292e3a4ad..560c837995fc 100644
--- a/fs/nfs/flexfilelayout/flexfilelayout.h
+++ b/fs/nfs/flexfilelayout/flexfilelayout.h
@@ -106,6 +106,11 @@ struct nfs4_flexfile_layout {
ktime_t last_report_time; /* Layoutstat report times */
};
+struct nfs4_flexfile_layoutreturn_args {
+ struct list_head errors;
+ unsigned int num_errors;
+};
+
static inline struct nfs4_flexfile_layout *
FF_LAYOUT_FROM_HDR(struct pnfs_layout_hdr *lo)
{
@@ -183,9 +188,12 @@ int ff_layout_track_ds_error(struct nfs4_flexfile_layout *flo,
struct nfs4_ff_layout_mirror *mirror, u64 offset,
u64 length, int status, enum nfs_opnum4 opnum,
gfp_t gfp_flags);
-int ff_layout_encode_ds_ioerr(struct nfs4_flexfile_layout *flo,
- struct xdr_stream *xdr, int *count,
- const struct pnfs_layout_range *range);
+int ff_layout_encode_ds_ioerr(struct xdr_stream *xdr, const struct list_head *head);
+void ff_layout_free_ds_ioerr(struct list_head *head);
+unsigned int ff_layout_fetch_ds_ioerr(struct pnfs_layout_hdr *lo,
+ const struct pnfs_layout_range *range,
+ struct list_head *head,
+ unsigned int maxnum);
struct nfs_fh *
nfs4_ff_layout_select_ds_fh(struct pnfs_layout_segment *lseg, u32 mirror_idx);
diff --git a/fs/nfs/flexfilelayout/flexfilelayoutdev.c b/fs/nfs/flexfilelayout/flexfilelayoutdev.c
index 75767aa38455..eb98395c3651 100644
--- a/fs/nfs/flexfilelayout/flexfilelayoutdev.c
+++ b/fs/nfs/flexfilelayout/flexfilelayoutdev.c
@@ -447,20 +447,26 @@ nfs4_ff_find_or_create_ds_client(struct pnfs_layout_segment *lseg, u32 ds_idx,
}
}
+void ff_layout_free_ds_ioerr(struct list_head *head)
+{
+ struct nfs4_ff_layout_ds_err *err;
+
+ while (!list_empty(head)) {
+ err = list_first_entry(head,
+ struct nfs4_ff_layout_ds_err,
+ list);
+ list_del(&err->list);
+ kfree(err);
+ }
+}
+
/* called with inode i_lock held */
-int ff_layout_encode_ds_ioerr(struct nfs4_flexfile_layout *flo,
- struct xdr_stream *xdr, int *count,
- const struct pnfs_layout_range *range)
+int ff_layout_encode_ds_ioerr(struct xdr_stream *xdr, const struct list_head *head)
{
- struct nfs4_ff_layout_ds_err *err, *n;
+ struct nfs4_ff_layout_ds_err *err;
__be32 *p;
- list_for_each_entry_safe(err, n, &flo->error_list, list) {
- if (!pnfs_is_range_intersecting(err->offset,
- pnfs_end_offset(err->offset, err->length),
- range->offset,
- pnfs_end_offset(range->offset, range->length)))
- continue;
+ list_for_each_entry(err, head, list) {
/* offset(8) + length(8) + stateid(NFS4_STATEID_SIZE)
* + array length + deviceid(NFS4_DEVICEID4_SIZE)
* + status(4) + opnum(4)
@@ -479,17 +485,59 @@ int ff_layout_encode_ds_ioerr(struct nfs4_flexfile_layout *flo,
NFS4_DEVICEID4_SIZE);
*p++ = cpu_to_be32(err->status);
*p++ = cpu_to_be32(err->opnum);
- *count += 1;
- list_del(&err->list);
- dprintk("%s: offset %llu length %llu status %d op %d count %d\n",
+ dprintk("%s: offset %llu length %llu status %d op %d\n",
__func__, err->offset, err->length, err->status,
- err->opnum, *count);
- kfree(err);
+ err->opnum);
}
return 0;
}
+static
+unsigned int do_layout_fetch_ds_ioerr(struct pnfs_layout_hdr *lo,
+ const struct pnfs_layout_range *range,
+ struct list_head *head,
+ unsigned int maxnum)
+{
+ struct nfs4_flexfile_layout *flo = FF_LAYOUT_FROM_HDR(lo);
+ struct inode *inode = lo->plh_inode;
+ struct nfs4_ff_layout_ds_err *err, *n;
+ unsigned int ret = 0;
+
+ spin_lock(&inode->i_lock);
+ list_for_each_entry_safe(err, n, &flo->error_list, list) {
+ if (!pnfs_is_range_intersecting(err->offset,
+ pnfs_end_offset(err->offset, err->length),
+ range->offset,
+ pnfs_end_offset(range->offset, range->length)))
+ continue;
+ if (!maxnum)
+ break;
+ list_move(&err->list, head);
+ maxnum--;
+ ret++;
+ }
+ spin_unlock(&inode->i_lock);
+ return ret;
+}
+
+unsigned int ff_layout_fetch_ds_ioerr(struct pnfs_layout_hdr *lo,
+ const struct pnfs_layout_range *range,
+ struct list_head *head,
+ unsigned int maxnum)
+{
+ unsigned int ret;
+
+ ret = do_layout_fetch_ds_ioerr(lo, range, head, maxnum);
+ /* If we're over the max, discard all remaining entries */
+ if (ret == maxnum) {
+ LIST_HEAD(discard);
+ do_layout_fetch_ds_ioerr(lo, range, &discard, -1);
+ ff_layout_free_ds_ioerr(&discard);
+ }
+ return ret;
+}
+
static bool ff_read_layout_has_available_ds(struct pnfs_layout_segment *lseg)
{
struct nfs4_ff_layout_mirror *mirror;
--
2.9.3
Cleanup to allow layout drivers to attach private data to layoutreturn,
and manage the data.
Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/nfs4proc.c | 6 ++++++
fs/nfs/nfs4xdr.c | 4 +++-
fs/nfs/pnfs.c | 1 +
include/linux/nfs_xdr.h | 2 ++
4 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index 1c629f5e1ae5..f992281c9b29 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -3039,6 +3039,7 @@ struct nfs4_closedata {
struct {
struct nfs4_layoutreturn_args arg;
struct nfs4_layoutreturn_res res;
+ struct nfs4_xdr_opaque_data ld_private;
u32 roc_barrier;
bool roc;
} lr;
@@ -3267,6 +3268,7 @@ int nfs4_do_close(struct nfs4_state *state, gfp_t gfp_mask, int wait)
if (IS_ERR(calldata->arg.seqid))
goto out_free_calldata;
calldata->arg.fmode = 0;
+ calldata->lr.arg.ld_private = &calldata->lr.ld_private;
calldata->res.fattr = &calldata->fattr;
calldata->res.seqid = calldata->arg.seqid;
calldata->res.server = server;
@@ -5599,6 +5601,7 @@ struct nfs4_delegreturndata {
struct {
struct nfs4_layoutreturn_args arg;
struct nfs4_layoutreturn_res res;
+ struct nfs4_xdr_opaque_data ld_private;
u32 roc_barrier;
bool roc;
} lr;
@@ -5735,6 +5738,7 @@ static int _nfs4_proc_delegreturn(struct inode *inode, struct rpc_cred *cred, co
data->res.fattr = &data->fattr;
data->res.server = server;
data->res.lr_ret = -NFS4ERR_NOMATCHING_LAYOUT;
+ data->lr.arg.ld_private = &data->lr.ld_private;
nfs_fattr_init(data->res.fattr);
data->timestamp = jiffies;
data->rpc_status = 0;
@@ -8633,6 +8637,8 @@ static void nfs4_layoutreturn_release(void *calldata)
nfs4_sequence_free_slot(&lrp->res.seq_res);
pnfs_put_layout_hdr(lrp->args.layout);
nfs_iput_and_deactive(lrp->inode);
+ if (lrp->ld_private.ops && lrp->ld_private.ops->free)
+ lrp->ld_private.ops->free(&lrp->ld_private);
kfree(calldata);
dprintk("<-- %s\n", __func__);
}
diff --git a/fs/nfs/nfs4xdr.c b/fs/nfs/nfs4xdr.c
index 84fdcf27138c..1c1768a8fcd1 100644
--- a/fs/nfs/nfs4xdr.c
+++ b/fs/nfs/nfs4xdr.c
@@ -2035,7 +2035,9 @@ encode_layoutreturn(struct xdr_stream *xdr,
spin_lock(&args->inode->i_lock);
encode_nfs4_stateid(xdr, &args->stateid);
spin_unlock(&args->inode->i_lock);
- if (lr_ops->encode_layoutreturn)
+ if (args->ld_private->ops && args->ld_private->ops->encode)
+ args->ld_private->ops->encode(xdr, args, args->ld_private);
+ else if (lr_ops->encode_layoutreturn)
lr_ops->encode_layoutreturn(xdr, args);
else
encode_uint32(xdr, 0);
diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 0b25a1c820ba..4631d15227ae 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -1086,6 +1086,7 @@ pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, const nfs4_stateid *stateid,
}
pnfs_init_layoutreturn_args(&lrp->args, lo, stateid, iomode);
+ lrp->args.ld_private = &lrp->ld_private;
lrp->clp = NFS_SERVER(ino)->nfs_client;
lrp->cred = lo->plh_lc_cred;
diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h
index 331a3200eb01..b64177d669fd 100644
--- a/include/linux/nfs_xdr.h
+++ b/include/linux/nfs_xdr.h
@@ -320,6 +320,7 @@ struct nfs4_layoutreturn_args {
struct pnfs_layout_range range;
nfs4_stateid stateid;
__u32 layout_type;
+ struct nfs4_xdr_opaque_data *ld_private;
};
struct nfs4_layoutreturn_res {
@@ -335,6 +336,7 @@ struct nfs4_layoutreturn {
struct nfs_client *clp;
struct inode *inode;
int rpc_status;
+ struct nfs4_xdr_opaque_data ld_private;
};
#define PNFS_LAYOUTSTATS_MAXSIZE 256
--
2.9.3
Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/flexfilelayout/flexfilelayout.c | 59 ++++++++++++++++++++--------------
fs/nfs/nfs42proc.c | 9 ++++--
fs/nfs/nfs42xdr.c | 5 +--
fs/nfs/pnfs.h | 1 -
include/linux/nfs_xdr.h | 3 +-
5 files changed, 44 insertions(+), 33 deletions(-)
diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c
index c18afd5cc0bb..e5078301720a 100644
--- a/fs/nfs/flexfilelayout/flexfilelayout.c
+++ b/fs/nfs/flexfilelayout/flexfilelayout.c
@@ -1988,7 +1988,7 @@ static int ff_layout_encode_ioerr(struct nfs4_flexfile_layout *flo,
}
/* report nothing for now */
-static void ff_layout_encode_iostats(struct nfs4_flexfile_layout *flo,
+static void ff_layout_encode_iostats_array(struct nfs4_flexfile_layout *flo,
struct xdr_stream *xdr,
const struct nfs4_layoutreturn_args *args)
{
@@ -2026,7 +2026,7 @@ ff_layout_encode_layoutreturn(struct xdr_stream *xdr,
BUG_ON(!start);
ff_layout_encode_ioerr(flo, xdr, args, ff_opaque->data);
- ff_layout_encode_iostats(flo, xdr, args);
+ ff_layout_encode_iostats_array(flo, xdr, args);
*start = cpu_to_be32((xdr->p - start - 1) * 4);
dprintk("%s: Return\n", __func__);
@@ -2191,21 +2191,18 @@ ff_layout_encode_io_latency(struct xdr_stream *xdr,
}
static void
-ff_layout_encode_layoutstats(struct xdr_stream *xdr,
- struct nfs42_layoutstat_args *args,
- struct nfs42_layoutstat_devinfo *devinfo)
+ff_layout_encode_ff_layoutupdate(struct xdr_stream *xdr,
+ const struct nfs42_layoutstat_devinfo *devinfo,
+ struct nfs4_ff_layout_mirror *mirror)
{
- struct nfs4_ff_layout_mirror *mirror = devinfo->layout_private;
struct nfs4_pnfs_ds_addr *da;
struct nfs4_pnfs_ds *ds = mirror->mirror_ds->ds;
struct nfs_fh *fh = &mirror->fh_versions[0];
- __be32 *p, *start;
+ __be32 *p;
da = list_first_entry(&ds->ds_addrs, struct nfs4_pnfs_ds_addr, da_node);
dprintk("%s: DS %s: encoding address %s\n",
__func__, ds->ds_remotestr, da->da_remotestr);
- /* layoutupdate length */
- start = xdr_reserve_space(xdr, 4);
/* netaddr4 */
ff_layout_encode_netaddr(xdr, da);
/* nfs_fh4 */
@@ -2222,10 +2219,36 @@ ff_layout_encode_layoutstats(struct xdr_stream *xdr,
/* bool */
p = xdr_reserve_space(xdr, 4);
*p = cpu_to_be32(false);
+}
+
+static void
+ff_layout_encode_layoutstats(struct xdr_stream *xdr, const void *args,
+ const struct nfs4_xdr_opaque_data *opaque)
+{
+ struct nfs42_layoutstat_devinfo *devinfo = container_of(opaque,
+ struct nfs42_layoutstat_devinfo, ld_private);
+ __be32 *start;
+
+ /* layoutupdate length */
+ start = xdr_reserve_space(xdr, 4);
+ ff_layout_encode_ff_layoutupdate(xdr, devinfo, opaque->data);
*start = cpu_to_be32((xdr->p - start - 1) * 4);
}
+static void
+ff_layout_free_layoutstats(struct nfs4_xdr_opaque_data *opaque)
+{
+ struct nfs4_ff_layout_mirror *mirror = opaque->data;
+
+ ff_layout_put_mirror(mirror);
+}
+
+static const struct nfs4_xdr_opaque_ops layoutstat_ops = {
+ .encode = ff_layout_encode_layoutstats,
+ .free = ff_layout_free_layoutstats,
+};
+
static int
ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo,
struct nfs42_layoutstat_devinfo *devinfo,
@@ -2257,8 +2280,8 @@ ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo,
devinfo->write_bytes = mirror->write_stat.io_stat.bytes_completed;
spin_unlock(&mirror->lock);
devinfo->layout_type = LAYOUT_FLEX_FILES;
- devinfo->layoutstats_encode = ff_layout_encode_layoutstats;
- devinfo->layout_private = mirror;
+ devinfo->ld_private.ops = &layoutstat_ops;
+ devinfo->ld_private.data = mirror;
devinfo++;
i++;
@@ -2291,19 +2314,6 @@ ff_layout_prepare_layoutstats(struct nfs42_layoutstat_args *args)
return 0;
}
-static void
-ff_layout_cleanup_layoutstats(struct nfs42_layoutstat_data *data)
-{
- struct nfs4_ff_layout_mirror *mirror;
- int i;
-
- for (i = 0; i < data->args.num_dev; i++) {
- mirror = data->args.devinfo[i].layout_private;
- data->args.devinfo[i].layout_private = NULL;
- ff_layout_put_mirror(mirror);
- }
-}
-
static struct pnfs_layoutdriver_type flexfilelayout_type = {
.id = LAYOUT_FLEX_FILES,
.name = "LAYOUT_FLEX_FILES",
@@ -2328,7 +2338,6 @@ static struct pnfs_layoutdriver_type flexfilelayout_type = {
.prepare_layoutreturn = ff_layout_prepare_layoutreturn,
.sync = pnfs_nfs_generic_sync,
.prepare_layoutstats = ff_layout_prepare_layoutstats,
- .cleanup_layoutstats = ff_layout_cleanup_layoutstats,
};
static int __init nfs4flexfilelayout_init(void)
diff --git a/fs/nfs/nfs42proc.c b/fs/nfs/nfs42proc.c
index 608501971fe0..d12ff9385f49 100644
--- a/fs/nfs/nfs42proc.c
+++ b/fs/nfs/nfs42proc.c
@@ -397,10 +397,13 @@ static void
nfs42_layoutstat_release(void *calldata)
{
struct nfs42_layoutstat_data *data = calldata;
- struct nfs_server *nfss = NFS_SERVER(data->args.inode);
+ struct nfs42_layoutstat_devinfo *devinfo = data->args.devinfo;
+ int i;
- if (nfss->pnfs_curr_ld->cleanup_layoutstats)
- nfss->pnfs_curr_ld->cleanup_layoutstats(data);
+ for (i = 0; i < data->args.num_dev; i++) {
+ if (devinfo[i].ld_private.ops && devinfo[i].ld_private.ops->free)
+ devinfo[i].ld_private.ops->free(&devinfo[i].ld_private);
+ }
pnfs_put_layout_hdr(NFS_I(data->args.inode)->layout);
smp_mb__before_atomic();
diff --git a/fs/nfs/nfs42xdr.c b/fs/nfs/nfs42xdr.c
index 8b2605882a20..6c7296454bbc 100644
--- a/fs/nfs/nfs42xdr.c
+++ b/fs/nfs/nfs42xdr.c
@@ -181,8 +181,9 @@ static void encode_layoutstats(struct xdr_stream *xdr,
NFS4_DEVICEID4_SIZE);
/* Encode layoutupdate4 */
*p++ = cpu_to_be32(devinfo->layout_type);
- if (devinfo->layoutstats_encode != NULL)
- devinfo->layoutstats_encode(xdr, args, devinfo);
+ if (devinfo->ld_private.ops)
+ devinfo->ld_private.ops->encode(xdr, args,
+ &devinfo->ld_private);
else
encode_uint32(xdr, 0);
}
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index d1e028175cd1..63f77b49a586 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -182,7 +182,6 @@ struct pnfs_layoutdriver_type {
struct xdr_stream *xdr,
const struct nfs4_layoutcommit_args *args);
int (*prepare_layoutstats) (struct nfs42_layoutstat_args *args);
- void (*cleanup_layoutstats) (struct nfs42_layoutstat_data *data);
};
struct pnfs_layout_hdr {
diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h
index b64177d669fd..617cfaa20ffc 100644
--- a/include/linux/nfs_xdr.h
+++ b/include/linux/nfs_xdr.h
@@ -357,8 +357,7 @@ struct nfs42_layoutstat_devinfo {
__u64 write_count;
__u64 write_bytes;
__u32 layout_type;
- layoutstats_encode_t layoutstats_encode;
- void *layout_private;
+ struct nfs4_xdr_opaque_data ld_private;
};
struct nfs42_layoutstat_args {
--
2.9.3
Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/flexfilelayout/flexfilelayout.c | 27 +++++++--------------------
1 file changed, 7 insertions(+), 20 deletions(-)
diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c
index 7f31c359a28a..f61df1641567 100644
--- a/fs/nfs/flexfilelayout/flexfilelayout.c
+++ b/fs/nfs/flexfilelayout/flexfilelayout.c
@@ -2227,14 +2227,13 @@ ff_layout_encode_layoutstats(struct xdr_stream *xdr,
}
static int
-ff_layout_mirror_prepare_stats(struct nfs42_layoutstat_args *args,
- struct pnfs_layout_hdr *lo,
+ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo,
+ struct nfs42_layoutstat_devinfo *devinfo,
int dev_limit)
{
struct nfs4_flexfile_layout *ff_layout = FF_LAYOUT_FROM_HDR(lo);
struct nfs4_ff_layout_mirror *mirror;
struct nfs4_deviceid_node *dev;
- struct nfs42_layoutstat_devinfo *devinfo;
int i = 0;
list_for_each_entry(mirror, &ff_layout->mirrors, mirrors) {
@@ -2248,7 +2247,6 @@ ff_layout_mirror_prepare_stats(struct nfs42_layoutstat_args *args,
if (!atomic_inc_not_zero(&mirror->ref))
continue;
dev = &mirror->mirror_ds->id_node;
- devinfo = &args->devinfo[i];
memcpy(&devinfo->dev_id, &dev->deviceid, NFS4_DEVICEID4_SIZE);
devinfo->offset = 0;
devinfo->length = NFS4_MAX_UINT64;
@@ -2260,6 +2258,7 @@ ff_layout_mirror_prepare_stats(struct nfs42_layoutstat_args *args,
devinfo->layoutstats_encode = ff_layout_encode_layoutstats;
devinfo->layout_private = mirror;
+ devinfo++;
i++;
}
return i;
@@ -2269,29 +2268,17 @@ static int
ff_layout_prepare_layoutstats(struct nfs42_layoutstat_args *args)
{
struct nfs4_flexfile_layout *ff_layout;
- struct nfs4_ff_layout_mirror *mirror;
- int dev_count = 0;
+ const int dev_count = PNFS_LAYOUTSTATS_MAXDEV;
- spin_lock(&args->inode->i_lock);
- ff_layout = FF_LAYOUT_FROM_HDR(NFS_I(args->inode)->layout);
- list_for_each_entry(mirror, &ff_layout->mirrors, mirrors) {
- if (atomic_read(&mirror->ref) != 0)
- dev_count ++;
- }
- spin_unlock(&args->inode->i_lock);
/* For now, send at most PNFS_LAYOUTSTATS_MAXDEV statistics */
- if (dev_count > PNFS_LAYOUTSTATS_MAXDEV) {
- dprintk("%s: truncating devinfo to limit (%d:%d)\n",
- __func__, dev_count, PNFS_LAYOUTSTATS_MAXDEV);
- dev_count = PNFS_LAYOUTSTATS_MAXDEV;
- }
args->devinfo = kmalloc_array(dev_count, sizeof(*args->devinfo), GFP_NOIO);
if (!args->devinfo)
return -ENOMEM;
spin_lock(&args->inode->i_lock);
- args->num_dev = ff_layout_mirror_prepare_stats(args,
- &ff_layout->generic_hdr, dev_count);
+ ff_layout = FF_LAYOUT_FROM_HDR(NFS_I(args->inode)->layout);
+ args->num_dev = ff_layout_mirror_prepare_stats(&ff_layout->generic_hdr,
+ &args->devinfo[0], dev_count);
spin_unlock(&args->inode->i_lock);
if (!args->num_dev) {
kfree(args->devinfo);
--
2.9.3
Add the ability to send an array of layoutstats entries as part of
layoutreturn.
Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/flexfilelayout/flexfilelayout.c | 82 +++++++++++++++++++++++++++++++---
fs/nfs/flexfilelayout/flexfilelayout.h | 3 ++
2 files changed, 79 insertions(+), 6 deletions(-)
diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c
index e5078301720a..ca1012a42e14 100644
--- a/fs/nfs/flexfilelayout/flexfilelayout.c
+++ b/fs/nfs/flexfilelayout/flexfilelayout.c
@@ -32,6 +32,12 @@ static struct group_info *ff_zero_group;
static void ff_layout_read_record_layoutstats_done(struct rpc_task *task,
struct nfs_pgio_header *hdr);
+static int ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo,
+ struct nfs42_layoutstat_devinfo *devinfo,
+ int dev_limit);
+static void ff_layout_encode_ff_layoutupdate(struct xdr_stream *xdr,
+ const struct nfs42_layoutstat_devinfo *devinfo,
+ struct nfs4_ff_layout_mirror *mirror);
static struct pnfs_layout_hdr *
ff_layout_alloc_layout_hdr(struct inode *inode, gfp_t gfp_flags)
@@ -1987,16 +1993,73 @@ static int ff_layout_encode_ioerr(struct nfs4_flexfile_layout *flo,
return ff_layout_encode_ds_ioerr(xdr, &ff_args->errors);
}
+static void
+encode_opaque_fixed(struct xdr_stream *xdr, const void *buf, size_t len)
+{
+ __be32 *p;
+
+ p = xdr_reserve_space(xdr, len);
+ xdr_encode_opaque_fixed(p, buf, len);
+}
+
+static void
+ff_layout_encode_ff_iostat_head(struct xdr_stream *xdr,
+ const nfs4_stateid *stateid,
+ const struct nfs42_layoutstat_devinfo *devinfo)
+{
+ __be32 *p;
+
+ p = xdr_reserve_space(xdr, 8 + 8);
+ p = xdr_encode_hyper(p, devinfo->offset);
+ p = xdr_encode_hyper(p, devinfo->length);
+ encode_opaque_fixed(xdr, stateid->data, NFS4_STATEID_SIZE);
+ p = xdr_reserve_space(xdr, 4*8);
+ p = xdr_encode_hyper(p, devinfo->read_count);
+ p = xdr_encode_hyper(p, devinfo->read_bytes);
+ p = xdr_encode_hyper(p, devinfo->write_count);
+ p = xdr_encode_hyper(p, devinfo->write_bytes);
+ encode_opaque_fixed(xdr, devinfo->dev_id.data, NFS4_DEVICEID4_SIZE);
+}
+
+static void
+ff_layout_encode_ff_iostat(struct xdr_stream *xdr,
+ const nfs4_stateid *stateid,
+ const struct nfs42_layoutstat_devinfo *devinfo)
+{
+ ff_layout_encode_ff_iostat_head(xdr, stateid, devinfo);
+ ff_layout_encode_ff_layoutupdate(xdr, devinfo,
+ devinfo->ld_private.data);
+}
+
/* report nothing for now */
-static void ff_layout_encode_iostats_array(struct nfs4_flexfile_layout *flo,
- struct xdr_stream *xdr,
- const struct nfs4_layoutreturn_args *args)
+static void ff_layout_encode_iostats_array(struct xdr_stream *xdr,
+ const struct nfs4_layoutreturn_args *args,
+ struct nfs4_flexfile_layoutreturn_args *ff_args)
{
__be32 *p;
+ int i;
p = xdr_reserve_space(xdr, 4);
- if (likely(p))
- *p = cpu_to_be32(0);
+ *p = cpu_to_be32(ff_args->num_dev);
+ for (i = 0; i < ff_args->num_dev; i++)
+ ff_layout_encode_ff_iostat(xdr,
+ &args->layout->plh_stateid,
+ &ff_args->devinfo[i]);
+}
+
+static void
+ff_layout_free_iostats_array(struct nfs42_layoutstat_devinfo *devinfo,
+ unsigned int num_entries)
+{
+ unsigned int i;
+
+ for (i = 0; i < num_entries; i++) {
+ if (!devinfo[i].ld_private.ops)
+ continue;
+ if (!devinfo[i].ld_private.ops->free)
+ continue;
+ devinfo[i].ld_private.ops->free(&devinfo[i].ld_private);
+ }
}
static struct nfs4_deviceid_node *
@@ -2026,7 +2089,7 @@ ff_layout_encode_layoutreturn(struct xdr_stream *xdr,
BUG_ON(!start);
ff_layout_encode_ioerr(flo, xdr, args, ff_opaque->data);
- ff_layout_encode_iostats_array(flo, xdr, args);
+ ff_layout_encode_iostats_array(xdr, args, ff_opaque->data);
*start = cpu_to_be32((xdr->p - start - 1) * 4);
dprintk("%s: Return\n", __func__);
@@ -2043,6 +2106,7 @@ ff_layout_free_layoutreturn(struct nfs4_xdr_opaque_data *args)
args->data = NULL;
ff_layout_free_ds_ioerr(&ff_args->errors);
+ ff_layout_free_iostats_array(ff_args->devinfo, ff_args->num_dev);
kfree(ff_args);
}
@@ -2056,6 +2120,7 @@ static int
ff_layout_prepare_layoutreturn(struct nfs4_layoutreturn_args *args)
{
struct nfs4_flexfile_layoutreturn_args *ff_args;
+ struct nfs4_flexfile_layout *ff_layout = FF_LAYOUT_FROM_HDR(args->layout);
ff_args = kmalloc(sizeof(*ff_args), GFP_KERNEL);
if (!ff_args)
@@ -2066,6 +2131,11 @@ ff_layout_prepare_layoutreturn(struct nfs4_layoutreturn_args *args)
&args->range, &ff_args->errors,
FF_LAYOUTRETURN_MAXERR);
+ spin_lock(&args->inode->i_lock);
+ ff_args->num_dev = ff_layout_mirror_prepare_stats(&ff_layout->generic_hdr,
+ &ff_args->devinfo[0], ARRAY_SIZE(ff_args->devinfo));
+ spin_unlock(&args->inode->i_lock);
+
args->ld_private->ops = &layoutreturn_ops;
args->ld_private->data = ff_args;
return 0;
diff --git a/fs/nfs/flexfilelayout/flexfilelayout.h b/fs/nfs/flexfilelayout/flexfilelayout.h
index 560c837995fc..35221fe390c5 100644
--- a/fs/nfs/flexfilelayout/flexfilelayout.h
+++ b/fs/nfs/flexfilelayout/flexfilelayout.h
@@ -21,6 +21,7 @@
/* LAYOUTSTATS report interval in ms */
#define FF_LAYOUTSTATS_REPORT_INTERVAL (60000L)
+#define FF_LAYOUTSTATS_MAXDEV 4
struct nfs4_ff_ds_version {
u32 version;
@@ -108,7 +109,9 @@ struct nfs4_flexfile_layout {
struct nfs4_flexfile_layoutreturn_args {
struct list_head errors;
+ struct nfs42_layoutstat_devinfo devinfo[FF_LAYOUTSTATS_MAXDEV];
unsigned int num_errors;
+ unsigned int num_dev;
};
static inline struct nfs4_flexfile_layout *
--
2.9.3