Include RPC statistics from all data servers in /proc/self/mountstats for pNFS
filelayout mounts.
The additional info printed on the "per-op statistics" line does not break
current mountstats(8) from nfs-utils.
Signed-off-by: Weston Andros Adamson <[email protected]>
---
This is clearly a better approach than I was taking trying to keep DS rpc stats
separate in /proc/self/mountstats.
FYI, there are no other callers of rpc_print_iostats().
The new output of /proc/self/mountstats works fine with current mountstats
userland tool. The additional data on the per-op statistics line doesn't make
it into the output, but I left it in /proc/self/mountstats because it doesn't
break anything and can be included in newer versions of mountstats.
I plan on adding the per-DS stats (rpc and others) somewhere in /proc/fs/nfsfs/
soon.
fs/nfs/nfs4filelayout.c | 30 ++++++++++++++++++++++++++
fs/nfs/pnfs.h | 23 ++++++++++++++++++++
fs/nfs/pnfs_dev.c | 45 ++++++++++++++++++++++++++++++++++++++++
fs/nfs/super.c | 3 +-
include/linux/sunrpc/metrics.h | 13 +++++++++-
net/sunrpc/stats.c | 38 ++++++++++++++++++++++++++++++++-
6 files changed, 147 insertions(+), 5 deletions(-)
diff --git a/fs/nfs/nfs4filelayout.c b/fs/nfs/nfs4filelayout.c
index 79be7ac..51c71e5 100644
--- a/fs/nfs/nfs4filelayout.c
+++ b/fs/nfs/nfs4filelayout.c
@@ -33,6 +33,8 @@
#include <linux/nfs_page.h>
#include <linux/module.h>
+#include <linux/sunrpc/metrics.h>
+
#include "internal.h"
#include "nfs4filelayout.h"
@@ -918,6 +920,33 @@ filelayout_free_deveiceid_node(struct nfs4_deviceid_node *d)
nfs4_fl_free_deviceid(container_of(d, struct nfs4_file_layout_dsaddr, id_node));
}
+static unsigned int
+filelayout_rpc_add_iostats(struct rpc_iostats *total,
+ unsigned int maxproc,
+ struct nfs4_deviceid_node *d)
+{
+ struct nfs4_file_layout_dsaddr *dsaddr;
+ struct nfs4_pnfs_ds *ds;
+ unsigned int num_ds = 0;
+ u32 i;
+
+ dsaddr = container_of(d, struct nfs4_file_layout_dsaddr, id_node);
+
+ for (i = 0; i < dsaddr->ds_num; i++) {
+ ds = dsaddr->ds_list[i];
+
+ /* only add iostats if nfs_client is different from MDS */
+ if (ds && ds->ds_clp && d->nfs_client != ds->ds_clp) {
+ struct rpc_clnt *clnt = ds->ds_clp->cl_rpcclient;
+ rpc_add_iostats(total, clnt->cl_metrics,
+ min(maxproc, clnt->cl_maxproc));
+ num_ds++;
+ }
+ }
+
+ return num_ds;
+}
+
static struct pnfs_layoutdriver_type filelayout_type = {
.id = LAYOUT_NFSV4_1_FILES,
.name = "LAYOUT_NFSV4_1_FILES",
@@ -932,6 +961,7 @@ static struct pnfs_layoutdriver_type filelayout_type = {
.read_pagelist = filelayout_read_pagelist,
.write_pagelist = filelayout_write_pagelist,
.free_deviceid_node = filelayout_free_deveiceid_node,
+ .ds_rpc_add_iostats = filelayout_rpc_add_iostats,
};
static int __init nfs4filelayout_init(void)
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index 53d593a..d1b82eb 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -72,6 +72,7 @@ enum layoutdriver_policy_flags {
};
struct nfs4_deviceid_node;
+struct rpc_iostats;
/* Per-layout driver specific registration structure */
struct pnfs_layoutdriver_type {
@@ -119,6 +120,9 @@ struct pnfs_layoutdriver_type {
void (*encode_layoutcommit) (struct pnfs_layout_hdr *layoutid,
struct xdr_stream *xdr,
const struct nfs4_layoutcommit_args *args);
+
+ unsigned int (*ds_rpc_add_iostats) (struct rpc_iostats *, unsigned int,
+ struct nfs4_deviceid_node *);
};
struct pnfs_layout_hdr {
@@ -239,6 +243,10 @@ void nfs4_init_deviceid_node(struct nfs4_deviceid_node *,
struct nfs4_deviceid_node *nfs4_insert_deviceid_node(struct nfs4_deviceid_node *);
bool nfs4_put_deviceid_node(struct nfs4_deviceid_node *);
void nfs4_deviceid_purge_client(const struct nfs_client *);
+void nfs4_deviceid_rpc_print_iostats(struct seq_file *seq,
+ const struct nfs_client *clp,
+ const struct pnfs_layoutdriver_type *ld);
+
static inline int lo_fail_bit(u32 iomode)
{
@@ -328,6 +336,15 @@ static inline int pnfs_return_layout(struct inode *ino)
return 0;
}
+static inline int
+pnfs_rpc_print_iostats(struct seq_file *m, struct nfs_server *nfss)
+{
+ if (!pnfs_enabled_sb(nfss))
+ return 0;
+ nfs4_deviceid_rpc_print_iostats(m, nfss->nfs_client,
+ nfss->pnfs_curr_ld);
+ return 1;
+}
#else /* CONFIG_NFS_V4_1 */
static inline void pnfs_destroy_all_layouts(struct nfs_client *clp)
@@ -429,6 +446,12 @@ static inline int pnfs_layoutcommit_inode(struct inode *inode, bool sync)
static inline void nfs4_deviceid_purge_client(struct nfs_client *ncl)
{
}
+
+static inline void
+pnfs_rpc_print_iostats(struct seq_file *m, struct nfs_server *nfss)
+{
+ return 0;
+}
#endif /* CONFIG_NFS_V4_1 */
#endif /* FS_NFS_PNFS_H */
diff --git a/fs/nfs/pnfs_dev.c b/fs/nfs/pnfs_dev.c
index 4f359d2..aa5d102 100644
--- a/fs/nfs/pnfs_dev.c
+++ b/fs/nfs/pnfs_dev.c
@@ -29,6 +29,9 @@
*/
#include <linux/export.h>
+
+#include <linux/sunrpc/metrics.h>
+
#include "pnfs.h"
#define NFSDBG_FACILITY NFSDBG_PNFS
@@ -115,6 +118,48 @@ nfs4_find_get_deviceid(const struct pnfs_layoutdriver_type *ld,
}
EXPORT_SYMBOL_GPL(nfs4_find_get_deviceid);
+
+void
+nfs4_deviceid_rpc_print_iostats(struct seq_file *seq,
+ const struct nfs_client *clp,
+ const struct pnfs_layoutdriver_type *ld)
+{
+ struct nfs4_deviceid_node *d;
+ struct hlist_node *n;
+ struct rpc_iostats *totals;
+ char msgbuf[64];
+ unsigned int num_ds = 0;
+ unsigned int maxproc;
+ long h;
+
+ if (!ld->ds_rpc_add_iostats)
+ return;
+ totals = rpc_alloc_iostats(clp->cl_rpcclient);
+ if (!totals)
+ return;
+ maxproc = clp->cl_rpcclient->cl_maxproc;
+ rpc_add_iostats(totals, clp->cl_rpcclient->cl_metrics, maxproc);
+
+ rcu_read_lock();
+ for (h = 0; h < NFS4_DEVICE_ID_HASH_SIZE; h++) {
+ hlist_for_each_entry_rcu(d, n, &nfs4_deviceid_cache[h], node) {
+ if (d->ld == ld && d->nfs_client == clp) {
+ if (atomic_read(&d->ref))
+ num_ds += ld->ds_rpc_add_iostats(totals,
+ maxproc, d);
+ }
+ }
+ }
+ rcu_read_unlock();
+
+ snprintf(msgbuf, sizeof(msgbuf),
+ " for the metadata server and %u data server%s", num_ds,
+ (num_ds != 1) ? "s" : "");
+ rpc_print_iostats(seq, clp->cl_rpcclient, totals, msgbuf);
+ rpc_free_iostats(totals);
+}
+EXPORT_SYMBOL_GPL(nfs4_deviceid_rpc_print_iostats);
+
/*
* Remove a deviceid from cache
*
diff --git a/fs/nfs/super.c b/fs/nfs/super.c
index d18a90b..76d0850 100644
--- a/fs/nfs/super.c
+++ b/fs/nfs/super.c
@@ -870,7 +870,8 @@ static int nfs_show_stats(struct seq_file *m, struct dentry *root)
#endif
seq_printf(m, "\n");
- rpc_print_iostats(m, nfss->client);
+ if (!pnfs_rpc_print_iostats(m, nfss))
+ rpc_print_iostats(m, nfss->client, NULL, NULL);
return 0;
}
diff --git a/include/linux/sunrpc/metrics.h b/include/linux/sunrpc/metrics.h
index b6edbc0..e6e04e4 100644
--- a/include/linux/sunrpc/metrics.h
+++ b/include/linux/sunrpc/metrics.h
@@ -75,14 +75,23 @@ struct rpc_clnt;
struct rpc_iostats * rpc_alloc_iostats(struct rpc_clnt *);
void rpc_count_iostats(struct rpc_task *);
-void rpc_print_iostats(struct seq_file *, struct rpc_clnt *);
+void rpc_print_iostats(struct seq_file *,
+ struct rpc_clnt *,
+ struct rpc_iostats *,
+ const char *);
+void rpc_add_iostats(struct rpc_iostats *,
+ struct rpc_iostats *, unsigned int);
void rpc_free_iostats(struct rpc_iostats *);
#else /* CONFIG_PROC_FS */
static inline struct rpc_iostats *rpc_alloc_iostats(struct rpc_clnt *clnt) { return NULL; }
static inline void rpc_count_iostats(struct rpc_task *task) {}
-static inline void rpc_print_iostats(struct seq_file *seq, struct rpc_clnt *clnt) {}
+static inline void rpc_print_iostats(struct seq_file *seq,
+ struct rpc_iostats *stat) {}
+static inline void rpc_add_iostats(struct rpc_iostats *total,
+ struct rpc_iostats *new,
+ unsigned int maxproc) {}
static inline void rpc_free_iostats(struct rpc_iostats *stats) {}
#endif /* CONFIG_PROC_FS */
diff --git a/net/sunrpc/stats.c b/net/sunrpc/stats.c
index 3c4f688..342df57 100644
--- a/net/sunrpc/stats.c
+++ b/net/sunrpc/stats.c
@@ -176,12 +176,16 @@ static void _print_name(struct seq_file *seq, unsigned int op,
seq_printf(seq, "\t%12u: ", op);
}
-void rpc_print_iostats(struct seq_file *seq, struct rpc_clnt *clnt)
+void rpc_print_iostats(struct seq_file *seq, struct rpc_clnt *clnt,
+ struct rpc_iostats *altstats,
+ const char *altstats_msg)
{
struct rpc_iostats *stats = clnt->cl_metrics;
struct rpc_xprt *xprt = clnt->cl_xprt;
unsigned int op, maxproc = clnt->cl_maxproc;
+ if (altstats)
+ stats = altstats;
if (!stats)
return;
@@ -192,7 +196,8 @@ void rpc_print_iostats(struct seq_file *seq, struct rpc_clnt *clnt)
if (xprt)
xprt->ops->print_stats(xprt, seq);
- seq_printf(seq, "\tper-op statistics\n");
+ seq_printf(seq, "\tper-op statistics%s\n",
+ (altstats && altstats_msg) ? altstats_msg : "");
for (op = 0; op < maxproc; op++) {
struct rpc_iostats *metrics = &stats[op];
_print_name(seq, op, clnt->cl_procinfo);
@@ -209,6 +214,35 @@ void rpc_print_iostats(struct seq_file *seq, struct rpc_clnt *clnt)
}
EXPORT_SYMBOL_GPL(rpc_print_iostats);
+void
+rpc_add_iostats(struct rpc_iostats *totals, struct rpc_iostats *new,
+ unsigned int maxproc)
+{
+ unsigned int op;
+
+ for (op = 0; op < maxproc; op++) {
+ struct rpc_iostats *op_totals = &totals[op],
+ *op_new = &new[op];
+
+ op_totals->om_ops += op_new->om_ops;
+ op_totals->om_ntrans += op_new->om_ntrans;
+ op_totals->om_timeouts += op_new->om_timeouts;
+
+ op_totals->om_bytes_sent += op_new->om_bytes_sent;
+ op_totals->om_bytes_recv += op_new->om_bytes_recv;
+
+ op_totals->om_queue = ktime_add(op_totals->om_queue,
+ op_new->om_queue);
+
+ op_totals->om_rtt = ktime_add(op_totals->om_rtt,
+ op_new->om_rtt);
+
+ op_totals->om_execute = ktime_add(op_totals->om_execute,
+ op_new->om_execute);
+ }
+}
+EXPORT_SYMBOL_GPL(rpc_add_iostats);
+
/*
* Register/unregister RPC proc files
*/
--
1.7.4.4
On Feb 9, 2012, at 11:41 AM, Weston Andros Adamson wrote:
> Include RPC statistics from all data servers in /proc/self/mountstats for pNFS
> filelayout mounts.
>
> The additional info printed on the "per-op statistics" line does not break
> current mountstats(8) from nfs-utils.
>
> Signed-off-by: Weston Andros Adamson <[email protected]>
> ---
>
> This is clearly a better approach than I was taking trying to keep DS rpc stats
> separate in /proc/self/mountstats.
I guess you could argue that since these are "mount" stats, it's a per mount point thing, not necessarily a per server thing. We've simply had it that way because prior to pNFS, you couldn't have more than one server per mount point.
At some point we might consider also displaying some fs_locations data here, and perhaps a count of how many migration or replication failover events have been seen. For another day.
> FYI, there are no other callers of rpc_print_iostats().
>
> The new output of /proc/self/mountstats works fine with current mountstats
> userland tool. The additional data on the per-op statistics line doesn't make
> it into the output, but I left it in /proc/self/mountstats because it doesn't
> break anything and can be included in newer versions of mountstats.
>
> I plan on adding the per-DS stats (rpc and others) somewhere in /proc/fs/nfsfs/
> soon.
>
> fs/nfs/nfs4filelayout.c | 30 ++++++++++++++++++++++++++
> fs/nfs/pnfs.h | 23 ++++++++++++++++++++
> fs/nfs/pnfs_dev.c | 45 ++++++++++++++++++++++++++++++++++++++++
> fs/nfs/super.c | 3 +-
> include/linux/sunrpc/metrics.h | 13 +++++++++-
> net/sunrpc/stats.c | 38 ++++++++++++++++++++++++++++++++-
> 6 files changed, 147 insertions(+), 5 deletions(-)
>
> diff --git a/fs/nfs/nfs4filelayout.c b/fs/nfs/nfs4filelayout.c
> index 79be7ac..51c71e5 100644
> --- a/fs/nfs/nfs4filelayout.c
> +++ b/fs/nfs/nfs4filelayout.c
> @@ -33,6 +33,8 @@
> #include <linux/nfs_page.h>
> #include <linux/module.h>
>
> +#include <linux/sunrpc/metrics.h>
> +
> #include "internal.h"
> #include "nfs4filelayout.h"
>
> @@ -918,6 +920,33 @@ filelayout_free_deveiceid_node(struct nfs4_deviceid_node *d)
> nfs4_fl_free_deviceid(container_of(d, struct nfs4_file_layout_dsaddr, id_node));
> }
>
> +static unsigned int
> +filelayout_rpc_add_iostats(struct rpc_iostats *total,
> + unsigned int maxproc,
> + struct nfs4_deviceid_node *d)
> +{
> + struct nfs4_file_layout_dsaddr *dsaddr;
> + struct nfs4_pnfs_ds *ds;
> + unsigned int num_ds = 0;
> + u32 i;
> +
> + dsaddr = container_of(d, struct nfs4_file_layout_dsaddr, id_node);
> +
> + for (i = 0; i < dsaddr->ds_num; i++) {
> + ds = dsaddr->ds_list[i];
> +
> + /* only add iostats if nfs_client is different from MDS */
> + if (ds && ds->ds_clp && d->nfs_client != ds->ds_clp) {
> + struct rpc_clnt *clnt = ds->ds_clp->cl_rpcclient;
> + rpc_add_iostats(total, clnt->cl_metrics,
> + min(maxproc, clnt->cl_maxproc));
> + num_ds++;
> + }
> + }
> +
> + return num_ds;
> +}
> +
> static struct pnfs_layoutdriver_type filelayout_type = {
> .id = LAYOUT_NFSV4_1_FILES,
> .name = "LAYOUT_NFSV4_1_FILES",
> @@ -932,6 +961,7 @@ static struct pnfs_layoutdriver_type filelayout_type = {
> .read_pagelist = filelayout_read_pagelist,
> .write_pagelist = filelayout_write_pagelist,
> .free_deviceid_node = filelayout_free_deveiceid_node,
> + .ds_rpc_add_iostats = filelayout_rpc_add_iostats,
> };
>
> static int __init nfs4filelayout_init(void)
> diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
> index 53d593a..d1b82eb 100644
> --- a/fs/nfs/pnfs.h
> +++ b/fs/nfs/pnfs.h
> @@ -72,6 +72,7 @@ enum layoutdriver_policy_flags {
> };
>
> struct nfs4_deviceid_node;
> +struct rpc_iostats;
>
> /* Per-layout driver specific registration structure */
> struct pnfs_layoutdriver_type {
> @@ -119,6 +120,9 @@ struct pnfs_layoutdriver_type {
> void (*encode_layoutcommit) (struct pnfs_layout_hdr *layoutid,
> struct xdr_stream *xdr,
> const struct nfs4_layoutcommit_args *args);
> +
> + unsigned int (*ds_rpc_add_iostats) (struct rpc_iostats *, unsigned int,
> + struct nfs4_deviceid_node *);
> };
>
> struct pnfs_layout_hdr {
> @@ -239,6 +243,10 @@ void nfs4_init_deviceid_node(struct nfs4_deviceid_node *,
> struct nfs4_deviceid_node *nfs4_insert_deviceid_node(struct nfs4_deviceid_node *);
> bool nfs4_put_deviceid_node(struct nfs4_deviceid_node *);
> void nfs4_deviceid_purge_client(const struct nfs_client *);
> +void nfs4_deviceid_rpc_print_iostats(struct seq_file *seq,
> + const struct nfs_client *clp,
> + const struct pnfs_layoutdriver_type *ld);
> +
>
> static inline int lo_fail_bit(u32 iomode)
> {
> @@ -328,6 +336,15 @@ static inline int pnfs_return_layout(struct inode *ino)
> return 0;
> }
>
> +static inline int
> +pnfs_rpc_print_iostats(struct seq_file *m, struct nfs_server *nfss)
> +{
> + if (!pnfs_enabled_sb(nfss))
> + return 0;
> + nfs4_deviceid_rpc_print_iostats(m, nfss->nfs_client,
> + nfss->pnfs_curr_ld);
> + return 1;
> +}
> #else /* CONFIG_NFS_V4_1 */
>
> static inline void pnfs_destroy_all_layouts(struct nfs_client *clp)
> @@ -429,6 +446,12 @@ static inline int pnfs_layoutcommit_inode(struct inode *inode, bool sync)
> static inline void nfs4_deviceid_purge_client(struct nfs_client *ncl)
> {
> }
> +
> +static inline void
> +pnfs_rpc_print_iostats(struct seq_file *m, struct nfs_server *nfss)
> +{
> + return 0;
> +}
> #endif /* CONFIG_NFS_V4_1 */
>
> #endif /* FS_NFS_PNFS_H */
> diff --git a/fs/nfs/pnfs_dev.c b/fs/nfs/pnfs_dev.c
> index 4f359d2..aa5d102 100644
> --- a/fs/nfs/pnfs_dev.c
> +++ b/fs/nfs/pnfs_dev.c
> @@ -29,6 +29,9 @@
> */
>
> #include <linux/export.h>
> +
> +#include <linux/sunrpc/metrics.h>
> +
> #include "pnfs.h"
>
> #define NFSDBG_FACILITY NFSDBG_PNFS
> @@ -115,6 +118,48 @@ nfs4_find_get_deviceid(const struct pnfs_layoutdriver_type *ld,
> }
> EXPORT_SYMBOL_GPL(nfs4_find_get_deviceid);
>
> +
> +void
> +nfs4_deviceid_rpc_print_iostats(struct seq_file *seq,
> + const struct nfs_client *clp,
> + const struct pnfs_layoutdriver_type *ld)
> +{
> + struct nfs4_deviceid_node *d;
> + struct hlist_node *n;
> + struct rpc_iostats *totals;
> + char msgbuf[64];
> + unsigned int num_ds = 0;
> + unsigned int maxproc;
> + long h;
> +
> + if (!ld->ds_rpc_add_iostats)
> + return;
> + totals = rpc_alloc_iostats(clp->cl_rpcclient);
> + if (!totals)
> + return;
> + maxproc = clp->cl_rpcclient->cl_maxproc;
> + rpc_add_iostats(totals, clp->cl_rpcclient->cl_metrics, maxproc);
> +
> + rcu_read_lock();
> + for (h = 0; h < NFS4_DEVICE_ID_HASH_SIZE; h++) {
> + hlist_for_each_entry_rcu(d, n, &nfs4_deviceid_cache[h], node) {
> + if (d->ld == ld && d->nfs_client == clp) {
> + if (atomic_read(&d->ref))
> + num_ds += ld->ds_rpc_add_iostats(totals,
> + maxproc, d);
> + }
> + }
> + }
> + rcu_read_unlock();
> +
> + snprintf(msgbuf, sizeof(msgbuf),
> + " for the metadata server and %u data server%s", num_ds,
> + (num_ds != 1) ? "s" : "");
> + rpc_print_iostats(seq, clp->cl_rpcclient, totals, msgbuf);
> + rpc_free_iostats(totals);
> +}
> +EXPORT_SYMBOL_GPL(nfs4_deviceid_rpc_print_iostats);
> +
> /*
> * Remove a deviceid from cache
> *
> diff --git a/fs/nfs/super.c b/fs/nfs/super.c
> index d18a90b..76d0850 100644
> --- a/fs/nfs/super.c
> +++ b/fs/nfs/super.c
> @@ -870,7 +870,8 @@ static int nfs_show_stats(struct seq_file *m, struct dentry *root)
> #endif
> seq_printf(m, "\n");
>
> - rpc_print_iostats(m, nfss->client);
> + if (!pnfs_rpc_print_iostats(m, nfss))
> + rpc_print_iostats(m, nfss->client, NULL, NULL);
>
> return 0;
> }
> diff --git a/include/linux/sunrpc/metrics.h b/include/linux/sunrpc/metrics.h
> index b6edbc0..e6e04e4 100644
> --- a/include/linux/sunrpc/metrics.h
> +++ b/include/linux/sunrpc/metrics.h
> @@ -75,14 +75,23 @@ struct rpc_clnt;
>
> struct rpc_iostats * rpc_alloc_iostats(struct rpc_clnt *);
> void rpc_count_iostats(struct rpc_task *);
> -void rpc_print_iostats(struct seq_file *, struct rpc_clnt *);
> +void rpc_print_iostats(struct seq_file *,
> + struct rpc_clnt *,
> + struct rpc_iostats *,
> + const char *);
> +void rpc_add_iostats(struct rpc_iostats *,
> + struct rpc_iostats *, unsigned int);
> void rpc_free_iostats(struct rpc_iostats *);
>
> #else /* CONFIG_PROC_FS */
>
> static inline struct rpc_iostats *rpc_alloc_iostats(struct rpc_clnt *clnt) { return NULL; }
> static inline void rpc_count_iostats(struct rpc_task *task) {}
> -static inline void rpc_print_iostats(struct seq_file *seq, struct rpc_clnt *clnt) {}
> +static inline void rpc_print_iostats(struct seq_file *seq,
> + struct rpc_iostats *stat) {}
> +static inline void rpc_add_iostats(struct rpc_iostats *total,
> + struct rpc_iostats *new,
> + unsigned int maxproc) {}
> static inline void rpc_free_iostats(struct rpc_iostats *stats) {}
>
> #endif /* CONFIG_PROC_FS */
> diff --git a/net/sunrpc/stats.c b/net/sunrpc/stats.c
> index 3c4f688..342df57 100644
> --- a/net/sunrpc/stats.c
> +++ b/net/sunrpc/stats.c
> @@ -176,12 +176,16 @@ static void _print_name(struct seq_file *seq, unsigned int op,
> seq_printf(seq, "\t%12u: ", op);
> }
>
> -void rpc_print_iostats(struct seq_file *seq, struct rpc_clnt *clnt)
> +void rpc_print_iostats(struct seq_file *seq, struct rpc_clnt *clnt,
> + struct rpc_iostats *altstats,
> + const char *altstats_msg)
> {
> struct rpc_iostats *stats = clnt->cl_metrics;
> struct rpc_xprt *xprt = clnt->cl_xprt;
> unsigned int op, maxproc = clnt->cl_maxproc;
>
> + if (altstats)
> + stats = altstats;
> if (!stats)
> return;
>
> @@ -192,7 +196,8 @@ void rpc_print_iostats(struct seq_file *seq, struct rpc_clnt *clnt)
> if (xprt)
> xprt->ops->print_stats(xprt, seq);
>
> - seq_printf(seq, "\tper-op statistics\n");
> + seq_printf(seq, "\tper-op statistics%s\n",
> + (altstats && altstats_msg) ? altstats_msg : "");
> for (op = 0; op < maxproc; op++) {
> struct rpc_iostats *metrics = &stats[op];
> _print_name(seq, op, clnt->cl_procinfo);
> @@ -209,6 +214,35 @@ void rpc_print_iostats(struct seq_file *seq, struct rpc_clnt *clnt)
> }
> EXPORT_SYMBOL_GPL(rpc_print_iostats);
>
> +void
> +rpc_add_iostats(struct rpc_iostats *totals, struct rpc_iostats *new,
> + unsigned int maxproc)
> +{
> + unsigned int op;
> +
> + for (op = 0; op < maxproc; op++) {
> + struct rpc_iostats *op_totals = &totals[op],
> + *op_new = &new[op];
> +
> + op_totals->om_ops += op_new->om_ops;
> + op_totals->om_ntrans += op_new->om_ntrans;
> + op_totals->om_timeouts += op_new->om_timeouts;
> +
> + op_totals->om_bytes_sent += op_new->om_bytes_sent;
> + op_totals->om_bytes_recv += op_new->om_bytes_recv;
> +
> + op_totals->om_queue = ktime_add(op_totals->om_queue,
> + op_new->om_queue);
> +
> + op_totals->om_rtt = ktime_add(op_totals->om_rtt,
> + op_new->om_rtt);
> +
> + op_totals->om_execute = ktime_add(op_totals->om_execute,
> + op_new->om_execute);
> + }
> +}
> +EXPORT_SYMBOL_GPL(rpc_add_iostats);
> +
> /*
> * Register/unregister RPC proc files
> */
> --
> 1.7.4.4
>
--
Chuck Lever
chuck[dot]lever[at]oracle[dot]com
On Feb 9, 2012, at 11:49 AM, Chuck Lever wrote:
>
> On Feb 9, 2012, at 11:41 AM, Weston Andros Adamson wrote:
>
>> Include RPC statistics from all data servers in /proc/self/mountstats for pNFS
>> filelayout mounts.
>>
>> The additional info printed on the "per-op statistics" line does not break
>> current mountstats(8) from nfs-utils.
>>
>> Signed-off-by: Weston Andros Adamson <[email protected]>
>> ---
>>
>> This is clearly a better approach than I was taking trying to keep DS rpc stats
>> separate in /proc/self/mountstats.
>
> I guess you could argue that since these are "mount" stats, it's a per mount point thing, not necessarily a per server thing. We've simply had it that way because prior to pNFS, you couldn't have more than one server per mount point.
Right, Trond made that point to me a while ago :)
For some reason I was afraid to change the sunrpc metrics interface until I realized that there were no users other than NFS.
>
> At some point we might consider also displaying some fs_locations data here, and perhaps a count of how many migration or replication failover events have been seen. For another day.
Agreed. First we need to fix the statsvers= issue with mountstats(8). I plan on poking at that today.
>
>> FYI, there are no other callers of rpc_print_iostats().
>>
>> The new output of /proc/self/mountstats works fine with current mountstats
>> userland tool. The additional data on the per-op statistics line doesn't make
>> it into the output, but I left it in /proc/self/mountstats because it doesn't
>> break anything and can be included in newer versions of mountstats.
>>
>> I plan on adding the per-DS stats (rpc and others) somewhere in /proc/fs/nfsfs/
>> soon.
>>
>> fs/nfs/nfs4filelayout.c | 30 ++++++++++++++++++++++++++
>> fs/nfs/pnfs.h | 23 ++++++++++++++++++++
>> fs/nfs/pnfs_dev.c | 45 ++++++++++++++++++++++++++++++++++++++++
>> fs/nfs/super.c | 3 +-
>> include/linux/sunrpc/metrics.h | 13 +++++++++-
>> net/sunrpc/stats.c | 38 ++++++++++++++++++++++++++++++++-
>> 6 files changed, 147 insertions(+), 5 deletions(-)
>>
>> diff --git a/fs/nfs/nfs4filelayout.c b/fs/nfs/nfs4filelayout.c
>> index 79be7ac..51c71e5 100644
>> --- a/fs/nfs/nfs4filelayout.c
>> +++ b/fs/nfs/nfs4filelayout.c
>> @@ -33,6 +33,8 @@
>> #include <linux/nfs_page.h>
>> #include <linux/module.h>
>>
>> +#include <linux/sunrpc/metrics.h>
>> +
>> #include "internal.h"
>> #include "nfs4filelayout.h"
>>
>> @@ -918,6 +920,33 @@ filelayout_free_deveiceid_node(struct nfs4_deviceid_node *d)
>> nfs4_fl_free_deviceid(container_of(d, struct nfs4_file_layout_dsaddr, id_node));
>> }
>>
>> +static unsigned int
>> +filelayout_rpc_add_iostats(struct rpc_iostats *total,
>> + unsigned int maxproc,
>> + struct nfs4_deviceid_node *d)
>> +{
>> + struct nfs4_file_layout_dsaddr *dsaddr;
>> + struct nfs4_pnfs_ds *ds;
>> + unsigned int num_ds = 0;
>> + u32 i;
>> +
>> + dsaddr = container_of(d, struct nfs4_file_layout_dsaddr, id_node);
>> +
>> + for (i = 0; i < dsaddr->ds_num; i++) {
>> + ds = dsaddr->ds_list[i];
>> +
>> + /* only add iostats if nfs_client is different from MDS */
>> + if (ds && ds->ds_clp && d->nfs_client != ds->ds_clp) {
>> + struct rpc_clnt *clnt = ds->ds_clp->cl_rpcclient;
>> + rpc_add_iostats(total, clnt->cl_metrics,
>> + min(maxproc, clnt->cl_maxproc));
>> + num_ds++;
>> + }
>> + }
>> +
>> + return num_ds;
>> +}
>> +
>> static struct pnfs_layoutdriver_type filelayout_type = {
>> .id = LAYOUT_NFSV4_1_FILES,
>> .name = "LAYOUT_NFSV4_1_FILES",
>> @@ -932,6 +961,7 @@ static struct pnfs_layoutdriver_type filelayout_type = {
>> .read_pagelist = filelayout_read_pagelist,
>> .write_pagelist = filelayout_write_pagelist,
>> .free_deviceid_node = filelayout_free_deveiceid_node,
>> + .ds_rpc_add_iostats = filelayout_rpc_add_iostats,
>> };
>>
>> static int __init nfs4filelayout_init(void)
>> diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
>> index 53d593a..d1b82eb 100644
>> --- a/fs/nfs/pnfs.h
>> +++ b/fs/nfs/pnfs.h
>> @@ -72,6 +72,7 @@ enum layoutdriver_policy_flags {
>> };
>>
>> struct nfs4_deviceid_node;
>> +struct rpc_iostats;
>>
>> /* Per-layout driver specific registration structure */
>> struct pnfs_layoutdriver_type {
>> @@ -119,6 +120,9 @@ struct pnfs_layoutdriver_type {
>> void (*encode_layoutcommit) (struct pnfs_layout_hdr *layoutid,
>> struct xdr_stream *xdr,
>> const struct nfs4_layoutcommit_args *args);
>> +
>> + unsigned int (*ds_rpc_add_iostats) (struct rpc_iostats *, unsigned int,
>> + struct nfs4_deviceid_node *);
>> };
>>
>> struct pnfs_layout_hdr {
>> @@ -239,6 +243,10 @@ void nfs4_init_deviceid_node(struct nfs4_deviceid_node *,
>> struct nfs4_deviceid_node *nfs4_insert_deviceid_node(struct nfs4_deviceid_node *);
>> bool nfs4_put_deviceid_node(struct nfs4_deviceid_node *);
>> void nfs4_deviceid_purge_client(const struct nfs_client *);
>> +void nfs4_deviceid_rpc_print_iostats(struct seq_file *seq,
>> + const struct nfs_client *clp,
>> + const struct pnfs_layoutdriver_type *ld);
>> +
>>
>> static inline int lo_fail_bit(u32 iomode)
>> {
>> @@ -328,6 +336,15 @@ static inline int pnfs_return_layout(struct inode *ino)
>> return 0;
>> }
>>
>> +static inline int
>> +pnfs_rpc_print_iostats(struct seq_file *m, struct nfs_server *nfss)
>> +{
>> + if (!pnfs_enabled_sb(nfss))
>> + return 0;
>> + nfs4_deviceid_rpc_print_iostats(m, nfss->nfs_client,
>> + nfss->pnfs_curr_ld);
>> + return 1;
>> +}
>> #else /* CONFIG_NFS_V4_1 */
>>
>> static inline void pnfs_destroy_all_layouts(struct nfs_client *clp)
>> @@ -429,6 +446,12 @@ static inline int pnfs_layoutcommit_inode(struct inode *inode, bool sync)
>> static inline void nfs4_deviceid_purge_client(struct nfs_client *ncl)
>> {
>> }
>> +
>> +static inline void
>> +pnfs_rpc_print_iostats(struct seq_file *m, struct nfs_server *nfss)
>> +{
>> + return 0;
>> +}
>> #endif /* CONFIG_NFS_V4_1 */
>>
>> #endif /* FS_NFS_PNFS_H */
>> diff --git a/fs/nfs/pnfs_dev.c b/fs/nfs/pnfs_dev.c
>> index 4f359d2..aa5d102 100644
>> --- a/fs/nfs/pnfs_dev.c
>> +++ b/fs/nfs/pnfs_dev.c
>> @@ -29,6 +29,9 @@
>> */
>>
>> #include <linux/export.h>
>> +
>> +#include <linux/sunrpc/metrics.h>
>> +
>> #include "pnfs.h"
>>
>> #define NFSDBG_FACILITY NFSDBG_PNFS
>> @@ -115,6 +118,48 @@ nfs4_find_get_deviceid(const struct pnfs_layoutdriver_type *ld,
>> }
>> EXPORT_SYMBOL_GPL(nfs4_find_get_deviceid);
>>
>> +
>> +void
>> +nfs4_deviceid_rpc_print_iostats(struct seq_file *seq,
>> + const struct nfs_client *clp,
>> + const struct pnfs_layoutdriver_type *ld)
>> +{
>> + struct nfs4_deviceid_node *d;
>> + struct hlist_node *n;
>> + struct rpc_iostats *totals;
>> + char msgbuf[64];
>> + unsigned int num_ds = 0;
>> + unsigned int maxproc;
>> + long h;
>> +
>> + if (!ld->ds_rpc_add_iostats)
>> + return;
>> + totals = rpc_alloc_iostats(clp->cl_rpcclient);
>> + if (!totals)
>> + return;
>> + maxproc = clp->cl_rpcclient->cl_maxproc;
>> + rpc_add_iostats(totals, clp->cl_rpcclient->cl_metrics, maxproc);
>> +
>> + rcu_read_lock();
>> + for (h = 0; h < NFS4_DEVICE_ID_HASH_SIZE; h++) {
>> + hlist_for_each_entry_rcu(d, n, &nfs4_deviceid_cache[h], node) {
>> + if (d->ld == ld && d->nfs_client == clp) {
>> + if (atomic_read(&d->ref))
>> + num_ds += ld->ds_rpc_add_iostats(totals,
>> + maxproc, d);
>> + }
>> + }
>> + }
>> + rcu_read_unlock();
>> +
>> + snprintf(msgbuf, sizeof(msgbuf),
>> + " for the metadata server and %u data server%s", num_ds,
>> + (num_ds != 1) ? "s" : "");
>> + rpc_print_iostats(seq, clp->cl_rpcclient, totals, msgbuf);
>> + rpc_free_iostats(totals);
>> +}
>> +EXPORT_SYMBOL_GPL(nfs4_deviceid_rpc_print_iostats);
>> +
>> /*
>> * Remove a deviceid from cache
>> *
>> diff --git a/fs/nfs/super.c b/fs/nfs/super.c
>> index d18a90b..76d0850 100644
>> --- a/fs/nfs/super.c
>> +++ b/fs/nfs/super.c
>> @@ -870,7 +870,8 @@ static int nfs_show_stats(struct seq_file *m, struct dentry *root)
>> #endif
>> seq_printf(m, "\n");
>>
>> - rpc_print_iostats(m, nfss->client);
>> + if (!pnfs_rpc_print_iostats(m, nfss))
>> + rpc_print_iostats(m, nfss->client, NULL, NULL);
>>
>> return 0;
>> }
>> diff --git a/include/linux/sunrpc/metrics.h b/include/linux/sunrpc/metrics.h
>> index b6edbc0..e6e04e4 100644
>> --- a/include/linux/sunrpc/metrics.h
>> +++ b/include/linux/sunrpc/metrics.h
>> @@ -75,14 +75,23 @@ struct rpc_clnt;
>>
>> struct rpc_iostats * rpc_alloc_iostats(struct rpc_clnt *);
>> void rpc_count_iostats(struct rpc_task *);
>> -void rpc_print_iostats(struct seq_file *, struct rpc_clnt *);
>> +void rpc_print_iostats(struct seq_file *,
>> + struct rpc_clnt *,
>> + struct rpc_iostats *,
>> + const char *);
>> +void rpc_add_iostats(struct rpc_iostats *,
>> + struct rpc_iostats *, unsigned int);
>> void rpc_free_iostats(struct rpc_iostats *);
>>
>> #else /* CONFIG_PROC_FS */
>>
>> static inline struct rpc_iostats *rpc_alloc_iostats(struct rpc_clnt *clnt) { return NULL; }
>> static inline void rpc_count_iostats(struct rpc_task *task) {}
>> -static inline void rpc_print_iostats(struct seq_file *seq, struct rpc_clnt *clnt) {}
>> +static inline void rpc_print_iostats(struct seq_file *seq,
>> + struct rpc_iostats *stat) {}
>> +static inline void rpc_add_iostats(struct rpc_iostats *total,
>> + struct rpc_iostats *new,
>> + unsigned int maxproc) {}
>> static inline void rpc_free_iostats(struct rpc_iostats *stats) {}
>>
>> #endif /* CONFIG_PROC_FS */
>> diff --git a/net/sunrpc/stats.c b/net/sunrpc/stats.c
>> index 3c4f688..342df57 100644
>> --- a/net/sunrpc/stats.c
>> +++ b/net/sunrpc/stats.c
>> @@ -176,12 +176,16 @@ static void _print_name(struct seq_file *seq, unsigned int op,
>> seq_printf(seq, "\t%12u: ", op);
>> }
>>
>> -void rpc_print_iostats(struct seq_file *seq, struct rpc_clnt *clnt)
>> +void rpc_print_iostats(struct seq_file *seq, struct rpc_clnt *clnt,
>> + struct rpc_iostats *altstats,
>> + const char *altstats_msg)
>> {
>> struct rpc_iostats *stats = clnt->cl_metrics;
>> struct rpc_xprt *xprt = clnt->cl_xprt;
>> unsigned int op, maxproc = clnt->cl_maxproc;
>>
>> + if (altstats)
>> + stats = altstats;
>> if (!stats)
>> return;
>>
>> @@ -192,7 +196,8 @@ void rpc_print_iostats(struct seq_file *seq, struct rpc_clnt *clnt)
>> if (xprt)
>> xprt->ops->print_stats(xprt, seq);
>>
>> - seq_printf(seq, "\tper-op statistics\n");
>> + seq_printf(seq, "\tper-op statistics%s\n",
>> + (altstats && altstats_msg) ? altstats_msg : "");
>> for (op = 0; op < maxproc; op++) {
>> struct rpc_iostats *metrics = &stats[op];
>> _print_name(seq, op, clnt->cl_procinfo);
>> @@ -209,6 +214,35 @@ void rpc_print_iostats(struct seq_file *seq, struct rpc_clnt *clnt)
>> }
>> EXPORT_SYMBOL_GPL(rpc_print_iostats);
>>
>> +void
>> +rpc_add_iostats(struct rpc_iostats *totals, struct rpc_iostats *new,
>> + unsigned int maxproc)
>> +{
>> + unsigned int op;
>> +
>> + for (op = 0; op < maxproc; op++) {
>> + struct rpc_iostats *op_totals = &totals[op],
>> + *op_new = &new[op];
>> +
>> + op_totals->om_ops += op_new->om_ops;
>> + op_totals->om_ntrans += op_new->om_ntrans;
>> + op_totals->om_timeouts += op_new->om_timeouts;
>> +
>> + op_totals->om_bytes_sent += op_new->om_bytes_sent;
>> + op_totals->om_bytes_recv += op_new->om_bytes_recv;
>> +
>> + op_totals->om_queue = ktime_add(op_totals->om_queue,
>> + op_new->om_queue);
>> +
>> + op_totals->om_rtt = ktime_add(op_totals->om_rtt,
>> + op_new->om_rtt);
>> +
>> + op_totals->om_execute = ktime_add(op_totals->om_execute,
>> + op_new->om_execute);
>> + }
>> +}
>> +EXPORT_SYMBOL_GPL(rpc_add_iostats);
>> +
>> /*
>> * Register/unregister RPC proc files
>> */
>> --
>> 1.7.4.4
>>
>
> --
> Chuck Lever
> chuck[dot]lever[at]oracle[dot]com
>
>
>
>
T24gVGh1LCAyMDEyLTAyLTA5IGF0IDExOjQxIC0wNTAwLCBXZXN0b24gQW5kcm9zIEFkYW1zb24g
d3JvdGU6DQo+IEluY2x1ZGUgUlBDIHN0YXRpc3RpY3MgZnJvbSBhbGwgZGF0YSBzZXJ2ZXJzIGlu
IC9wcm9jL3NlbGYvbW91bnRzdGF0cyBmb3IgcE5GUw0KPiBmaWxlbGF5b3V0IG1vdW50cy4NCj4g
DQo+IFRoZSBhZGRpdGlvbmFsIGluZm8gcHJpbnRlZCBvbiB0aGUgInBlci1vcCBzdGF0aXN0aWNz
IiBsaW5lIGRvZXMgbm90IGJyZWFrDQo+IGN1cnJlbnQgbW91bnRzdGF0cyg4KSBmcm9tIG5mcy11
dGlscy4NCj4gDQo+IFNpZ25lZC1vZmYtYnk6IFdlc3RvbiBBbmRyb3MgQWRhbXNvbiA8ZHJvc0Bu
ZXRhcHAuY29tPg0KPiAtLS0NCj4gDQo+IFRoaXMgaXMgY2xlYXJseSBhIGJldHRlciBhcHByb2Fj
aCB0aGFuIEkgd2FzIHRha2luZyB0cnlpbmcgdG8ga2VlcCBEUyBycGMgc3RhdHMNCj4gc2VwYXJh
dGUgaW4gL3Byb2Mvc2VsZi9tb3VudHN0YXRzLg0KPiANCj4gRllJLCB0aGVyZSBhcmUgbm8gb3Ro
ZXIgY2FsbGVycyBvZiBycGNfcHJpbnRfaW9zdGF0cygpLg0KPiANCj4gVGhlIG5ldyBvdXRwdXQg
b2YgL3Byb2Mvc2VsZi9tb3VudHN0YXRzIHdvcmtzIGZpbmUgd2l0aCBjdXJyZW50IG1vdW50c3Rh
dHMgDQo+IHVzZXJsYW5kIHRvb2wuICBUaGUgYWRkaXRpb25hbCBkYXRhIG9uIHRoZSBwZXItb3Ag
c3RhdGlzdGljcyBsaW5lIGRvZXNuJ3QgbWFrZQ0KPiBpdCBpbnRvIHRoZSBvdXRwdXQsIGJ1dCBJ
IGxlZnQgaXQgaW4gL3Byb2Mvc2VsZi9tb3VudHN0YXRzIGJlY2F1c2UgaXQgZG9lc24ndA0KPiBi
cmVhayBhbnl0aGluZyBhbmQgY2FuIGJlIGluY2x1ZGVkIGluIG5ld2VyIHZlcnNpb25zIG9mIG1v
dW50c3RhdHMuDQo+IA0KPiBJIHBsYW4gb24gYWRkaW5nIHRoZSBwZXItRFMgc3RhdHMgKHJwYyBh
bmQgb3RoZXJzKSBzb21ld2hlcmUgaW4gL3Byb2MvZnMvbmZzZnMvDQo+IHNvb24uDQo+IA0KPiAg
ZnMvbmZzL25mczRmaWxlbGF5b3V0LmMgICAgICAgIHwgICAzMCArKysrKysrKysrKysrKysrKysr
KysrKysrKw0KPiAgZnMvbmZzL3BuZnMuaCAgICAgICAgICAgICAgICAgIHwgICAyMyArKysrKysr
KysrKysrKysrKysrKw0KPiAgZnMvbmZzL3BuZnNfZGV2LmMgICAgICAgICAgICAgIHwgICA0NSAr
KysrKysrKysrKysrKysrKysrKysrKysrKysrKysrKysrKysrKysrDQo+ICBmcy9uZnMvc3VwZXIu
YyAgICAgICAgICAgICAgICAgfCAgICAzICstDQo+ICBpbmNsdWRlL2xpbnV4L3N1bnJwYy9tZXRy
aWNzLmggfCAgIDEzICsrKysrKysrKy0NCj4gIG5ldC9zdW5ycGMvc3RhdHMuYyAgICAgICAgICAg
ICB8ICAgMzggKysrKysrKysrKysrKysrKysrKysrKysrKysrKysrKystDQo+ICA2IGZpbGVzIGNo
YW5nZWQsIDE0NyBpbnNlcnRpb25zKCspLCA1IGRlbGV0aW9ucygtKQ0KPiANCj4gZGlmZiAtLWdp
dCBhL2ZzL25mcy9uZnM0ZmlsZWxheW91dC5jIGIvZnMvbmZzL25mczRmaWxlbGF5b3V0LmMNCj4g
aW5kZXggNzliZTdhYy4uNTFjNzFlNSAxMDA2NDQNCj4gLS0tIGEvZnMvbmZzL25mczRmaWxlbGF5
b3V0LmMNCj4gKysrIGIvZnMvbmZzL25mczRmaWxlbGF5b3V0LmMNCj4gQEAgLTMzLDYgKzMzLDgg
QEANCj4gICNpbmNsdWRlIDxsaW51eC9uZnNfcGFnZS5oPg0KPiAgI2luY2x1ZGUgPGxpbnV4L21v
ZHVsZS5oPg0KPiAgDQo+ICsjaW5jbHVkZSA8bGludXgvc3VucnBjL21ldHJpY3MuaD4NCj4gKw0K
PiAgI2luY2x1ZGUgImludGVybmFsLmgiDQo+ICAjaW5jbHVkZSAibmZzNGZpbGVsYXlvdXQuaCIN
Cj4gIA0KPiBAQCAtOTE4LDYgKzkyMCwzMyBAQCBmaWxlbGF5b3V0X2ZyZWVfZGV2ZWljZWlkX25v
ZGUoc3RydWN0IG5mczRfZGV2aWNlaWRfbm9kZSAqZCkNCj4gIAluZnM0X2ZsX2ZyZWVfZGV2aWNl
aWQoY29udGFpbmVyX29mKGQsIHN0cnVjdCBuZnM0X2ZpbGVfbGF5b3V0X2RzYWRkciwgaWRfbm9k
ZSkpOw0KPiAgfQ0KPiAgDQo+ICtzdGF0aWMgdW5zaWduZWQgaW50DQo+ICtmaWxlbGF5b3V0X3Jw
Y19hZGRfaW9zdGF0cyhzdHJ1Y3QgcnBjX2lvc3RhdHMgKnRvdGFsLA0KPiArCQkJICAgdW5zaWdu
ZWQgaW50IG1heHByb2MsDQo+ICsJCQkgICBzdHJ1Y3QgbmZzNF9kZXZpY2VpZF9ub2RlICpkKQ0K
PiArew0KPiArCXN0cnVjdCBuZnM0X2ZpbGVfbGF5b3V0X2RzYWRkciAqZHNhZGRyOw0KPiArCXN0
cnVjdCBuZnM0X3BuZnNfZHMgKmRzOw0KPiArCXVuc2lnbmVkIGludCBudW1fZHMgPSAwOw0KPiAr
CXUzMiBpOw0KPiArDQo+ICsJZHNhZGRyID0gY29udGFpbmVyX29mKGQsIHN0cnVjdCBuZnM0X2Zp
bGVfbGF5b3V0X2RzYWRkciwgaWRfbm9kZSk7DQo+ICsNCj4gKwlmb3IgKGkgPSAwOyBpIDwgZHNh
ZGRyLT5kc19udW07IGkrKykgew0KPiArCQlkcyA9IGRzYWRkci0+ZHNfbGlzdFtpXTsNCj4gKw0K
PiArCQkvKiBvbmx5IGFkZCBpb3N0YXRzIGlmIG5mc19jbGllbnQgaXMgZGlmZmVyZW50IGZyb20g
TURTICovDQo+ICsJCWlmIChkcyAmJiBkcy0+ZHNfY2xwICYmIGQtPm5mc19jbGllbnQgIT0gZHMt
PmRzX2NscCkgew0KPiArCQkJc3RydWN0IHJwY19jbG50ICpjbG50ID0gZHMtPmRzX2NscC0+Y2xf
cnBjY2xpZW50Ow0KDQpIbW1tLi4uLiBTaG91bGRuJ3QgdGhpcyByYXRoZXIgYmUgdGhlIE5GU19D
TElFTlQoaW5vZGUpPyBBbHNvLCB3aHkgbm90DQpkbyB0aGlzIGluIHRoZSByZWFkL3dyaXRlIGNv
bXBsZXRpb24gY29kZT8gV2h5IHdhbGsgdGhlIGRldmljZWlkIGxpc3QgYXQNCmFsbD8NCg0KPiAr
CQkJcnBjX2FkZF9pb3N0YXRzKHRvdGFsLCBjbG50LT5jbF9tZXRyaWNzLA0KPiArCQkJCQltaW4o
bWF4cHJvYywgY2xudC0+Y2xfbWF4cHJvYykpOw0KPiArCQkJbnVtX2RzKys7DQo+ICsJCX0NCj4g
Kwl9DQo+ICsNCj4gKwlyZXR1cm4gbnVtX2RzOw0KPiArfQ0KPiArDQo+ICBzdGF0aWMgc3RydWN0
IHBuZnNfbGF5b3V0ZHJpdmVyX3R5cGUgZmlsZWxheW91dF90eXBlID0gew0KPiAgCS5pZAkJCT0g
TEFZT1VUX05GU1Y0XzFfRklMRVMsDQo+ICAJLm5hbWUJCQk9ICJMQVlPVVRfTkZTVjRfMV9GSUxF
UyIsDQo+IEBAIC05MzIsNiArOTYxLDcgQEAgc3RhdGljIHN0cnVjdCBwbmZzX2xheW91dGRyaXZl
cl90eXBlIGZpbGVsYXlvdXRfdHlwZSA9IHsNCj4gIAkucmVhZF9wYWdlbGlzdAkJPSBmaWxlbGF5
b3V0X3JlYWRfcGFnZWxpc3QsDQo+ICAJLndyaXRlX3BhZ2VsaXN0CQk9IGZpbGVsYXlvdXRfd3Jp
dGVfcGFnZWxpc3QsDQo+ICAJLmZyZWVfZGV2aWNlaWRfbm9kZQk9IGZpbGVsYXlvdXRfZnJlZV9k
ZXZlaWNlaWRfbm9kZSwNCj4gKwkuZHNfcnBjX2FkZF9pb3N0YXRzCT0gZmlsZWxheW91dF9ycGNf
YWRkX2lvc3RhdHMsDQo+ICB9Ow0KPiAgDQo+ICBzdGF0aWMgaW50IF9faW5pdCBuZnM0ZmlsZWxh
eW91dF9pbml0KHZvaWQpDQo+IGRpZmYgLS1naXQgYS9mcy9uZnMvcG5mcy5oIGIvZnMvbmZzL3Bu
ZnMuaA0KPiBpbmRleCA1M2Q1OTNhLi5kMWI4MmViIDEwMDY0NA0KPiAtLS0gYS9mcy9uZnMvcG5m
cy5oDQo+ICsrKyBiL2ZzL25mcy9wbmZzLmgNCj4gQEAgLTcyLDYgKzcyLDcgQEAgZW51bSBsYXlv
dXRkcml2ZXJfcG9saWN5X2ZsYWdzIHsNCj4gIH07DQo+ICANCj4gIHN0cnVjdCBuZnM0X2Rldmlj
ZWlkX25vZGU7DQo+ICtzdHJ1Y3QgcnBjX2lvc3RhdHM7DQo+ICANCj4gIC8qIFBlci1sYXlvdXQg
ZHJpdmVyIHNwZWNpZmljIHJlZ2lzdHJhdGlvbiBzdHJ1Y3R1cmUgKi8NCj4gIHN0cnVjdCBwbmZz
X2xheW91dGRyaXZlcl90eXBlIHsNCj4gQEAgLTExOSw2ICsxMjAsOSBAQCBzdHJ1Y3QgcG5mc19s
YXlvdXRkcml2ZXJfdHlwZSB7DQo+ICAJdm9pZCAoKmVuY29kZV9sYXlvdXRjb21taXQpIChzdHJ1
Y3QgcG5mc19sYXlvdXRfaGRyICpsYXlvdXRpZCwNCj4gIAkJCQkgICAgIHN0cnVjdCB4ZHJfc3Ry
ZWFtICp4ZHIsDQo+ICAJCQkJICAgICBjb25zdCBzdHJ1Y3QgbmZzNF9sYXlvdXRjb21taXRfYXJn
cyAqYXJncyk7DQo+ICsNCj4gKwl1bnNpZ25lZCBpbnQgKCpkc19ycGNfYWRkX2lvc3RhdHMpIChz
dHJ1Y3QgcnBjX2lvc3RhdHMgKiwgdW5zaWduZWQgaW50LA0KPiArCQkJCQkgICAgc3RydWN0IG5m
czRfZGV2aWNlaWRfbm9kZSAqKTsNCj4gIH07DQo+ICANCj4gIHN0cnVjdCBwbmZzX2xheW91dF9o
ZHIgew0KPiBAQCAtMjM5LDYgKzI0MywxMCBAQCB2b2lkIG5mczRfaW5pdF9kZXZpY2VpZF9ub2Rl
KHN0cnVjdCBuZnM0X2RldmljZWlkX25vZGUgKiwNCj4gIHN0cnVjdCBuZnM0X2RldmljZWlkX25v
ZGUgKm5mczRfaW5zZXJ0X2RldmljZWlkX25vZGUoc3RydWN0IG5mczRfZGV2aWNlaWRfbm9kZSAq
KTsNCj4gIGJvb2wgbmZzNF9wdXRfZGV2aWNlaWRfbm9kZShzdHJ1Y3QgbmZzNF9kZXZpY2VpZF9u
b2RlICopOw0KPiAgdm9pZCBuZnM0X2RldmljZWlkX3B1cmdlX2NsaWVudChjb25zdCBzdHJ1Y3Qg
bmZzX2NsaWVudCAqKTsNCj4gK3ZvaWQgbmZzNF9kZXZpY2VpZF9ycGNfcHJpbnRfaW9zdGF0cyhz
dHJ1Y3Qgc2VxX2ZpbGUgKnNlcSwNCj4gKwkJCQkgICAgIGNvbnN0IHN0cnVjdCBuZnNfY2xpZW50
ICpjbHAsDQo+ICsJCQkJICAgICBjb25zdCBzdHJ1Y3QgcG5mc19sYXlvdXRkcml2ZXJfdHlwZSAq
bGQpOw0KPiArDQo+ICANCj4gIHN0YXRpYyBpbmxpbmUgaW50IGxvX2ZhaWxfYml0KHUzMiBpb21v
ZGUpDQo+ICB7DQo+IEBAIC0zMjgsNiArMzM2LDE1IEBAIHN0YXRpYyBpbmxpbmUgaW50IHBuZnNf
cmV0dXJuX2xheW91dChzdHJ1Y3QgaW5vZGUgKmlubykNCj4gIAlyZXR1cm4gMDsNCj4gIH0NCj4g
IA0KPiArc3RhdGljIGlubGluZSBpbnQNCj4gK3BuZnNfcnBjX3ByaW50X2lvc3RhdHMoc3RydWN0
IHNlcV9maWxlICptLCBzdHJ1Y3QgbmZzX3NlcnZlciAqbmZzcykNCj4gK3sNCj4gKwlpZiAoIXBu
ZnNfZW5hYmxlZF9zYihuZnNzKSkNCj4gKwkJcmV0dXJuIDA7DQo+ICsJbmZzNF9kZXZpY2VpZF9y
cGNfcHJpbnRfaW9zdGF0cyhtLCBuZnNzLT5uZnNfY2xpZW50LA0KPiArCQkJCQluZnNzLT5wbmZz
X2N1cnJfbGQpOw0KPiArCXJldHVybiAxOw0KPiArfQ0KPiAgI2Vsc2UgIC8qIENPTkZJR19ORlNf
VjRfMSAqLw0KPiAgDQo+ICBzdGF0aWMgaW5saW5lIHZvaWQgcG5mc19kZXN0cm95X2FsbF9sYXlv
dXRzKHN0cnVjdCBuZnNfY2xpZW50ICpjbHApDQo+IEBAIC00MjksNiArNDQ2LDEyIEBAIHN0YXRp
YyBpbmxpbmUgaW50IHBuZnNfbGF5b3V0Y29tbWl0X2lub2RlKHN0cnVjdCBpbm9kZSAqaW5vZGUs
IGJvb2wgc3luYykNCj4gIHN0YXRpYyBpbmxpbmUgdm9pZCBuZnM0X2RldmljZWlkX3B1cmdlX2Ns
aWVudChzdHJ1Y3QgbmZzX2NsaWVudCAqbmNsKQ0KPiAgew0KPiAgfQ0KPiArDQo+ICtzdGF0aWMg
aW5saW5lIHZvaWQNCj4gK3BuZnNfcnBjX3ByaW50X2lvc3RhdHMoc3RydWN0IHNlcV9maWxlICpt
LCBzdHJ1Y3QgbmZzX3NlcnZlciAqbmZzcykNCj4gK3sNCj4gKwlyZXR1cm4gMDsNCj4gK30NCj4g
ICNlbmRpZiAvKiBDT05GSUdfTkZTX1Y0XzEgKi8NCj4gIA0KPiAgI2VuZGlmIC8qIEZTX05GU19Q
TkZTX0ggKi8NCj4gZGlmZiAtLWdpdCBhL2ZzL25mcy9wbmZzX2Rldi5jIGIvZnMvbmZzL3BuZnNf
ZGV2LmMNCj4gaW5kZXggNGYzNTlkMi4uYWE1ZDEwMiAxMDA2NDQNCj4gLS0tIGEvZnMvbmZzL3Bu
ZnNfZGV2LmMNCj4gKysrIGIvZnMvbmZzL3BuZnNfZGV2LmMNCj4gQEAgLTI5LDYgKzI5LDkgQEAN
Cj4gICAqLw0KPiAgDQo+ICAjaW5jbHVkZSA8bGludXgvZXhwb3J0Lmg+DQo+ICsNCj4gKyNpbmNs
dWRlIDxsaW51eC9zdW5ycGMvbWV0cmljcy5oPg0KPiArDQo+ICAjaW5jbHVkZSAicG5mcy5oIg0K
PiAgDQo+ICAjZGVmaW5lIE5GU0RCR19GQUNJTElUWQkJTkZTREJHX1BORlMNCj4gQEAgLTExNSw2
ICsxMTgsNDggQEAgbmZzNF9maW5kX2dldF9kZXZpY2VpZChjb25zdCBzdHJ1Y3QgcG5mc19sYXlv
dXRkcml2ZXJfdHlwZSAqbGQsDQo+ICB9DQo+ICBFWFBPUlRfU1lNQk9MX0dQTChuZnM0X2ZpbmRf
Z2V0X2RldmljZWlkKTsNCj4gIA0KPiArDQo+ICt2b2lkDQo+ICtuZnM0X2RldmljZWlkX3JwY19w
cmludF9pb3N0YXRzKHN0cnVjdCBzZXFfZmlsZSAqc2VxLA0KPiArCQkJCWNvbnN0IHN0cnVjdCBu
ZnNfY2xpZW50ICpjbHAsDQo+ICsJCQkJY29uc3Qgc3RydWN0IHBuZnNfbGF5b3V0ZHJpdmVyX3R5
cGUgKmxkKQ0KPiArew0KPiArCXN0cnVjdCBuZnM0X2RldmljZWlkX25vZGUgKmQ7DQo+ICsJc3Ry
dWN0IGhsaXN0X25vZGUgKm47DQo+ICsJc3RydWN0IHJwY19pb3N0YXRzICp0b3RhbHM7DQo+ICsJ
Y2hhciBtc2didWZbNjRdOw0KPiArCXVuc2lnbmVkIGludCBudW1fZHMgPSAwOw0KPiArCXVuc2ln
bmVkIGludCBtYXhwcm9jOw0KPiArCWxvbmcgaDsNCj4gKw0KPiArCWlmICghbGQtPmRzX3JwY19h
ZGRfaW9zdGF0cykNCj4gKwkJcmV0dXJuOw0KPiArCXRvdGFscyA9IHJwY19hbGxvY19pb3N0YXRz
KGNscC0+Y2xfcnBjY2xpZW50KTsNCj4gKwlpZiAoIXRvdGFscykNCj4gKwkJcmV0dXJuOw0KPiAr
CW1heHByb2MgPSBjbHAtPmNsX3JwY2NsaWVudC0+Y2xfbWF4cHJvYzsNCj4gKwlycGNfYWRkX2lv
c3RhdHModG90YWxzLCBjbHAtPmNsX3JwY2NsaWVudC0+Y2xfbWV0cmljcywgbWF4cHJvYyk7DQo+
ICsNCj4gKwlyY3VfcmVhZF9sb2NrKCk7DQo+ICsJZm9yIChoID0gMDsgaCA8IE5GUzRfREVWSUNF
X0lEX0hBU0hfU0laRTsgaCsrKSB7DQo+ICsJCWhsaXN0X2Zvcl9lYWNoX2VudHJ5X3JjdShkLCBu
LCAmbmZzNF9kZXZpY2VpZF9jYWNoZVtoXSwgbm9kZSkgew0KPiArCQkJaWYgKGQtPmxkID09IGxk
ICYmIGQtPm5mc19jbGllbnQgPT0gY2xwKSB7DQo+ICsJCQkJaWYgKGF0b21pY19yZWFkKCZkLT5y
ZWYpKQ0KPiArCQkJCQludW1fZHMgKz0gbGQtPmRzX3JwY19hZGRfaW9zdGF0cyh0b3RhbHMsDQo+
ICsJCQkJCQkJbWF4cHJvYywgZCk7DQo+ICsJCQl9DQo+ICsJCX0NCj4gKwl9DQo+ICsJcmN1X3Jl
YWRfdW5sb2NrKCk7DQo+ICsNCj4gKwlzbnByaW50Zihtc2didWYsIHNpemVvZihtc2didWYpLA0K
PiArCQkgIiBmb3IgdGhlIG1ldGFkYXRhIHNlcnZlciBhbmQgJXUgZGF0YSBzZXJ2ZXIlcyIsIG51
bV9kcywNCj4gKwkJIChudW1fZHMgIT0gMSkgPyAicyIgOiAiIik7DQo+ICsJcnBjX3ByaW50X2lv
c3RhdHMoc2VxLCBjbHAtPmNsX3JwY2NsaWVudCwgdG90YWxzLCBtc2didWYpOw0KPiArCXJwY19m
cmVlX2lvc3RhdHModG90YWxzKTsNCj4gK30NCj4gK0VYUE9SVF9TWU1CT0xfR1BMKG5mczRfZGV2
aWNlaWRfcnBjX3ByaW50X2lvc3RhdHMpOw0KDQpJcyB0aGlzIG5lZWRlZD8gQUZBSUNTIGl0IGlz
IHVzZWQgaW4gZnMvbmZzL3N1cGVyLmMsIHdoaWNoIGlzIGluIHRoZQ0Kc2FtZSBtb2R1bGUgKGku
ZS4gbmZzLmtvKS4NCg0KPiAgLyoNCj4gICAqIFJlbW92ZSBhIGRldmljZWlkIGZyb20gY2FjaGUN
Cj4gICAqDQo+IGRpZmYgLS1naXQgYS9mcy9uZnMvc3VwZXIuYyBiL2ZzL25mcy9zdXBlci5jDQo+
IGluZGV4IGQxOGE5MGIuLjc2ZDA4NTAgMTAwNjQ0DQo+IC0tLSBhL2ZzL25mcy9zdXBlci5jDQo+
ICsrKyBiL2ZzL25mcy9zdXBlci5jDQo+IEBAIC04NzAsNyArODcwLDggQEAgc3RhdGljIGludCBu
ZnNfc2hvd19zdGF0cyhzdHJ1Y3Qgc2VxX2ZpbGUgKm0sIHN0cnVjdCBkZW50cnkgKnJvb3QpDQo+
ICAjZW5kaWYNCj4gIAlzZXFfcHJpbnRmKG0sICJcbiIpOw0KPiAgDQo+IC0JcnBjX3ByaW50X2lv
c3RhdHMobSwgbmZzcy0+Y2xpZW50KTsNCj4gKwlpZiAoIXBuZnNfcnBjX3ByaW50X2lvc3RhdHMo
bSwgbmZzcykpDQo+ICsJCXJwY19wcmludF9pb3N0YXRzKG0sIG5mc3MtPmNsaWVudCwgTlVMTCwg
TlVMTCk7DQo+ICANCj4gIAlyZXR1cm4gMDsNCj4gIH0NCj4gZGlmZiAtLWdpdCBhL2luY2x1ZGUv
bGludXgvc3VucnBjL21ldHJpY3MuaCBiL2luY2x1ZGUvbGludXgvc3VucnBjL21ldHJpY3MuaA0K
PiBpbmRleCBiNmVkYmMwLi5lNmUwNGU0IDEwMDY0NA0KPiAtLS0gYS9pbmNsdWRlL2xpbnV4L3N1
bnJwYy9tZXRyaWNzLmgNCj4gKysrIGIvaW5jbHVkZS9saW51eC9zdW5ycGMvbWV0cmljcy5oDQo+
IEBAIC03NSwxNCArNzUsMjMgQEAgc3RydWN0IHJwY19jbG50Ow0KPiAgDQo+ICBzdHJ1Y3QgcnBj
X2lvc3RhdHMgKglycGNfYWxsb2NfaW9zdGF0cyhzdHJ1Y3QgcnBjX2NsbnQgKik7DQo+ICB2b2lk
CQkJcnBjX2NvdW50X2lvc3RhdHMoc3RydWN0IHJwY190YXNrICopOw0KPiAtdm9pZAkJCXJwY19w
cmludF9pb3N0YXRzKHN0cnVjdCBzZXFfZmlsZSAqLCBzdHJ1Y3QgcnBjX2NsbnQgKik7DQo+ICt2
b2lkCQkJcnBjX3ByaW50X2lvc3RhdHMoc3RydWN0IHNlcV9maWxlICosDQo+ICsJCQkJCSAgc3Ry
dWN0IHJwY19jbG50ICosDQo+ICsJCQkJCSAgc3RydWN0IHJwY19pb3N0YXRzICosDQo+ICsJCQkJ
CSAgY29uc3QgY2hhciAqKTsNCj4gK3ZvaWQJCQlycGNfYWRkX2lvc3RhdHMoc3RydWN0IHJwY19p
b3N0YXRzICosDQo+ICsJCQkJCXN0cnVjdCBycGNfaW9zdGF0cyAqLCB1bnNpZ25lZCBpbnQpOw0K
PiAgdm9pZAkJCXJwY19mcmVlX2lvc3RhdHMoc3RydWN0IHJwY19pb3N0YXRzICopOw0KPiAgDQo+
ICAjZWxzZSAgLyogIENPTkZJR19QUk9DX0ZTICAqLw0KPiAgDQo+ICBzdGF0aWMgaW5saW5lIHN0
cnVjdCBycGNfaW9zdGF0cyAqcnBjX2FsbG9jX2lvc3RhdHMoc3RydWN0IHJwY19jbG50ICpjbG50
KSB7IHJldHVybiBOVUxMOyB9DQo+ICBzdGF0aWMgaW5saW5lIHZvaWQgcnBjX2NvdW50X2lvc3Rh
dHMoc3RydWN0IHJwY190YXNrICp0YXNrKSB7fQ0KPiAtc3RhdGljIGlubGluZSB2b2lkIHJwY19w
cmludF9pb3N0YXRzKHN0cnVjdCBzZXFfZmlsZSAqc2VxLCBzdHJ1Y3QgcnBjX2NsbnQgKmNsbnQp
IHt9DQo+ICtzdGF0aWMgaW5saW5lIHZvaWQgcnBjX3ByaW50X2lvc3RhdHMoc3RydWN0IHNlcV9m
aWxlICpzZXEsDQo+ICsJCQkJICAgICBzdHJ1Y3QgcnBjX2lvc3RhdHMgKnN0YXQpIHt9DQo+ICtz
dGF0aWMgaW5saW5lIHZvaWQgcnBjX2FkZF9pb3N0YXRzKHN0cnVjdCBycGNfaW9zdGF0cyAqdG90
YWwsDQo+ICsJCQkJICAgc3RydWN0IHJwY19pb3N0YXRzICpuZXcsDQo+ICsJCQkJICAgdW5zaWdu
ZWQgaW50IG1heHByb2MpIHt9DQo+ICBzdGF0aWMgaW5saW5lIHZvaWQgcnBjX2ZyZWVfaW9zdGF0
cyhzdHJ1Y3QgcnBjX2lvc3RhdHMgKnN0YXRzKSB7fQ0KPiAgDQo+ICAjZW5kaWYgIC8qICBDT05G
SUdfUFJPQ19GUyAgKi8NCj4gZGlmZiAtLWdpdCBhL25ldC9zdW5ycGMvc3RhdHMuYyBiL25ldC9z
dW5ycGMvc3RhdHMuYw0KPiBpbmRleCAzYzRmNjg4Li4zNDJkZjU3IDEwMDY0NA0KPiAtLS0gYS9u
ZXQvc3VucnBjL3N0YXRzLmMNCj4gKysrIGIvbmV0L3N1bnJwYy9zdGF0cy5jDQo+IEBAIC0xNzYs
MTIgKzE3NiwxNiBAQCBzdGF0aWMgdm9pZCBfcHJpbnRfbmFtZShzdHJ1Y3Qgc2VxX2ZpbGUgKnNl
cSwgdW5zaWduZWQgaW50IG9wLA0KPiAgCQlzZXFfcHJpbnRmKHNlcSwgIlx0JTEydTogIiwgb3Ap
Ow0KPiAgfQ0KPiAgDQo+IC12b2lkIHJwY19wcmludF9pb3N0YXRzKHN0cnVjdCBzZXFfZmlsZSAq
c2VxLCBzdHJ1Y3QgcnBjX2NsbnQgKmNsbnQpDQo+ICt2b2lkIHJwY19wcmludF9pb3N0YXRzKHN0
cnVjdCBzZXFfZmlsZSAqc2VxLCBzdHJ1Y3QgcnBjX2NsbnQgKmNsbnQsDQo+ICsJCSAgICAgICBz
dHJ1Y3QgcnBjX2lvc3RhdHMgKmFsdHN0YXRzLA0KPiArCQkgICAgICAgY29uc3QgY2hhciAqYWx0
c3RhdHNfbXNnKQ0KPiAgew0KPiAgCXN0cnVjdCBycGNfaW9zdGF0cyAqc3RhdHMgPSBjbG50LT5j
bF9tZXRyaWNzOw0KPiAgCXN0cnVjdCBycGNfeHBydCAqeHBydCA9IGNsbnQtPmNsX3hwcnQ7DQo+
ICAJdW5zaWduZWQgaW50IG9wLCBtYXhwcm9jID0gY2xudC0+Y2xfbWF4cHJvYzsNCj4gIA0KPiAr
CWlmIChhbHRzdGF0cykNCj4gKwkJc3RhdHMgPSBhbHRzdGF0czsNCj4gIAlpZiAoIXN0YXRzKQ0K
PiAgCQlyZXR1cm47DQo+ICANCj4gQEAgLTE5Miw3ICsxOTYsOCBAQCB2b2lkIHJwY19wcmludF9p
b3N0YXRzKHN0cnVjdCBzZXFfZmlsZSAqc2VxLCBzdHJ1Y3QgcnBjX2NsbnQgKmNsbnQpDQo+ICAJ
aWYgKHhwcnQpDQo+ICAJCXhwcnQtPm9wcy0+cHJpbnRfc3RhdHMoeHBydCwgc2VxKTsNCj4gIA0K
PiAtCXNlcV9wcmludGYoc2VxLCAiXHRwZXItb3Agc3RhdGlzdGljc1xuIik7DQo+ICsJc2VxX3By
aW50ZihzZXEsICJcdHBlci1vcCBzdGF0aXN0aWNzJXNcbiIsDQo+ICsJCSAgIChhbHRzdGF0cyAm
JiBhbHRzdGF0c19tc2cpID8gYWx0c3RhdHNfbXNnIDogIiIpOw0KPiAgCWZvciAob3AgPSAwOyBv
cCA8IG1heHByb2M7IG9wKyspIHsNCj4gIAkJc3RydWN0IHJwY19pb3N0YXRzICptZXRyaWNzID0g
JnN0YXRzW29wXTsNCj4gIAkJX3ByaW50X25hbWUoc2VxLCBvcCwgY2xudC0+Y2xfcHJvY2luZm8p
Ow0KPiBAQCAtMjA5LDYgKzIxNCwzNSBAQCB2b2lkIHJwY19wcmludF9pb3N0YXRzKHN0cnVjdCBz
ZXFfZmlsZSAqc2VxLCBzdHJ1Y3QgcnBjX2NsbnQgKmNsbnQpDQo+ICB9DQo+ICBFWFBPUlRfU1lN
Qk9MX0dQTChycGNfcHJpbnRfaW9zdGF0cyk7DQo+ICANCj4gK3ZvaWQNCj4gK3JwY19hZGRfaW9z
dGF0cyhzdHJ1Y3QgcnBjX2lvc3RhdHMgKnRvdGFscywgc3RydWN0IHJwY19pb3N0YXRzICpuZXcs
DQo+ICsJCXVuc2lnbmVkIGludCBtYXhwcm9jKQ0KPiArew0KPiArCXVuc2lnbmVkIGludCBvcDsN
Cj4gKw0KPiArCWZvciAob3AgPSAwOyBvcCA8IG1heHByb2M7IG9wKyspIHsNCj4gKwkJc3RydWN0
IHJwY19pb3N0YXRzICpvcF90b3RhbHMgPSAmdG90YWxzW29wXSwNCj4gKwkJCQkgICAqb3BfbmV3
ID0gJm5ld1tvcF07DQo+ICsNCj4gKwkJb3BfdG90YWxzLT5vbV9vcHMgKz0gb3BfbmV3LT5vbV9v
cHM7DQo+ICsJCW9wX3RvdGFscy0+b21fbnRyYW5zICs9IG9wX25ldy0+b21fbnRyYW5zOw0KPiAr
CQlvcF90b3RhbHMtPm9tX3RpbWVvdXRzICs9IG9wX25ldy0+b21fdGltZW91dHM7DQo+ICsNCj4g
KwkJb3BfdG90YWxzLT5vbV9ieXRlc19zZW50ICs9IG9wX25ldy0+b21fYnl0ZXNfc2VudDsNCj4g
KwkJb3BfdG90YWxzLT5vbV9ieXRlc19yZWN2ICs9IG9wX25ldy0+b21fYnl0ZXNfcmVjdjsNCj4g
Kw0KPiArCQlvcF90b3RhbHMtPm9tX3F1ZXVlID0ga3RpbWVfYWRkKG9wX3RvdGFscy0+b21fcXVl
dWUsDQo+ICsJCQkJCSAgICAgICBvcF9uZXctPm9tX3F1ZXVlKTsNCj4gKw0KPiArCQlvcF90b3Rh
bHMtPm9tX3J0dCA9IGt0aW1lX2FkZChvcF90b3RhbHMtPm9tX3J0dCwNCj4gKwkJCQkJICAgICAg
b3BfbmV3LT5vbV9ydHQpOw0KPiArDQo+ICsJCW9wX3RvdGFscy0+b21fZXhlY3V0ZSA9IGt0aW1l
X2FkZChvcF90b3RhbHMtPm9tX2V4ZWN1dGUsDQo+ICsJCQkJCQkgb3BfbmV3LT5vbV9leGVjdXRl
KTsNCj4gKwl9DQo+ICt9DQo+ICtFWFBPUlRfU1lNQk9MX0dQTChycGNfYWRkX2lvc3RhdHMpOw0K
PiArDQo+ICAvKg0KPiAgICogUmVnaXN0ZXIvdW5yZWdpc3RlciBSUEMgcHJvYyBmaWxlcw0KPiAg
ICovDQoNCi0tIA0KVHJvbmQgTXlrbGVidXN0DQpMaW51eCBORlMgY2xpZW50IG1haW50YWluZXIN
Cg0KTmV0QXBwDQpUcm9uZC5NeWtsZWJ1c3RAbmV0YXBwLmNvbQ0Kd3d3Lm5ldGFwcC5jb20NCg0K
On Feb 9, 2012, at 1:16 PM, Myklebust, Trond wrote:
> On Thu, 2012-02-09 at 11:41 -0500, Weston Andros Adamson wrote:
>> Include RPC statistics from all data servers in /proc/self/mountstats for pNFS
>> filelayout mounts.
>>
>> The additional info printed on the "per-op statistics" line does not break
>> current mountstats(8) from nfs-utils.
>>
>> Signed-off-by: Weston Andros Adamson <[email protected]>
>> ---
>>
>> This is clearly a better approach than I was taking trying to keep DS rpc stats
>> separate in /proc/self/mountstats.
>>
>> FYI, there are no other callers of rpc_print_iostats().
>>
>> The new output of /proc/self/mountstats works fine with current mountstats
>> userland tool. The additional data on the per-op statistics line doesn't make
>> it into the output, but I left it in /proc/self/mountstats because it doesn't
>> break anything and can be included in newer versions of mountstats.
>>
>> I plan on adding the per-DS stats (rpc and others) somewhere in /proc/fs/nfsfs/
>> soon.
>>
>> fs/nfs/nfs4filelayout.c | 30 ++++++++++++++++++++++++++
>> fs/nfs/pnfs.h | 23 ++++++++++++++++++++
>> fs/nfs/pnfs_dev.c | 45 ++++++++++++++++++++++++++++++++++++++++
>> fs/nfs/super.c | 3 +-
>> include/linux/sunrpc/metrics.h | 13 +++++++++-
>> net/sunrpc/stats.c | 38 ++++++++++++++++++++++++++++++++-
>> 6 files changed, 147 insertions(+), 5 deletions(-)
>>
>> diff --git a/fs/nfs/nfs4filelayout.c b/fs/nfs/nfs4filelayout.c
>> index 79be7ac..51c71e5 100644
>> --- a/fs/nfs/nfs4filelayout.c
>> +++ b/fs/nfs/nfs4filelayout.c
>> @@ -33,6 +33,8 @@
>> #include <linux/nfs_page.h>
>> #include <linux/module.h>
>>
>> +#include <linux/sunrpc/metrics.h>
>> +
>> #include "internal.h"
>> #include "nfs4filelayout.h"
>>
>> @@ -918,6 +920,33 @@ filelayout_free_deveiceid_node(struct nfs4_deviceid_node *d)
>> nfs4_fl_free_deviceid(container_of(d, struct nfs4_file_layout_dsaddr, id_node));
>> }
>>
>> +static unsigned int
>> +filelayout_rpc_add_iostats(struct rpc_iostats *total,
>> + unsigned int maxproc,
>> + struct nfs4_deviceid_node *d)
>> +{
>> + struct nfs4_file_layout_dsaddr *dsaddr;
>> + struct nfs4_pnfs_ds *ds;
>> + unsigned int num_ds = 0;
>> + u32 i;
>> +
>> + dsaddr = container_of(d, struct nfs4_file_layout_dsaddr, id_node);
>> +
>> + for (i = 0; i < dsaddr->ds_num; i++) {
>> + ds = dsaddr->ds_list[i];
>> +
>> + /* only add iostats if nfs_client is different from MDS */
>> + if (ds && ds->ds_clp && d->nfs_client != ds->ds_clp) {
>> + struct rpc_clnt *clnt = ds->ds_clp->cl_rpcclient;
>
> Hmmm.... Shouldn't this rather be the NFS_CLIENT(inode)? Also, why not
> do this in the read/write completion code? Why walk the deviceid list at
> all?
I'm not really following you. Note that these are rpc_iostats not nfs_iostats.
nfs_iostats have been properly counting operations to DSs for some time now.
rpc_iostats count operations, rtt, exectime, etc per rpc procedure (or in the case of NFSv4.X there is a hack to use operations within compounds).
We have to walk the device list to get access to the underlying DS's nfs_client->rpc_client -- I can't just put this in the read/write completion because, unlike nfs_iostats, rpc_iostats count all operations and not just read/write events/bytes (not to mention that the accumulation is done at the rpc layer).
I do have an implementation that doesn't need to walk the deviceid list by allowing a shared rpc_iostats struct between multiple rpc_clients (in addition to the current rpc_iostats structure), but that required adding locking and reference counting -- all for printing stats (obviously not what we want).
Does that make sense? If so, I'll repost with the fix from below.
Also, it just occurred to me that you might want me to separate this into two patches one for changes in net/sunrpc and one for fs/nfs...
-dros
>
>> + rpc_add_iostats(total, clnt->cl_metrics,
>> + min(maxproc, clnt->cl_maxproc));
>> + num_ds++;
>> + }
>> + }
>> +
>> + return num_ds;
>> +}
>> +
>> static struct pnfs_layoutdriver_type filelayout_type = {
>> .id = LAYOUT_NFSV4_1_FILES,
>> .name = "LAYOUT_NFSV4_1_FILES",
>> @@ -932,6 +961,7 @@ static struct pnfs_layoutdriver_type filelayout_type = {
>> .read_pagelist = filelayout_read_pagelist,
>> .write_pagelist = filelayout_write_pagelist,
>> .free_deviceid_node = filelayout_free_deveiceid_node,
>> + .ds_rpc_add_iostats = filelayout_rpc_add_iostats,
>> };
>>
>> static int __init nfs4filelayout_init(void)
>> diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
>> index 53d593a..d1b82eb 100644
>> --- a/fs/nfs/pnfs.h
>> +++ b/fs/nfs/pnfs.h
>> @@ -72,6 +72,7 @@ enum layoutdriver_policy_flags {
>> };
>>
>> struct nfs4_deviceid_node;
>> +struct rpc_iostats;
>>
>> /* Per-layout driver specific registration structure */
>> struct pnfs_layoutdriver_type {
>> @@ -119,6 +120,9 @@ struct pnfs_layoutdriver_type {
>> void (*encode_layoutcommit) (struct pnfs_layout_hdr *layoutid,
>> struct xdr_stream *xdr,
>> const struct nfs4_layoutcommit_args *args);
>> +
>> + unsigned int (*ds_rpc_add_iostats) (struct rpc_iostats *, unsigned int,
>> + struct nfs4_deviceid_node *);
>> };
>>
>> struct pnfs_layout_hdr {
>> @@ -239,6 +243,10 @@ void nfs4_init_deviceid_node(struct nfs4_deviceid_node *,
>> struct nfs4_deviceid_node *nfs4_insert_deviceid_node(struct nfs4_deviceid_node *);
>> bool nfs4_put_deviceid_node(struct nfs4_deviceid_node *);
>> void nfs4_deviceid_purge_client(const struct nfs_client *);
>> +void nfs4_deviceid_rpc_print_iostats(struct seq_file *seq,
>> + const struct nfs_client *clp,
>> + const struct pnfs_layoutdriver_type *ld);
>> +
>>
>> static inline int lo_fail_bit(u32 iomode)
>> {
>> @@ -328,6 +336,15 @@ static inline int pnfs_return_layout(struct inode *ino)
>> return 0;
>> }
>>
>> +static inline int
>> +pnfs_rpc_print_iostats(struct seq_file *m, struct nfs_server *nfss)
>> +{
>> + if (!pnfs_enabled_sb(nfss))
>> + return 0;
>> + nfs4_deviceid_rpc_print_iostats(m, nfss->nfs_client,
>> + nfss->pnfs_curr_ld);
>> + return 1;
>> +}
>> #else /* CONFIG_NFS_V4_1 */
>>
>> static inline void pnfs_destroy_all_layouts(struct nfs_client *clp)
>> @@ -429,6 +446,12 @@ static inline int pnfs_layoutcommit_inode(struct inode *inode, bool sync)
>> static inline void nfs4_deviceid_purge_client(struct nfs_client *ncl)
>> {
>> }
>> +
>> +static inline void
>> +pnfs_rpc_print_iostats(struct seq_file *m, struct nfs_server *nfss)
>> +{
>> + return 0;
>> +}
>> #endif /* CONFIG_NFS_V4_1 */
>>
>> #endif /* FS_NFS_PNFS_H */
>> diff --git a/fs/nfs/pnfs_dev.c b/fs/nfs/pnfs_dev.c
>> index 4f359d2..aa5d102 100644
>> --- a/fs/nfs/pnfs_dev.c
>> +++ b/fs/nfs/pnfs_dev.c
>> @@ -29,6 +29,9 @@
>> */
>>
>> #include <linux/export.h>
>> +
>> +#include <linux/sunrpc/metrics.h>
>> +
>> #include "pnfs.h"
>>
>> #define NFSDBG_FACILITY NFSDBG_PNFS
>> @@ -115,6 +118,48 @@ nfs4_find_get_deviceid(const struct pnfs_layoutdriver_type *ld,
>> }
>> EXPORT_SYMBOL_GPL(nfs4_find_get_deviceid);
>>
>> +
>> +void
>> +nfs4_deviceid_rpc_print_iostats(struct seq_file *seq,
>> + const struct nfs_client *clp,
>> + const struct pnfs_layoutdriver_type *ld)
>> +{
>> + struct nfs4_deviceid_node *d;
>> + struct hlist_node *n;
>> + struct rpc_iostats *totals;
>> + char msgbuf[64];
>> + unsigned int num_ds = 0;
>> + unsigned int maxproc;
>> + long h;
>> +
>> + if (!ld->ds_rpc_add_iostats)
>> + return;
>> + totals = rpc_alloc_iostats(clp->cl_rpcclient);
>> + if (!totals)
>> + return;
>> + maxproc = clp->cl_rpcclient->cl_maxproc;
>> + rpc_add_iostats(totals, clp->cl_rpcclient->cl_metrics, maxproc);
>> +
>> + rcu_read_lock();
>> + for (h = 0; h < NFS4_DEVICE_ID_HASH_SIZE; h++) {
>> + hlist_for_each_entry_rcu(d, n, &nfs4_deviceid_cache[h], node) {
>> + if (d->ld == ld && d->nfs_client == clp) {
>> + if (atomic_read(&d->ref))
>> + num_ds += ld->ds_rpc_add_iostats(totals,
>> + maxproc, d);
>> + }
>> + }
>> + }
>> + rcu_read_unlock();
>> +
>> + snprintf(msgbuf, sizeof(msgbuf),
>> + " for the metadata server and %u data server%s", num_ds,
>> + (num_ds != 1) ? "s" : "");
>> + rpc_print_iostats(seq, clp->cl_rpcclient, totals, msgbuf);
>> + rpc_free_iostats(totals);
>> +}
>> +EXPORT_SYMBOL_GPL(nfs4_deviceid_rpc_print_iostats);
>
> Is this needed? AFAICS it is used in fs/nfs/super.c, which is in the
> same module (i.e. nfs.ko).
oops!
>
>> /*
>> * Remove a deviceid from cache
>> *
>> diff --git a/fs/nfs/super.c b/fs/nfs/super.c
>> index d18a90b..76d0850 100644
>> --- a/fs/nfs/super.c
>> +++ b/fs/nfs/super.c
>> @@ -870,7 +870,8 @@ static int nfs_show_stats(struct seq_file *m, struct dentry *root)
>> #endif
>> seq_printf(m, "\n");
>>
>> - rpc_print_iostats(m, nfss->client);
>> + if (!pnfs_rpc_print_iostats(m, nfss))
>> + rpc_print_iostats(m, nfss->client, NULL, NULL);
>>
>> return 0;
>> }
>> diff --git a/include/linux/sunrpc/metrics.h b/include/linux/sunrpc/metrics.h
>> index b6edbc0..e6e04e4 100644
>> --- a/include/linux/sunrpc/metrics.h
>> +++ b/include/linux/sunrpc/metrics.h
>> @@ -75,14 +75,23 @@ struct rpc_clnt;
>>
>> struct rpc_iostats * rpc_alloc_iostats(struct rpc_clnt *);
>> void rpc_count_iostats(struct rpc_task *);
>> -void rpc_print_iostats(struct seq_file *, struct rpc_clnt *);
>> +void rpc_print_iostats(struct seq_file *,
>> + struct rpc_clnt *,
>> + struct rpc_iostats *,
>> + const char *);
>> +void rpc_add_iostats(struct rpc_iostats *,
>> + struct rpc_iostats *, unsigned int);
>> void rpc_free_iostats(struct rpc_iostats *);
>>
>> #else /* CONFIG_PROC_FS */
>>
>> static inline struct rpc_iostats *rpc_alloc_iostats(struct rpc_clnt *clnt) { return NULL; }
>> static inline void rpc_count_iostats(struct rpc_task *task) {}
>> -static inline void rpc_print_iostats(struct seq_file *seq, struct rpc_clnt *clnt) {}
>> +static inline void rpc_print_iostats(struct seq_file *seq,
>> + struct rpc_iostats *stat) {}
>> +static inline void rpc_add_iostats(struct rpc_iostats *total,
>> + struct rpc_iostats *new,
>> + unsigned int maxproc) {}
>> static inline void rpc_free_iostats(struct rpc_iostats *stats) {}
>>
>> #endif /* CONFIG_PROC_FS */
>> diff --git a/net/sunrpc/stats.c b/net/sunrpc/stats.c
>> index 3c4f688..342df57 100644
>> --- a/net/sunrpc/stats.c
>> +++ b/net/sunrpc/stats.c
>> @@ -176,12 +176,16 @@ static void _print_name(struct seq_file *seq, unsigned int op,
>> seq_printf(seq, "\t%12u: ", op);
>> }
>>
>> -void rpc_print_iostats(struct seq_file *seq, struct rpc_clnt *clnt)
>> +void rpc_print_iostats(struct seq_file *seq, struct rpc_clnt *clnt,
>> + struct rpc_iostats *altstats,
>> + const char *altstats_msg)
>> {
>> struct rpc_iostats *stats = clnt->cl_metrics;
>> struct rpc_xprt *xprt = clnt->cl_xprt;
>> unsigned int op, maxproc = clnt->cl_maxproc;
>>
>> + if (altstats)
>> + stats = altstats;
>> if (!stats)
>> return;
>>
>> @@ -192,7 +196,8 @@ void rpc_print_iostats(struct seq_file *seq, struct rpc_clnt *clnt)
>> if (xprt)
>> xprt->ops->print_stats(xprt, seq);
>>
>> - seq_printf(seq, "\tper-op statistics\n");
>> + seq_printf(seq, "\tper-op statistics%s\n",
>> + (altstats && altstats_msg) ? altstats_msg : "");
>> for (op = 0; op < maxproc; op++) {
>> struct rpc_iostats *metrics = &stats[op];
>> _print_name(seq, op, clnt->cl_procinfo);
>> @@ -209,6 +214,35 @@ void rpc_print_iostats(struct seq_file *seq, struct rpc_clnt *clnt)
>> }
>> EXPORT_SYMBOL_GPL(rpc_print_iostats);
>>
>> +void
>> +rpc_add_iostats(struct rpc_iostats *totals, struct rpc_iostats *new,
>> + unsigned int maxproc)
>> +{
>> + unsigned int op;
>> +
>> + for (op = 0; op < maxproc; op++) {
>> + struct rpc_iostats *op_totals = &totals[op],
>> + *op_new = &new[op];
>> +
>> + op_totals->om_ops += op_new->om_ops;
>> + op_totals->om_ntrans += op_new->om_ntrans;
>> + op_totals->om_timeouts += op_new->om_timeouts;
>> +
>> + op_totals->om_bytes_sent += op_new->om_bytes_sent;
>> + op_totals->om_bytes_recv += op_new->om_bytes_recv;
>> +
>> + op_totals->om_queue = ktime_add(op_totals->om_queue,
>> + op_new->om_queue);
>> +
>> + op_totals->om_rtt = ktime_add(op_totals->om_rtt,
>> + op_new->om_rtt);
>> +
>> + op_totals->om_execute = ktime_add(op_totals->om_execute,
>> + op_new->om_execute);
>> + }
>> +}
>> +EXPORT_SYMBOL_GPL(rpc_add_iostats);
>> +
>> /*
>> * Register/unregister RPC proc files
>> */
>
> --
> Trond Myklebust
> Linux NFS client maintainer
>
> NetApp
> [email protected]
> http://www.netapp.com
>
T24gVGh1LCAyMDEyLTAyLTA5IGF0IDE5OjEzICswMDAwLCBBZGFtc29uLCBEcm9zIHdyb3RlOg0K
PiBPbiBGZWIgOSwgMjAxMiwgYXQgMToxNiBQTSwgTXlrbGVidXN0LCBUcm9uZCB3cm90ZToNCj4g
DQo+ID4gT24gVGh1LCAyMDEyLTAyLTA5IGF0IDExOjQxIC0wNTAwLCBXZXN0b24gQW5kcm9zIEFk
YW1zb24gd3JvdGU6DQo+ID4+IEluY2x1ZGUgUlBDIHN0YXRpc3RpY3MgZnJvbSBhbGwgZGF0YSBz
ZXJ2ZXJzIGluIC9wcm9jL3NlbGYvbW91bnRzdGF0cyBmb3IgcE5GUw0KPiA+PiBmaWxlbGF5b3V0
IG1vdW50cy4NCj4gPj4gDQo+ID4+IFRoZSBhZGRpdGlvbmFsIGluZm8gcHJpbnRlZCBvbiB0aGUg
InBlci1vcCBzdGF0aXN0aWNzIiBsaW5lIGRvZXMgbm90IGJyZWFrDQo+ID4+IGN1cnJlbnQgbW91
bnRzdGF0cyg4KSBmcm9tIG5mcy11dGlscy4NCj4gPj4gDQo+ID4+IFNpZ25lZC1vZmYtYnk6IFdl
c3RvbiBBbmRyb3MgQWRhbXNvbiA8ZHJvc0BuZXRhcHAuY29tPg0KPiA+PiAtLS0NCj4gPj4gDQo+
ID4+IFRoaXMgaXMgY2xlYXJseSBhIGJldHRlciBhcHByb2FjaCB0aGFuIEkgd2FzIHRha2luZyB0
cnlpbmcgdG8ga2VlcCBEUyBycGMgc3RhdHMNCj4gPj4gc2VwYXJhdGUgaW4gL3Byb2Mvc2VsZi9t
b3VudHN0YXRzLg0KPiA+PiANCj4gPj4gRllJLCB0aGVyZSBhcmUgbm8gb3RoZXIgY2FsbGVycyBv
ZiBycGNfcHJpbnRfaW9zdGF0cygpLg0KPiA+PiANCj4gPj4gVGhlIG5ldyBvdXRwdXQgb2YgL3By
b2Mvc2VsZi9tb3VudHN0YXRzIHdvcmtzIGZpbmUgd2l0aCBjdXJyZW50IG1vdW50c3RhdHMgDQo+
ID4+IHVzZXJsYW5kIHRvb2wuICBUaGUgYWRkaXRpb25hbCBkYXRhIG9uIHRoZSBwZXItb3Agc3Rh
dGlzdGljcyBsaW5lIGRvZXNuJ3QgbWFrZQ0KPiA+PiBpdCBpbnRvIHRoZSBvdXRwdXQsIGJ1dCBJ
IGxlZnQgaXQgaW4gL3Byb2Mvc2VsZi9tb3VudHN0YXRzIGJlY2F1c2UgaXQgZG9lc24ndA0KPiA+
PiBicmVhayBhbnl0aGluZyBhbmQgY2FuIGJlIGluY2x1ZGVkIGluIG5ld2VyIHZlcnNpb25zIG9m
IG1vdW50c3RhdHMuDQo+ID4+IA0KPiA+PiBJIHBsYW4gb24gYWRkaW5nIHRoZSBwZXItRFMgc3Rh
dHMgKHJwYyBhbmQgb3RoZXJzKSBzb21ld2hlcmUgaW4gL3Byb2MvZnMvbmZzZnMvDQo+ID4+IHNv
b24uDQo+ID4+IA0KPiA+PiBmcy9uZnMvbmZzNGZpbGVsYXlvdXQuYyAgICAgICAgfCAgIDMwICsr
KysrKysrKysrKysrKysrKysrKysrKysrDQo+ID4+IGZzL25mcy9wbmZzLmggICAgICAgICAgICAg
ICAgICB8ICAgMjMgKysrKysrKysrKysrKysrKysrKysNCj4gPj4gZnMvbmZzL3BuZnNfZGV2LmMg
ICAgICAgICAgICAgIHwgICA0NSArKysrKysrKysrKysrKysrKysrKysrKysrKysrKysrKysrKysr
KysrDQo+ID4+IGZzL25mcy9zdXBlci5jICAgICAgICAgICAgICAgICB8ICAgIDMgKy0NCj4gPj4g
aW5jbHVkZS9saW51eC9zdW5ycGMvbWV0cmljcy5oIHwgICAxMyArKysrKysrKystDQo+ID4+IG5l
dC9zdW5ycGMvc3RhdHMuYyAgICAgICAgICAgICB8ICAgMzggKysrKysrKysrKysrKysrKysrKysr
KysrKysrKysrKystDQo+ID4+IDYgZmlsZXMgY2hhbmdlZCwgMTQ3IGluc2VydGlvbnMoKyksIDUg
ZGVsZXRpb25zKC0pDQo+ID4+IA0KPiA+PiBkaWZmIC0tZ2l0IGEvZnMvbmZzL25mczRmaWxlbGF5
b3V0LmMgYi9mcy9uZnMvbmZzNGZpbGVsYXlvdXQuYw0KPiA+PiBpbmRleCA3OWJlN2FjLi41MWM3
MWU1IDEwMDY0NA0KPiA+PiAtLS0gYS9mcy9uZnMvbmZzNGZpbGVsYXlvdXQuYw0KPiA+PiArKysg
Yi9mcy9uZnMvbmZzNGZpbGVsYXlvdXQuYw0KPiA+PiBAQCAtMzMsNiArMzMsOCBAQA0KPiA+PiAj
aW5jbHVkZSA8bGludXgvbmZzX3BhZ2UuaD4NCj4gPj4gI2luY2x1ZGUgPGxpbnV4L21vZHVsZS5o
Pg0KPiA+PiANCj4gPj4gKyNpbmNsdWRlIDxsaW51eC9zdW5ycGMvbWV0cmljcy5oPg0KPiA+PiAr
DQo+ID4+ICNpbmNsdWRlICJpbnRlcm5hbC5oIg0KPiA+PiAjaW5jbHVkZSAibmZzNGZpbGVsYXlv
dXQuaCINCj4gPj4gDQo+ID4+IEBAIC05MTgsNiArOTIwLDMzIEBAIGZpbGVsYXlvdXRfZnJlZV9k
ZXZlaWNlaWRfbm9kZShzdHJ1Y3QgbmZzNF9kZXZpY2VpZF9ub2RlICpkKQ0KPiA+PiAJbmZzNF9m
bF9mcmVlX2RldmljZWlkKGNvbnRhaW5lcl9vZihkLCBzdHJ1Y3QgbmZzNF9maWxlX2xheW91dF9k
c2FkZHIsIGlkX25vZGUpKTsNCj4gPj4gfQ0KPiA+PiANCj4gPj4gK3N0YXRpYyB1bnNpZ25lZCBp
bnQNCj4gPj4gK2ZpbGVsYXlvdXRfcnBjX2FkZF9pb3N0YXRzKHN0cnVjdCBycGNfaW9zdGF0cyAq
dG90YWwsDQo+ID4+ICsJCQkgICB1bnNpZ25lZCBpbnQgbWF4cHJvYywNCj4gPj4gKwkJCSAgIHN0
cnVjdCBuZnM0X2RldmljZWlkX25vZGUgKmQpDQo+ID4+ICt7DQo+ID4+ICsJc3RydWN0IG5mczRf
ZmlsZV9sYXlvdXRfZHNhZGRyICpkc2FkZHI7DQo+ID4+ICsJc3RydWN0IG5mczRfcG5mc19kcyAq
ZHM7DQo+ID4+ICsJdW5zaWduZWQgaW50IG51bV9kcyA9IDA7DQo+ID4+ICsJdTMyIGk7DQo+ID4+
ICsNCj4gPj4gKwlkc2FkZHIgPSBjb250YWluZXJfb2YoZCwgc3RydWN0IG5mczRfZmlsZV9sYXlv
dXRfZHNhZGRyLCBpZF9ub2RlKTsNCj4gPj4gKw0KPiA+PiArCWZvciAoaSA9IDA7IGkgPCBkc2Fk
ZHItPmRzX251bTsgaSsrKSB7DQo+ID4+ICsJCWRzID0gZHNhZGRyLT5kc19saXN0W2ldOw0KPiA+
PiArDQo+ID4+ICsJCS8qIG9ubHkgYWRkIGlvc3RhdHMgaWYgbmZzX2NsaWVudCBpcyBkaWZmZXJl
bnQgZnJvbSBNRFMgKi8NCj4gPj4gKwkJaWYgKGRzICYmIGRzLT5kc19jbHAgJiYgZC0+bmZzX2Ns
aWVudCAhPSBkcy0+ZHNfY2xwKSB7DQo+ID4+ICsJCQlzdHJ1Y3QgcnBjX2NsbnQgKmNsbnQgPSBk
cy0+ZHNfY2xwLT5jbF9ycGNjbGllbnQ7DQo+ID4gDQo+ID4gSG1tbS4uLi4gU2hvdWxkbid0IHRo
aXMgcmF0aGVyIGJlIHRoZSBORlNfQ0xJRU5UKGlub2RlKT8gQWxzbywgd2h5IG5vdA0KPiA+IGRv
IHRoaXMgaW4gdGhlIHJlYWQvd3JpdGUgY29tcGxldGlvbiBjb2RlPyBXaHkgd2FsayB0aGUgZGV2
aWNlaWQgbGlzdCBhdA0KPiA+IGFsbD8NCj4gDQo+IEknbSBub3QgcmVhbGx5IGZvbGxvd2luZyB5
b3UuICBOb3RlIHRoYXQgdGhlc2UgYXJlIHJwY19pb3N0YXRzIG5vdCBuZnNfaW9zdGF0cy4gIA0K
PiBuZnNfaW9zdGF0cyBoYXZlIGJlZW4gcHJvcGVybHkgY291bnRpbmcgb3BlcmF0aW9ucyB0byBE
U3MgZm9yIHNvbWUgdGltZSBub3cuDQo+IHJwY19pb3N0YXRzIGNvdW50IG9wZXJhdGlvbnMsIHJ0
dCwgZXhlY3RpbWUsIGV0YyBwZXIgcnBjIHByb2NlZHVyZSAob3IgaW4gdGhlIGNhc2Ugb2YgTkZT
djQuWCB0aGVyZSBpcyBhIGhhY2sgdG8gdXNlIG9wZXJhdGlvbnMgd2l0aGluIGNvbXBvdW5kcyku
DQo+IA0KPiBXZSBoYXZlIHRvIHdhbGsgdGhlIGRldmljZSBsaXN0IHRvIGdldCBhY2Nlc3MgdG8g
dGhlIHVuZGVybHlpbmcgRFMncyBuZnNfY2xpZW50LT5ycGNfY2xpZW50IC0tIEkgY2FuJ3QganVz
dCBwdXQgdGhpcyBpbiB0aGUgcmVhZC93cml0ZSBjb21wbGV0aW9uIGJlY2F1c2UsIHVubGlrZSBu
ZnNfaW9zdGF0cywgcnBjX2lvc3RhdHMgY291bnQgYWxsIG9wZXJhdGlvbnMgYW5kIG5vdCBqdXN0
IHJlYWQvd3JpdGUgZXZlbnRzL2J5dGVzIChub3QgdG8gbWVudGlvbiB0aGF0IHRoZSBhY2N1bXVs
YXRpb24gaXMgZG9uZSBhdCB0aGUgcnBjIGxheWVyKS4NCg0KSW4gZG9pbmcgdGhpcywgeW91IGFy
ZSBhc3N1bWluZyB0aGF0IERTZXMgYXJlIG5vdCBzaGFyZWQgYW1vbmcgbXVsdGlwbGUNCmZpbGVz
eXN0ZW1zIG9yIE1EU2VzIChpLmUuIHRoYXQgZWFjaCBEUyBpcyB1c2VkIGJ5IGEgc2luZ2xlIHN0
cnVjdA0KbmZzX3NlcnZlcikuIEZvciBzb21ldGhpbmcgbGlrZSBhIE5ldEFwcCBmaWxlciBpbiBj
LW1vZGUsIHRoYXQgaXMNCmNsZWFybHkgbm90IGEgdmFsaWQgYXNzdW1wdGlvbi4NCg0KPiBJIGRv
IGhhdmUgYW4gaW1wbGVtZW50YXRpb24gdGhhdCBkb2Vzbid0IG5lZWQgdG8gd2FsayB0aGUgZGV2
aWNlaWQgbGlzdCBieSBhbGxvd2luZyBhIHNoYXJlZCBycGNfaW9zdGF0cyBzdHJ1Y3QgYmV0d2Vl
biBtdWx0aXBsZSBycGNfY2xpZW50cyAoaW4gYWRkaXRpb24gdG8gdGhlIGN1cnJlbnQgcnBjX2lv
c3RhdHMgc3RydWN0dXJlKSwgYnV0IHRoYXQgcmVxdWlyZWQgYWRkaW5nIGxvY2tpbmcgYW5kIHJl
ZmVyZW5jZSBjb3VudGluZyAtLSBhbGwgZm9yIHByaW50aW5nIHN0YXRzIChvYnZpb3VzbHkgbm90
IHdoYXQgd2Ugd2FudCkuDQoNClRoaXMgbWlnaHQgYmUgbW9yZSBpbiBsaW5lIHdpdGggd2hhdCB3
ZSB3YW50LiBOb3RlIHRoYXQgaXQgc2hvdWxkIGJlDQplYXN5IHRvIGNsb25lIGFuIHJwY19jbGll
bnQgYW5kIHRoZW4gcmVwbGFjZSBpdHMgcnBjX2lvc3RhdHMgc3RydWN0LiBJDQpkb24ndCB0aGlu
ayB0aGF0IG5lZWRzIGFueSBleHRyYSBsb2NraW5nLiBXZSdyZSBhbHJlYWR5IGlnbm9yaW5nIGxv
Y2tpbmcNCmhlcmUgYmV0d2VlbiBkaWZmZXJlbnQgcnBjX3Rhc2tzLCBzbyB0aHJvd2luZyBpbiBk
aWZmZXJlbnQgcnBjX2NsaWVudHMNCnRvIHRoZSBtaXggd2lsbCBtYWtlIG5vIGRpZmZlcmVuY2Uu
DQoNCg0KLS0gDQpUcm9uZCBNeWtsZWJ1c3QNCkxpbnV4IE5GUyBjbGllbnQgbWFpbnRhaW5lcg0K
DQpOZXRBcHANClRyb25kLk15a2xlYnVzdEBuZXRhcHAuY29tDQp3d3cubmV0YXBwLmNvbQ0KDQo=
On Feb 9, 2012, at 2:27 PM, Myklebust, Trond wrote:
> On Thu, 2012-02-09 at 19:13 +0000, Adamson, Dros wrote:
>> On Feb 9, 2012, at 1:16 PM, Myklebust, Trond wrote:
>>
>>> On Thu, 2012-02-09 at 11:41 -0500, Weston Andros Adamson wrote:
>>>> Include RPC statistics from all data servers in /proc/self/mountstats for pNFS
>>>> filelayout mounts.
>>>>
>>>> The additional info printed on the "per-op statistics" line does not break
>>>> current mountstats(8) from nfs-utils.
>>>>
>>>> Signed-off-by: Weston Andros Adamson <[email protected]>
>>>> ---
>>>>
>>>> This is clearly a better approach than I was taking trying to keep DS rpc stats
>>>> separate in /proc/self/mountstats.
>>>>
>>>> FYI, there are no other callers of rpc_print_iostats().
>>>>
>>>> The new output of /proc/self/mountstats works fine with current mountstats
>>>> userland tool. The additional data on the per-op statistics line doesn't make
>>>> it into the output, but I left it in /proc/self/mountstats because it doesn't
>>>> break anything and can be included in newer versions of mountstats.
>>>>
>>>> I plan on adding the per-DS stats (rpc and others) somewhere in /proc/fs/nfsfs/
>>>> soon.
>>>>
>>>> fs/nfs/nfs4filelayout.c | 30 ++++++++++++++++++++++++++
>>>> fs/nfs/pnfs.h | 23 ++++++++++++++++++++
>>>> fs/nfs/pnfs_dev.c | 45 ++++++++++++++++++++++++++++++++++++++++
>>>> fs/nfs/super.c | 3 +-
>>>> include/linux/sunrpc/metrics.h | 13 +++++++++-
>>>> net/sunrpc/stats.c | 38 ++++++++++++++++++++++++++++++++-
>>>> 6 files changed, 147 insertions(+), 5 deletions(-)
>>>>
>>>> diff --git a/fs/nfs/nfs4filelayout.c b/fs/nfs/nfs4filelayout.c
>>>> index 79be7ac..51c71e5 100644
>>>> --- a/fs/nfs/nfs4filelayout.c
>>>> +++ b/fs/nfs/nfs4filelayout.c
>>>> @@ -33,6 +33,8 @@
>>>> #include <linux/nfs_page.h>
>>>> #include <linux/module.h>
>>>>
>>>> +#include <linux/sunrpc/metrics.h>
>>>> +
>>>> #include "internal.h"
>>>> #include "nfs4filelayout.h"
>>>>
>>>> @@ -918,6 +920,33 @@ filelayout_free_deveiceid_node(struct nfs4_deviceid_node *d)
>>>> nfs4_fl_free_deviceid(container_of(d, struct nfs4_file_layout_dsaddr, id_node));
>>>> }
>>>>
>>>> +static unsigned int
>>>> +filelayout_rpc_add_iostats(struct rpc_iostats *total,
>>>> + unsigned int maxproc,
>>>> + struct nfs4_deviceid_node *d)
>>>> +{
>>>> + struct nfs4_file_layout_dsaddr *dsaddr;
>>>> + struct nfs4_pnfs_ds *ds;
>>>> + unsigned int num_ds = 0;
>>>> + u32 i;
>>>> +
>>>> + dsaddr = container_of(d, struct nfs4_file_layout_dsaddr, id_node);
>>>> +
>>>> + for (i = 0; i < dsaddr->ds_num; i++) {
>>>> + ds = dsaddr->ds_list[i];
>>>> +
>>>> + /* only add iostats if nfs_client is different from MDS */
>>>> + if (ds && ds->ds_clp && d->nfs_client != ds->ds_clp) {
>>>> + struct rpc_clnt *clnt = ds->ds_clp->cl_rpcclient;
>>>
>>> Hmmm.... Shouldn't this rather be the NFS_CLIENT(inode)? Also, why not
>>> do this in the read/write completion code? Why walk the deviceid list at
>>> all?
>>
>> I'm not really following you. Note that these are rpc_iostats not nfs_iostats.
>> nfs_iostats have been properly counting operations to DSs for some time now.
>> rpc_iostats count operations, rtt, exectime, etc per rpc procedure (or in the case of NFSv4.X there is a hack to use operations within compounds).
>>
>> We have to walk the device list to get access to the underlying DS's nfs_client->rpc_client -- I can't just put this in the read/write completion because, unlike nfs_iostats, rpc_iostats count all operations and not just read/write events/bytes (not to mention that the accumulation is done at the rpc layer).
>
> In doing this, you are assuming that DSes are not shared among multiple
> filesystems or MDSes (i.e. that each DS is used by a single struct
> nfs_server). For something like a NetApp filer in c-mode, that is
> clearly not a valid assumption.
True - I had thought of this, but I thought it was fine since I was just carrying this assumption forward...
The current mountstats code already prints nfs_server->nfs_client->rpc_client->rpc_iostats -- which clearly doesn't differentiate between multiple nfs_servers that could be sharing the nfs_client.
>
>> I do have an implementation that doesn't need to walk the deviceid list by allowing a shared rpc_iostats struct between multiple rpc_clients (in addition to the current rpc_iostats structure), but that required adding locking and reference counting -- all for printing stats (obviously not what we want).
>
> This might be more in line with what we want. Note that it should be
> easy to clone an rpc_client and then replace its rpc_iostats struct. I
> don't think that needs any extra locking. We're already ignoring locking
> here between different rpc_tasks, so throwing in different rpc_clients
> to the mix will make no difference.
Yeah, that's easy enough and i guess we could ignore locking, but we are still left with the same problem: how is this supposed to account for different mount points using the same nfs_client? nfs_client only has one rpc_client member. I doubt we want to make a hash lookup on nfs_server to get the right rpc_client (which could all use the same underlying xprt).
Maybe it's time to move these stats into fs/nfs/ where they really belong? We could get rid of the hack that overloads procnum with opnum from inside the compound for v4+ and finally only show a specific mount's RPC stats.
Thoughts?
-dros