Split the svc_stat structure pointed to by svc_serv.sv_stats into
a structure per-cpu, and aggregate all the per-cpu structures just
before emitting them to userspace via /proc. This avoids bouncing
cachelines when updating stats. See next patch for performance
numbers.
Signed-off-by: Greg Banks <[email protected]>
Reviewed-by: David Chinner <[email protected]>
Reviewed-by: Peter Leckie <pleckie-cP1dWloDopni96+mSzHFpQC/[email protected]>
---
include/linux/sunrpc/svc.h | 6 +++---
net/sunrpc/stats.c | 34 +++++++++++++++++++++++-----------
net/sunrpc/svc.c | 8 ++++----
3 files changed, 30 insertions(+), 18 deletions(-)
Index: bfields/include/linux/sunrpc/svc.h
===================================================================
--- bfields.orig/include/linux/sunrpc/svc.h
+++ bfields/include/linux/sunrpc/svc.h
@@ -76,7 +76,7 @@ struct svc_pool {
*/
struct svc_serv {
struct svc_program * sv_program; /* RPC program */
- struct svc_stat * sv_stats; /* RPC statistics */
+ struct svc_stat * sv_stats_percpu;/* RPC statistics */
spinlock_t sv_lock;
unsigned int sv_nrthreads; /* # of server threads */
unsigned int sv_maxconn; /* max connections allowed or
@@ -110,8 +110,8 @@ struct svc_serv {
#endif /* CONFIG_NFSD_V4_1 */
};
#define SVC_INC_STAT(serv, field) \
- ((serv)->sv_stats ? \
- ++((serv)->sv_stats->field) : 0)
+ ((serv)->sv_stats_percpu ? \
+ ++(per_cpu_ptr((serv)->sv_stats_percpu, smp_processor_id())->field) : 0)
/*
* We use sv_nrthreads as a reference count. svc_destroy() drops
Index: bfields/net/sunrpc/stats.c
===================================================================
--- bfields.orig/net/sunrpc/stats.c
+++ bfields/net/sunrpc/stats.c
@@ -75,17 +75,48 @@ static const struct file_operations rpc_
};
/*
+ * Accumulate all the per-cpu struct svc_stat
+ * into one global total for emission to userspace.
+ * Relies on struct svc_stat being composed of
+ * unsigned ints without gaps, so it can be treated
+ * as an array of unsigned ints.
+ *
+ * Note: we iterate over all possible CPUs instead
+ * of just the online ones to avoid counters going
+ * backwards when CPUs go offline.
+ */
+static void svc_stat_accum(const struct svc_serv *serv,
+ struct svc_stat *sp)
+{
+ unsigned int *usp = (unsigned int *)sp;
+ int cpu;
+ int i;
+
+ memset(sp, 0, sizeof(*sp));
+ for_each_possible_cpu(cpu) {
+ unsigned int *ucsp = (unsigned int *)
+ per_cpu_ptr(serv->sv_stats_percpu, cpu);
+ for (i = 0 ; i < sizeof(*sp)/sizeof(unsigned int) ; i++)
+ usp[i] += ucsp[i];
+ }
+}
+
+
+/*
* Get RPC server stats
*/
void svc_seq_show(struct seq_file *seq, const struct svc_serv *serv)
{
/* TODO: report call counts from the non-primary programs */
const struct svc_program *prog = serv->sv_program;
- struct svc_stat *statp = serv->sv_stats;
+ struct svc_stat accum;
+ struct svc_stat *statp = &accum;
const struct svc_procedure *proc;
const struct svc_version *vers;
unsigned int i, j;
+ svc_stat_accum(serv, &accum);
+
seq_printf(seq,
"net %u %u %u %u\n",
statp->netcnt,
Index: bfields/net/sunrpc/svc.c
===================================================================
--- bfields.orig/net/sunrpc/svc.c
+++ bfields/net/sunrpc/svc.c
@@ -395,8 +395,9 @@ __svc_create(struct svc_program *prog, u
init_timer(&serv->sv_temptimer);
spin_lock_init(&serv->sv_lock);
- serv->sv_stats = kzalloc(sizeof(struct svc_stat), GFP_KERNEL);
- if (!serv->sv_stats) {
+ serv->sv_stats_percpu = __alloc_percpu(sizeof(struct svc_stat),
+ __alignof__(struct svc_stat));
+ if (!serv->sv_stats_percpu) {
kfree(serv);
return NULL;
}
@@ -406,7 +407,7 @@ __svc_create(struct svc_program *prog, u
kcalloc(serv->sv_nrpools, sizeof(struct svc_pool),
GFP_KERNEL);
if (!serv->sv_pools) {
- kfree(serv->sv_stats);
+ free_percpu(serv->sv_stats_percpu);
kfree(serv);
return NULL;
}
@@ -495,7 +496,7 @@ svc_destroy(struct svc_serv *serv)
svc_unregister(serv);
kfree(serv->sv_pools);
- kfree(serv->sv_stats);
+ free_percpu(serv->sv_stats_percpu);
kfree(serv);
}
EXPORT_SYMBOL_GPL(svc_destroy);
--
Greg