knfsd: Add more statistics to the duplicate request cache; we now track
the memory usage of the data structure, the age of evicted entries,
and probes along hash chains.
Also, reduce contention for the cachelines of the global nfsdstats
structure by keeping repcache stats per-bucket and aggregating them
on demand when userspace reads /proc.
Signed-off-by: Greg Banks <[email protected]>
---
fs/nfsd/nfscache.c | 72 ++++++++++++++++++++++++++++++++--
fs/nfsd/stats.c | 11 +++++
include/linux/nfsd/cache.h | 1
include/linux/nfsd/stats.h | 7 +++
4 files changed, 88 insertions(+), 3 deletions(-)
Index: linux-git-20061009/fs/nfsd/stats.c
===================================================================
--- linux-git-20061009.orig/fs/nfsd/stats.c 2006-10-11 14:07:49.459633262 +1000
+++ linux-git-20061009/fs/nfsd/stats.c 2006-10-11 14:21:56.653517425 +1000
@@ -34,6 +34,7 @@
#include <linux/sunrpc/stats.h>
#include <linux/nfsd/nfsd.h>
#include <linux/nfsd/stats.h>
+#include <linux/nfsd/cache.h>
struct nfsd_stats nfsdstats;
struct svc_stat nfsd_svcstats = {
@@ -44,6 +45,8 @@ static int nfsd_proc_show(struct seq_fil
{
int i;
+ nfsd_cache_update_stats();
+
seq_printf(seq, "rc %u %u %u\nfh %u %u %u %u %u\nio %u %u\n",
nfsdstats.rchits,
nfsdstats.rcmisses,
@@ -72,6 +75,14 @@ static int nfsd_proc_show(struct seq_fil
/* show my rpc info */
svc_seq_show(seq, &nfsd_svcstats);
+ /* extended repcache stats */
+ seq_printf(seq, "rc2 %u %u %u %u %u\n",
+ nfsdstats.rcprobes,
+ nfsdstats.rcexpands,
+ nfsdstats.rcentries,
+ nfsdstats.rcmem,
+ (1000 * nfsdstats.rcage) / HZ);
+
#ifdef CONFIG_NFSD_V4
/* Show count for individual nfsv4 operations */
/* Writing operation numbers 0 1 2 also for maintaining uniformity */
Index: linux-git-20061009/include/linux/nfsd/stats.h
===================================================================
--- linux-git-20061009.orig/include/linux/nfsd/stats.h 2006-10-11 14:07:49.563619730 +1000
+++ linux-git-20061009/include/linux/nfsd/stats.h 2006-10-11 14:08:07.581275049 +1000
@@ -29,6 +29,13 @@ struct nfsd_stats {
unsigned int ra_size; /* size of ra cache */
unsigned int ra_depth[11]; /* number of times ra entry was found that deep
* in the cache (10percentiles). [10] = not found */
+ /* extended repcache stats */
+ unsigned int rcprobes; /* counter: walks down hash chains */
+ unsigned int rcexpands; /* counter: when the cache is expanded */
+ unsigned int rcentries; /* instant: # entries */
+ unsigned int rcmem; /* instant: bytes of memory used */
+ unsigned int rcage; /* instant: age in milliseconds of last
+ * entry reused from the LRU list */
#ifdef CONFIG_NFSD_V4
unsigned int nfs4_opcount[LAST_NFS4_OP + 1]; /* count of individual nfsv4 operations */
#endif
Index: linux-git-20061009/fs/nfsd/nfscache.c
===================================================================
--- linux-git-20061009.orig/fs/nfsd/nfscache.c 2006-10-11 14:08:07.533281294 +1000
+++ linux-git-20061009/fs/nfsd/nfscache.c 2006-10-11 14:08:07.581275049 +1000
@@ -83,6 +83,21 @@ struct svc_cache_bucket
/* parameters for expand rate limiting */
unsigned long last;
unsigned long nhits;
+ /* stats, stored per-bucket and accumulated on demand */
+ struct
+ {
+ unsigned int rchits; /* repcache hits */
+ unsigned int rcmisses; /* repcache hits */
+ unsigned int rcnocache; /* uncached reqs */
+
+ /* extended repcache stats */
+ unsigned int rcprobes; /* counter: walks down hash chains */
+ unsigned int rcexpands; /* counter: when the cache is expanded */
+ unsigned int rcentries; /* instant: # entries */
+ unsigned int rcmem; /* instant: bytes of memory used */
+ unsigned int rcage; /* instant: age in milliseconds of last
+ * entry reused from the LRU list */
+ } stats;
} ____cacheline_aligned_in_smp;
static struct svc_cache_bucket cache_buckets[CACHE_NUM_BUCKETS];
@@ -161,6 +176,8 @@ nfsd_cache_bucket_expand(struct svc_cach
spin_lock(&b->lock);
b->size += (nrecs-i);
+ b->stats.rcentries += (nrecs-i);
+ b->stats.rcmem += (nrecs-i) * sizeof(struct svc_cacherep);
list_splice(&lru, &b->lru);
spin_unlock(&b->lock);
@@ -187,6 +204,8 @@ nfsd_cache_init(void)
return;
}
memset(b->hash, 0, HASHSIZE * sizeof(struct hlist_head));
+
+ b->stats.rcmem += HASHSIZE * sizeof(struct hlist_head);
}
cache_disabled = 0;
@@ -269,6 +288,7 @@ nfsd_cache_lookup(struct svc_rqst *rqstp
int rtn;
int safe = 0;
int expand = 0;
+ unsigned int nprobes = 0;
h = request_hash(xid, &rqstp->rq_addr);
b = bucket_for_hash(h);
@@ -276,7 +296,7 @@ nfsd_cache_lookup(struct svc_rqst *rqstp
rqstp->rq_cacherep = NULL;
if (cache_disabled || type == RC_NOCACHE) {
- nfsdstats.rcnocache++;
+ b->stats.rcnocache++;
return RC_DOIT;
}
@@ -286,17 +306,20 @@ nfsd_cache_lookup(struct svc_rqst *rqstp
rh = &b->hash[h];
age = jiffies - 120*HZ;
hlist_for_each_entry(rp, hn, rh, c_hash) {
+ nprobes++;
if (rp->c_state != RC_UNUSED &&
xid == rp->c_xid &&
compare_sockaddr_in(&rqstp->rq_addr, &rp->c_addr) &&
proc == rp->c_proc &&
proto == rp->c_prot && vers == rp->c_vers &&
time_after(rp->c_timestamp, age)) {
- nfsdstats.rchits++;
+ b->stats.rchits++;
+ b->stats.rcprobes += nprobes;
goto found_entry;
}
}
- nfsdstats.rcmisses++;
+ b->stats.rcmisses++;
+ b->stats.rcprobes += nprobes;
/* This loop shouldn't take more than a few iterations normally */
list_for_each_entry(rp, &b->lru, c_lru) {
@@ -324,12 +347,14 @@ nfsd_cache_lookup(struct svc_rqst *rqstp
if (rp->c_state != RC_UNUSED) {
/* reusing an existing cache entry */
age = jiffies - rp->c_timestamp;
+ b->stats.rcage = age;
if (age < CACHE_THRESH_AGE &&
b->size < CACHE_BUCKET_MAX_SIZE &&
nfsd_cache_expand_ratelimit(b)) {
expand = CACHE_BUCKET_INCREMENT;
if (b->size + expand > CACHE_BUCKET_MAX_SIZE)
expand = CACHE_BUCKET_MAX_SIZE - b->size;
+ b->stats.rcexpands++;
}
}
@@ -350,6 +375,7 @@ nfsd_cache_lookup(struct svc_rqst *rqstp
if (rp->c_type == RC_REPLBUFF) {
kfree(rp->c_replvec.iov_base);
rp->c_replvec.iov_base = NULL;
+ b->stats.rcmem -= rp->c_replvec.iov_len;
}
rp->c_type = RC_NOCACHE;
out:
@@ -419,6 +445,7 @@ nfsd_cache_update(struct svc_rqst *rqstp
struct kvec *resv = &rqstp->rq_res.head[0], *cachv;
int len;
struct svc_cache_bucket *b;
+ unsigned int moremem = 0;
if (!(rp = rqstp->rq_cacherep) || cache_disabled)
return;
@@ -451,6 +478,7 @@ nfsd_cache_update(struct svc_rqst *rqstp
}
cachv->iov_len = len << 2;
memcpy(cachv->iov_base, statp, len << 2);
+ moremem = len << 2;
break;
}
spin_lock(&b->lock);
@@ -459,6 +487,8 @@ nfsd_cache_update(struct svc_rqst *rqstp
rp->c_type = cachetype;
rp->c_state = RC_DONE;
rp->c_timestamp = jiffies;
+ if (moremem)
+ b->stats.rcmem += moremem;
spin_unlock(&b->lock);
return;
}
@@ -482,3 +512,39 @@ nfsd_cache_append(struct svc_rqst *rqstp
vec->iov_len += data->iov_len;
return 1;
}
+
+void
+nfsd_cache_update_stats(void)
+{
+ unsigned int bucket;
+ unsigned int i;
+ struct svc_cache_bucket totbucket;
+ unsigned int *totp = (unsigned int *)&totbucket.stats;
+ unsigned int *bp;
+ unsigned int minage = ~0;
+
+ memset(&totbucket, 0, sizeof(totbucket));
+
+ for (bucket = 0 ; bucket < CACHE_NUM_BUCKETS ; bucket++)
+ {
+ struct svc_cache_bucket *b = &cache_buckets[bucket];
+
+ /* stats are accumulated across buckets unlocked,
+ * but who cares they're only stats anyway */
+ bp = (unsigned int *)&b->stats;
+ for (i = 0 ; i < sizeof(b->stats)/sizeof(unsigned int) ; i++) {
+ totp[i] += bp[i];
+ }
+ if (b->stats.rcage < minage)
+ minage = b->stats.rcage;
+ }
+ nfsdstats.rchits = totbucket.stats.rchits;
+ nfsdstats.rcmisses = totbucket.stats.rcmisses;
+ nfsdstats.rcnocache = totbucket.stats.rcnocache;
+ nfsdstats.rcprobes = totbucket.stats.rcprobes;
+ nfsdstats.rcexpands = totbucket.stats.rcexpands;
+ nfsdstats.rcentries = totbucket.stats.rcentries;
+ nfsdstats.rcmem = totbucket.stats.rcmem;
+ nfsdstats.rcage = minage;
+}
+
Index: linux-git-20061009/include/linux/nfsd/cache.h
===================================================================
--- linux-git-20061009.orig/include/linux/nfsd/cache.h 2006-10-11 14:08:07.185326575 +1000
+++ linux-git-20061009/include/linux/nfsd/cache.h 2006-10-11 14:08:07.585274528 +1000
@@ -75,6 +75,7 @@ void nfsd_cache_init(void);
void nfsd_cache_shutdown(void);
int nfsd_cache_lookup(struct svc_rqst *, int);
void nfsd_cache_update(struct svc_rqst *, int, u32 *);
+void nfsd_cache_update_stats(void);
#endif /* __KERNEL__ */
#endif /* NFSCACHE_H */
Greg.
--
Greg Banks, R&D Software Engineer, SGI Australian Software Group.
I don't speak for SGI.
-------------------------------------------------------------------------
Using Tomcat but need to do more? Need to support web services, security?
Get stuff done quickly with pre-integrated technology to make your job easier
Download IBM WebSphere Application Server v.1.0.1 based on Apache Geronimo
http://sel.as-us.falkag.net/sel?cmd=lnk&kid=120709&bid=263057&dat=121642
_______________________________________________
NFS maillist - [email protected]
https://lists.sourceforge.net/lists/listinfo/nfs