2020-01-27 15:01:08

by Trond Myklebust

Subject: [PATCH 4/5] NFSv4: Add accounting for the number of active delegations held

In order to better manage our delegation caching, add a counter
to track the number of active delegations.

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/delegation.c | 36 ++++++++++++++++++++++++------------
1 file changed, 24 insertions(+), 12 deletions(-)

diff --git a/fs/nfs/delegation.c b/fs/nfs/delegation.c
index 90e50f32f3e0..a777b3d0e720 100644
--- a/fs/nfs/delegation.c
+++ b/fs/nfs/delegation.c
@@ -25,13 +25,29 @@
#include "internal.h"
#include "nfs4trace.h"

-static void nfs_free_delegation(struct nfs_delegation *delegation)
+static atomic_long_t nfs_active_delegations;
+
+static void __nfs_free_delegation(struct nfs_delegation *delegation)
{
put_cred(delegation->cred);
delegation->cred = NULL;
kfree_rcu(delegation, rcu);
}

+static void nfs_mark_delegation_revoked(struct nfs_delegation *delegation)
+{
+ if (!test_and_set_bit(NFS_DELEGATION_REVOKED, &delegation->flags)) {
+ delegation->stateid.type = NFS4_INVALID_STATEID_TYPE;
+ atomic_long_dec(&nfs_active_delegations);
+ }
+}
+
+static void nfs_free_delegation(struct nfs_delegation *delegation)
+{
+ nfs_mark_delegation_revoked(delegation);
+ __nfs_free_delegation(delegation);
+}
+
/**
* nfs_mark_delegation_referenced - set delegation's REFERENCED flag
* @delegation: delegation to process
@@ -343,7 +359,8 @@ nfs_update_inplace_delegation(struct nfs_delegation *delegation,
delegation->stateid.seqid = update->stateid.seqid;
smp_wmb();
delegation->type = update->type;
- clear_bit(NFS_DELEGATION_REVOKED, &delegation->flags);
+ if (test_and_clear_bit(NFS_DELEGATION_REVOKED, &delegation->flags))
+ atomic_long_inc(&nfs_active_delegations);
}
}

@@ -423,6 +440,8 @@ int nfs_inode_set_delegation(struct inode *inode, const struct cred *cred,
rcu_assign_pointer(nfsi->delegation, delegation);
delegation = NULL;

+ atomic_long_inc(&nfs_active_delegations);
+
trace_nfs4_set_delegation(inode, type);

spin_lock(&inode->i_lock);
@@ -432,7 +451,7 @@ int nfs_inode_set_delegation(struct inode *inode, const struct cred *cred,
out:
spin_unlock(&clp->cl_lock);
if (delegation != NULL)
- nfs_free_delegation(delegation);
+ __nfs_free_delegation(delegation);
if (freeme != NULL) {
nfs_do_return_delegation(inode, freeme, 0);
nfs_free_delegation(freeme);
@@ -796,13 +815,6 @@ static void nfs_client_mark_return_unused_delegation_types(struct nfs_client *cl
rcu_read_unlock();
}

-static void nfs_mark_delegation_revoked(struct nfs_server *server,
- struct nfs_delegation *delegation)
-{
- set_bit(NFS_DELEGATION_REVOKED, &delegation->flags);
- delegation->stateid.type = NFS4_INVALID_STATEID_TYPE;
-}
-
static void nfs_revoke_delegation(struct inode *inode,
const nfs4_stateid *stateid)
{
@@ -830,7 +842,7 @@ static void nfs_revoke_delegation(struct inode *inode,
}
spin_unlock(&delegation->lock);
}
- nfs_mark_delegation_revoked(NFS_SERVER(inode), delegation);
+ nfs_mark_delegation_revoked(delegation);
ret = true;
out:
rcu_read_unlock();
@@ -869,7 +881,7 @@ void nfs_delegation_mark_returned(struct inode *inode,
delegation->stateid.seqid = stateid->seqid;
}

- nfs_mark_delegation_revoked(NFS_SERVER(inode), delegation);
+ nfs_mark_delegation_revoked(delegation);

out_clear_returning:
clear_bit(NFS_DELEGATION_RETURNING, &delegation->flags);
--
2.24.1

2020-01-27 15:01:34

by Trond Myklebust

Subject: [PATCH 5/5] NFSv4: Limit the total number of cached delegations

Delegations can be expensive to return, and can cause scalability issues
for the server. Let's therefore try to limit the number of inactive
delegations we hold.
Once the number of delegations is above a certain threshold, start
to return them on close.

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/delegation.c | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/fs/nfs/delegation.c b/fs/nfs/delegation.c
index a777b3d0e720..4a841071d8a7 100644
--- a/fs/nfs/delegation.c
+++ b/fs/nfs/delegation.c
@@ -25,7 +25,10 @@
#include "internal.h"
#include "nfs4trace.h"

+#define NFS_DEFAULT_DELEGATION_WATERMARK (5000U)
+
static atomic_long_t nfs_active_delegations;
+static unsigned nfs_delegation_watermark = NFS_DEFAULT_DELEGATION_WATERMARK;

static void __nfs_free_delegation(struct nfs_delegation *delegation)
{
@@ -676,7 +679,8 @@ void nfs4_inode_return_delegation_on_close(struct inode *inode)
delegation = nfs4_get_valid_delegation(inode);
if (!delegation)
goto out;
- if (test_bit(NFS_DELEGATION_RETURN_IF_CLOSED, &delegation->flags)) {
+ if (test_bit(NFS_DELEGATION_RETURN_IF_CLOSED, &delegation->flags) ||
+ atomic_long_read(&nfs_active_delegations) >= nfs_delegation_watermark) {
spin_lock(&delegation->lock);
if (delegation->inode &&
list_empty(&NFS_I(inode)->open_files) &&
@@ -1365,3 +1369,5 @@ bool nfs4_delegation_flush_on_close(const struct inode *inode)
rcu_read_unlock();
return ret;
}
+
+module_param_named(delegation_watermark, nfs_delegation_watermark, uint, 0644);
--
2.24.1