by Dai Ngo

[permalink] [raw]

Subject: Re: [PATCH RFC v17 06/11] NFSD: Update find_clp_in_name_tree() to handle courtesy client

On 3/22/22 2:50 PM, J. Bruce Fields wrote:
> On Thu, Mar 17, 2022 at 12:43:43AM -0700, Dai Ngo wrote:
>> Update find_clp_in_name_tree to check and expire courtesy client.
>>
>> Update find_confirmed_client_by_name to discard the courtesy
>> client by setting CLIENT_EXPIRED.
>>
>> Update nfsd4_setclientid to expire client with CLIENT_EXPIRED
>> state to prevent multiple confirmed clients with the same name
>> on the conf_id_hashtbl list.

This should be nn->conf_name_tree instead of conf_id_hashtbl list.

> I could use a little more "why" here.
>
> I'll give it another read, but right now I'm just not understanding how
> this is meant to work.

Here is the problem description when running pynfs/nfs4.0 CID1 (need to
run CID5 first to create ton of clients) which requires the change in
nfsd4_setclientid to get the test to pass:

1st setclientid(name=clientA,verf=X)
. find confirmed courtesy client with same name clientA (clientid:1) and
mark it DESTROY and leave it on the confirmed list.
. assign new clientA with clientid:2
. put clientid:2 on unconfirmed list
. return clientid:2

1st setclientid_confirm(clientid:2)
. does not find any confirmed for clientid:2
. find unconfirmed clientid:2
. does not find any confirmed clientA (find_confirmed_client_by_name)
since courtesy clientA was marked DESTROY
. put clientid:2: to conf_name_tree list

PROBLEM: new clientA (clientid:2) and courtesy clientA (clientid:1)
are both now on the conf_name_tree list.

clientid:2 does OPEN -> stidX

2nd setclientid(name=clientA, verf=Y)
. does not find any confirmed clientA: skip courtesy clientA since it
was marked DESTROY and does not find confirmed client:2 due to the
PROBLEM noted above; duplicate entry in binary tree.
. create new clientid:3 and put it on unconfirmed list
. return clientid:3

2nd setclientid_confirm(clientid:3)
. does not find any confirmed clientid:3
. find unconfirmed clientid:3
. does not find any confirmed clientA: skip courtesy clientA since it
was marked DESTROY and does not find confirmed client:2 due to the
PROBLEM noted above; duplicate entry in binary tree.
. put clientid:3 on conf_name_tree list

clientid:3 does CLOSE(stidX) -> NFS4_OK since clientid:2 still valid <ERROR!>

-Dai

>
> --b.
>
>> Signed-off-by: Dai Ngo <[email protected]>
>> ---
>> fs/nfsd/nfs4state.c | 27 ++++++++++++++++++++++++---
>> fs/nfsd/state.h | 22 ++++++++++++++++++++++
>> 2 files changed, 46 insertions(+), 3 deletions(-)
>>
>> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
>> index c6b5e05c9c34..dc0e60bf694c 100644
>> --- a/fs/nfsd/nfs4state.c
>> +++ b/fs/nfsd/nfs4state.c
>> @@ -2834,8 +2834,11 @@ find_clp_in_name_tree(struct xdr_netobj *name, struct rb_root *root)
>> node = node->rb_left;
>> else if (cmp < 0)
>> node = node->rb_right;
>> - else
>> + else {
>> + if (nfsd4_courtesy_clnt_expired(clp))
>> + return NULL;
>> return clp;
>> + }
>> }
>> return NULL;
>> }
>> @@ -2914,8 +2917,15 @@ static bool clp_used_exchangeid(struct nfs4_client *clp)
>> static struct nfs4_client *
>> find_confirmed_client_by_name(struct xdr_netobj *name, struct nfsd_net *nn)
>> {
>> + struct nfs4_client *clp;
>> +
>> lockdep_assert_held(&nn->client_lock);
>> - return find_clp_in_name_tree(name, &nn->conf_name_tree);
>> + clp = find_clp_in_name_tree(name, &nn->conf_name_tree);
>> + if (clp && clp->cl_cs_client_state == NFSD4_CLIENT_RECONNECTED) {
>> + nfsd4_discard_courtesy_clnt(clp);
>> + clp = NULL;
>> + }
>> + return clp;
>> }
>>
>> static struct nfs4_client *
>> @@ -4032,12 +4042,19 @@ nfsd4_setclientid(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
>> struct nfs4_client *unconf = NULL;
>> __be32 status;
>> struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
>> + struct nfs4_client *cclient = NULL;
>>
>> new = create_client(clname, rqstp, &clverifier);
>> if (new == NULL)
>> return nfserr_jukebox;
>> spin_lock(&nn->client_lock);
>> - conf = find_confirmed_client_by_name(&clname, nn);
>> + /* find confirmed client by name */
>> + conf = find_clp_in_name_tree(&clname, &nn->conf_name_tree);
>> + if (conf && conf->cl_cs_client_state == NFSD4_CLIENT_RECONNECTED) {
>> + cclient = conf;
>> + conf = NULL;
>> + }
>> +
>> if (conf && client_has_state(conf)) {
>> status = nfserr_clid_inuse;
>> if (clp_used_exchangeid(conf))
>> @@ -4068,7 +4085,11 @@ nfsd4_setclientid(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
>> new = NULL;
>> status = nfs_ok;
>> out:
>> + if (cclient)
>> + unhash_client_locked(cclient);
>> spin_unlock(&nn->client_lock);
>> + if (cclient)
>> + expire_client(cclient);
>> if (new)
>> free_client(new);
>> if (unconf) {
>> diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
>> index d156ae3ab46c..14b2c158ccca 100644
>> --- a/fs/nfsd/state.h
>> +++ b/fs/nfsd/state.h
>> @@ -735,6 +735,7 @@ extern void nfsd4_client_record_remove(struct nfs4_client *clp);
>> extern int nfsd4_client_record_check(struct nfs4_client *clp);
>> extern void nfsd4_record_grace_done(struct nfsd_net *nn);
>>
>> +/* courteous server */
>> static inline bool
>> nfsd4_expire_courtesy_clnt(struct nfs4_client *clp)
>> {
>> @@ -749,4 +750,25 @@ nfsd4_expire_courtesy_clnt(struct nfs4_client *clp)
>> return rc;
>> }
>>
>> +static inline void
>> +nfsd4_discard_courtesy_clnt(struct nfs4_client *clp)
>> +{
>> + spin_lock(&clp->cl_cs_lock);
>> + clp->cl_cs_client_state = NFSD4_CLIENT_EXPIRED;
>> + spin_unlock(&clp->cl_cs_lock);
>> +}
>> +
>> +static inline bool
>> +nfsd4_courtesy_clnt_expired(struct nfs4_client *clp)
>> +{
>> + bool rc = false;
>> +
>> + spin_lock(&clp->cl_cs_lock);
>> + if (clp->cl_cs_client_state == NFSD4_CLIENT_EXPIRED)
>> + rc = true;
>> + if (clp->cl_cs_client_state == NFSD4_CLIENT_COURTESY)
>> + clp->cl_cs_client_state = NFSD4_CLIENT_RECONNECTED;
>> + spin_unlock(&clp->cl_cs_lock);
>> + return rc;
>> +}
>> #endif /* NFSD4_STATE_H */
>> --
>> 2.9.5

2022-03-24 12:25:19

by Dai Ngo

[permalink] [raw]

Subject: Re: [PATCH RFC v17 05/11] NFSD: Update nfs4_get_vfs_file() to handle courtesy client

On 3/21/22 2:44 PM, J. Bruce Fields wrote:
> On Thu, Mar 17, 2022 at 12:43:42AM -0700, Dai Ngo wrote:
>> Update nfs4_get_vfs_file and nfs4_upgrade_open to handle share
>> reservation conflict with courtesy client.
>>
>> Update nfs4_get_vfs_file and nfs4_upgrade_open to handle share
>> reservation conflict with courtesy client.
>>
>> When we have deny/access conflict we walk the fi_stateids of the
>> file in question, looking for open stateid and check the deny/access
>> of that stateid against the one from the open request. If there is
>> a conflict then we check if the client that owns that stateid is
>> a courtesy client. If it is then we set the client state to
>> CLIENT_EXPIRED and allow the open request to continue. We have
>> to scan all the stateid's of the file since the conflict can be
>> caused by multiple open stateid's.
>>
>> Client with CLIENT_EXPIRED is expired by the laundromat.
>>
>> Signed-off-by: Dai Ngo <[email protected]>
>> ---
>> fs/nfsd/nfs4state.c | 99 ++++++++++++++++++++++++++++++++++++++++++++++++-----
>> 1 file changed, 91 insertions(+), 8 deletions(-)
>>
>> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
>> index f20c75890594..c6b5e05c9c34 100644
>> --- a/fs/nfsd/nfs4state.c
>> +++ b/fs/nfsd/nfs4state.c
>> @@ -4950,9 +4950,75 @@ nfsd4_truncate(struct svc_rqst *rqstp, struct svc_fh *fh,
>> return nfsd_setattr(rqstp, fh, &iattr, 0, (time64_t)0);
>> }
>>
>> +static bool
>> +nfs4_check_access_deny_bmap(struct nfs4_ol_stateid *stp, u32 access,
>> + bool share_access)
>> +{
>> + if (share_access) {
>> + if (!stp->st_deny_bmap)
>> + return false;
>> +
>> + if ((stp->st_deny_bmap & (1 << NFS4_SHARE_DENY_BOTH)) ||
>> + (access & NFS4_SHARE_ACCESS_READ &&
>> + stp->st_deny_bmap & (1 << NFS4_SHARE_DENY_READ)) ||
>> + (access & NFS4_SHARE_ACCESS_WRITE &&
>> + stp->st_deny_bmap & (1 << NFS4_SHARE_DENY_WRITE))) {
>> + return true;
>> + }
>> + return false;
> Looks like you could do this with just:
>
> return share_access & bmap_to_share_mode(stp->st_deny_bmap);

Thanks, this makes it much simpler. Fix in v18.

>
>
>> + }
>> + if ((access & NFS4_SHARE_DENY_BOTH) ||
>> + (access & NFS4_SHARE_DENY_READ &&
>> + stp->st_access_bmap & (1 << NFS4_SHARE_ACCESS_READ)) ||
>> + (access & NFS4_SHARE_DENY_WRITE &&
>> + stp->st_access_bmap & (1 << NFS4_SHARE_ACCESS_WRITE))) {
>> + return true;
>> + }
> Likewise.

Fix in v18.

>
> Also, I think it'd be simpler to check for both access and deny
> conflicts here, instead of just one or the other.

I'm not clear here.

I remove nfs4_check_access_deny_bmap, fold this functionality
into nfs4_resolve_deny_conflicts_locked by making use of
bmap_to_share_mode.

>
>> + return false;
>> +}
>> +
>> +/*
>> + * Check whether courtesy clients have conflicting access
>> + *
>> + * access: is op_share_access if share_access is true.
>> + * Check if access mode, op_share_access, would conflict with
>> + * the current deny mode of the file 'fp'.
>> + * access: is op_share_deny if share_access is false.
>> + * Check if the deny mode, op_share_deny, would conflict with
>> + * current access of the file 'fp'.
>> + * stp: skip checking this entry.
>> + * new_stp: normal open, not open upgrade.
>> + *
>> + * Function returns:
>> + * true - access/deny mode conflict with normal client.
>> + * false - no conflict or conflict with courtesy client(s) is resolved.
>> + */
>> +static bool
>> +nfs4_resolve_deny_conflicts_locked(struct nfs4_file *fp, bool new_stp,
>> + struct nfs4_ol_stateid *stp, u32 access, bool share_access)
>> +{
>> + struct nfs4_ol_stateid *st;
>> + struct nfs4_client *clp;
>> + bool conflict = false;
>> +
>> + lockdep_assert_held(&fp->fi_lock);
>> + list_for_each_entry(st, &fp->fi_stateids, st_perfile) {
>> + if (st->st_openstp || (st == stp && new_stp) ||
> I'd split this into separate if statements and add a comment at least
> for the st_openstp case, which isn't too obvious:
>
> if (st->st_openstp) /* ignore lock stateids */
> continue;

Fix in v18.

>
>> + (!nfs4_check_access_deny_bmap(st,
>> + access, share_access)))
>> + continue;
>> + clp = st->st_stid.sc_client;
>> + if (nfsd4_expire_courtesy_clnt(clp))
>> + continue;
>> + conflict = true;
>> + break;
>> + }
>> + return conflict;
>> +}
>> +
>> static __be32 nfs4_get_vfs_file(struct svc_rqst *rqstp, struct nfs4_file *fp,
>> struct svc_fh *cur_fh, struct nfs4_ol_stateid *stp,
>> - struct nfsd4_open *open)
>> + struct nfsd4_open *open, bool new_stp)
>> {
>> struct nfsd_file *nf = NULL;
>> __be32 status;
>> @@ -4968,15 +5034,29 @@ static __be32 nfs4_get_vfs_file(struct svc_rqst *rqstp, struct nfs4_file *fp,
>> */
>> status = nfs4_file_check_deny(fp, open->op_share_deny);
>> if (status != nfs_ok) {
>> - spin_unlock(&fp->fi_lock);
>> - goto out;
>> + if (status != nfserr_share_denied) {
>> + spin_unlock(&fp->fi_lock);
>> + goto out;
>> + }
>> + if (nfs4_resolve_deny_conflicts_locked(fp, new_stp,
>> + stp, open->op_share_deny, false)) {
>> + spin_unlock(&fp->fi_lock);
>> + goto out;
>> + }
>> }
>>
>> /* set access to the file */
>> status = nfs4_file_get_access(fp, open->op_share_access);
>> if (status != nfs_ok) {
>> - spin_unlock(&fp->fi_lock);
>> - goto out;
>> + if (status != nfserr_share_denied) {
>> + spin_unlock(&fp->fi_lock);
>> + goto out;
>> + }
>> + if (nfs4_resolve_deny_conflicts_locked(fp, new_stp,
>> + stp, open->op_share_access, true)) {
>> + spin_unlock(&fp->fi_lock);
>> + goto out;
>> + }
>> }
>>
>> /* Set access bits in stateid */
>> @@ -5027,7 +5107,7 @@ nfs4_upgrade_open(struct svc_rqst *rqstp, struct nfs4_file *fp, struct svc_fh *c
>> unsigned char old_deny_bmap = stp->st_deny_bmap;
>>
>> if (!test_access(open->op_share_access, stp))
>> - return nfs4_get_vfs_file(rqstp, fp, cur_fh, stp, open);
>> + return nfs4_get_vfs_file(rqstp, fp, cur_fh, stp, open, false);
>>
>> /* test and set deny mode */
>> spin_lock(&fp->fi_lock);
>> @@ -5036,7 +5116,10 @@ nfs4_upgrade_open(struct svc_rqst *rqstp, struct nfs4_file *fp, struct svc_fh *c
>> set_deny(open->op_share_deny, stp);
>> fp->fi_share_deny |=
>> (open->op_share_deny & NFS4_SHARE_DENY_BOTH);
>> - }
>> + } else if (status == nfserr_share_denied &&
>> + !nfs4_resolve_deny_conflicts_locked(fp, false, stp,
>> + open->op_share_deny, false))
> Looks to me like these nfs4_resolve_deny_conflicts_locked() calls could
> go into nfs4_file_check_deny and nfs4_file_get_access instead.

Fix in v18.

-Dai

>
> --b.
>
>> + status = nfs_ok;
>> spin_unlock(&fp->fi_lock);
>>
>> if (status != nfs_ok)
>> @@ -5376,7 +5459,7 @@ nfsd4_process_open2(struct svc_rqst *rqstp, struct svc_fh *current_fh, struct nf
>> goto out;
>> }
>> } else {
>> - status = nfs4_get_vfs_file(rqstp, fp, current_fh, stp, open);
>> + status = nfs4_get_vfs_file(rqstp, fp, current_fh, stp, open, true);
>> if (status) {
>> stp->st_stid.sc_type = NFS4_CLOSED_STID;
>> release_open_stateid(stp);
>> --
>> 2.9.5