When there is layout state on a filesystem that is being "unlocked" that
is now revoked, which involves closing the nfsd_file and releasing the
vfs lease.
To avoid races, all users of ->ls_file - after the layout state has been
successfully created - now need to take a counted reference under
rcu_read_lock(). To support this, ->fence_client and
nfsd4_cb_layout_fail() now take a second argument being the nfsd_file.
Signed-off-by: NeilBrown <[email protected]>
---
fs/nfsd/blocklayout.c | 4 ++--
fs/nfsd/nfs4layouts.c | 38 +++++++++++++++++++++++++++-----------
fs/nfsd/nfs4state.c | 28 +++++++++++++++++++++-------
fs/nfsd/pnfs.h | 7 ++++++-
4 files changed, 56 insertions(+), 21 deletions(-)
diff --git a/fs/nfsd/blocklayout.c b/fs/nfsd/blocklayout.c
index 46fd74d91ea9..3c040c81c77d 100644
--- a/fs/nfsd/blocklayout.c
+++ b/fs/nfsd/blocklayout.c
@@ -328,10 +328,10 @@ nfsd4_scsi_proc_layoutcommit(struct inode *inode,
}
static void
-nfsd4_scsi_fence_client(struct nfs4_layout_stateid *ls)
+nfsd4_scsi_fence_client(struct nfs4_layout_stateid *ls, struct nfsd_file *file)
{
struct nfs4_client *clp = ls->ls_stid.sc_client;
- struct block_device *bdev = ls->ls_file->nf_file->f_path.mnt->mnt_sb->s_bdev;
+ struct block_device *bdev = file->nf_file->f_path.mnt->mnt_sb->s_bdev;
bdev->bd_disk->fops->pr_ops->pr_preempt(bdev, NFSD_MDS_PR_KEY,
nfsd4_scsi_pr_key(clp), 0, true);
diff --git a/fs/nfsd/nfs4layouts.c b/fs/nfsd/nfs4layouts.c
index 77656126ad2a..dbc52413ce57 100644
--- a/fs/nfsd/nfs4layouts.c
+++ b/fs/nfsd/nfs4layouts.c
@@ -152,6 +152,18 @@ void nfsd4_setup_layout_type(struct svc_export *exp)
#endif
}
+void nfsd4_close_layout(struct nfs4_layout_stateid *ls)
+{
+ struct nfsd_file *fl = xchg(&ls->ls_file, NULL);
+
+ if (fl) {
+ if (!nfsd4_layout_ops[ls->ls_layout_type]->disable_recalls)
+ vfs_setlease(fl->nf_file, F_UNLCK, NULL,
+ (void **)&ls);
+ nfsd_file_put(fl);
+ }
+}
+
static void
nfsd4_free_layout_stateid(struct nfs4_stid *stid)
{
@@ -169,9 +181,7 @@ nfsd4_free_layout_stateid(struct nfs4_stid *stid)
list_del_init(&ls->ls_perfile);
spin_unlock(&fp->fi_lock);
- if (!nfsd4_layout_ops[ls->ls_layout_type]->disable_recalls)
- vfs_setlease(ls->ls_file->nf_file, F_UNLCK, NULL, (void **)&ls);
- nfsd_file_put(ls->ls_file);
+ nfsd4_close_layout(ls);
if (ls->ls_recalled)
atomic_dec(&ls->ls_stid.sc_file->fi_lo_recalls);
@@ -605,7 +615,7 @@ nfsd4_return_all_file_layouts(struct nfs4_client *clp, struct nfs4_file *fp)
}
static void
-nfsd4_cb_layout_fail(struct nfs4_layout_stateid *ls)
+nfsd4_cb_layout_fail(struct nfs4_layout_stateid *ls, struct nfsd_file *file)
{
struct nfs4_client *clp = ls->ls_stid.sc_client;
char addr_str[INET6_ADDRSTRLEN];
@@ -627,7 +637,7 @@ nfsd4_cb_layout_fail(struct nfs4_layout_stateid *ls)
argv[0] = (char *)nfsd_recall_failed;
argv[1] = addr_str;
- argv[2] = ls->ls_file->nf_file->f_path.mnt->mnt_sb->s_id;
+ argv[2] = file->nf_file->f_path.mnt->mnt_sb->s_id;
argv[3] = NULL;
error = call_usermodehelper(nfsd_recall_failed, argv, envp,
@@ -657,6 +667,7 @@ nfsd4_cb_layout_done(struct nfsd4_callback *cb, struct rpc_task *task)
struct nfsd_net *nn;
ktime_t now, cutoff;
const struct nfsd4_layout_ops *ops;
+ struct nfsd_file *fl;
trace_nfsd_cb_layout_done(&ls->ls_stid.sc_stateid, task);
switch (task->tk_status) {
@@ -688,12 +699,17 @@ nfsd4_cb_layout_done(struct nfsd4_callback *cb, struct rpc_task *task)
* Unknown error or non-responding client, we'll need to fence.
*/
trace_nfsd_layout_recall_fail(&ls->ls_stid.sc_stateid);
-
- ops = nfsd4_layout_ops[ls->ls_layout_type];
- if (ops->fence_client)
- ops->fence_client(ls);
- else
- nfsd4_cb_layout_fail(ls);
+ rcu_read_lock();
+ fl = nfsd_file_get(ls->ls_file);
+ rcu_read_unlock();
+ if (fl) {
+ ops = nfsd4_layout_ops[ls->ls_layout_type];
+ if (ops->fence_client)
+ ops->fence_client(ls, fl);
+ else
+ nfsd4_cb_layout_fail(ls, fl);
+ nfsd_file_put(fl);
+ }
return 1;
case -NFS4ERR_NOMATCHING_LAYOUT:
trace_nfsd_layout_recall_done(&ls->ls_stid.sc_stateid);
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 3d85c88ec4d7..d82ca209eb96 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1712,7 +1712,8 @@ void nfsd4_revoke_states(struct net *net, struct super_block *sb)
unsigned int idhashval;
unsigned int sc_types;
- sc_types = NFS4_OPEN_STID | NFS4_LOCK_STID | NFS4_DELEG_STID;
+ sc_types = (NFS4_OPEN_STID | NFS4_LOCK_STID |
+ NFS4_DELEG_STID | NFS4_LAYOUT_STID);
spin_lock(&nn->client_lock);
for (idhashval = 0; idhashval < CLIENT_HASH_MASK; idhashval++) {
@@ -1725,6 +1726,7 @@ void nfsd4_revoke_states(struct net *net, struct super_block *sb)
if (stid) {
struct nfs4_ol_stateid *stp;
struct nfs4_delegation *dp;
+ struct nfs4_layout_stateid *ls;
spin_unlock(&nn->client_lock);
switch (stid->sc_type) {
@@ -1780,6 +1782,10 @@ void nfsd4_revoke_states(struct net *net, struct super_block *sb)
if (dp)
revoke_delegation(dp);
break;
+ case NFS4_LAYOUT_STID:
+ ls = layoutstateid(stid);
+ nfsd4_close_layout(ls);
+ break;
}
nfs4_put_stid(stid);
spin_lock(&nn->client_lock);
@@ -2859,17 +2865,25 @@ static int nfs4_show_layout(struct seq_file *s, struct nfs4_stid *st)
struct nfsd_file *file;
ls = container_of(st, struct nfs4_layout_stateid, ls_stid);
- file = ls->ls_file;
+ rcu_read_lock();
+ file = nfsd_file_get(ls->ls_file);
+ rcu_read_unlock();
- seq_printf(s, "- ");
+ seq_puts(s, "- ");
nfs4_show_stateid(s, &st->sc_stateid);
- seq_printf(s, ": { type: layout, ");
+ seq_puts(s, ": { type: layout");
/* XXX: What else would be useful? */
- nfs4_show_superblock(s, file);
- seq_printf(s, ", ");
- nfs4_show_fname(s, file);
+ if (file) {
+ seq_puts(s, ", ");
+ nfs4_show_superblock(s, file);
+ seq_puts(s, ", ");
+ nfs4_show_fname(s, file);
+ nfsd_file_put(file);
+ }
+ if (st->sc_status & NFS4_STID_ADMIN_REVOKED)
+ seq_puts(s, ", admin-revoked");
seq_printf(s, " }\n");
return 0;
diff --git a/fs/nfsd/pnfs.h b/fs/nfsd/pnfs.h
index de1e0dfed06a..f2777577865e 100644
--- a/fs/nfsd/pnfs.h
+++ b/fs/nfsd/pnfs.h
@@ -37,7 +37,8 @@ struct nfsd4_layout_ops {
__be32 (*proc_layoutcommit)(struct inode *inode,
struct nfsd4_layoutcommit *lcp);
- void (*fence_client)(struct nfs4_layout_stateid *ls);
+ void (*fence_client)(struct nfs4_layout_stateid *ls,
+ struct nfsd_file *file);
};
extern const struct nfsd4_layout_ops *nfsd4_layout_ops[];
@@ -72,6 +73,7 @@ void nfsd4_setup_layout_type(struct svc_export *exp);
void nfsd4_return_all_client_layouts(struct nfs4_client *);
void nfsd4_return_all_file_layouts(struct nfs4_client *clp,
struct nfs4_file *fp);
+void nfsd4_close_layout(struct nfs4_layout_stateid *ls);
int nfsd4_init_pnfs(void);
void nfsd4_exit_pnfs(void);
#else
@@ -89,6 +91,9 @@ static inline void nfsd4_return_all_file_layouts(struct nfs4_client *clp,
struct nfs4_file *fp)
{
}
+static inline void nfsd4_close_layout(struct nfs4_layout_stateid *ls)
+{
+}
static inline void nfsd4_exit_pnfs(void)
{
}
--
2.42.1
Hi NeilBrown,
kernel test robot noticed the following build warnings:
[auto build test WARNING on linus/master]
[also build test WARNING on v6.7-rc2 next-20231124]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/NeilBrown/nfsd-hold-cl_lock-for-hash_delegation_locked/20231124-123723
base: linus/master
patch link: https://lore.kernel.org/r/20231124002925.1816-12-neilb%40suse.de
patch subject: [PATCH 11/11] nfsd: allow layout state to be admin-revoked.
config: alpha-defconfig (https://download.01.org/0day-ci/archive/20231124/[email protected]/config)
compiler: alpha-linux-gcc (GCC) 13.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20231124/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All warnings (new ones prefixed by >>):
In file included from fs/nfsd/export.c:24:
>> fs/nfsd/pnfs.h:94:46: warning: 'struct nfs4_layout_stateid' declared inside parameter list will not be visible outside of this definition or declaration
94 | static inline void nfsd4_close_layout(struct nfs4_layout_stateid *ls)
| ^~~~~~~~~~~~~~~~~~~
fs/nfsd/export.c: In function 'exp_rootfh':
fs/nfsd/export.c:1016:34: warning: variable 'inode' set but not used [-Wunused-but-set-variable]
1016 | struct inode *inode;
| ^~~~~
vim +94 fs/nfsd/pnfs.h
86
87 static inline void nfsd4_return_all_client_layouts(struct nfs4_client *clp)
88 {
89 }
90 static inline void nfsd4_return_all_file_layouts(struct nfs4_client *clp,
91 struct nfs4_file *fp)
92 {
93 }
> 94 static inline void nfsd4_close_layout(struct nfs4_layout_stateid *ls)
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
Hi NeilBrown,
kernel test robot noticed the following build warnings:
[auto build test WARNING on linus/master]
[also build test WARNING on v6.7-rc2 next-20231124]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/NeilBrown/nfsd-hold-cl_lock-for-hash_delegation_locked/20231124-123723
base: linus/master
patch link: https://lore.kernel.org/r/20231124002925.1816-12-neilb%40suse.de
patch subject: [PATCH 11/11] nfsd: allow layout state to be admin-revoked.
config: x86_64-buildonly-randconfig-002-20231124 (https://download.01.org/0day-ci/archive/20231124/[email protected]/config)
compiler: clang version 16.0.4 (https://github.com/llvm/llvm-project.git ae42196bc493ffe877a7e3dff8be32035dea4d07)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20231124/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All warnings (new ones prefixed by >>):
In file included from fs/nfsd/export.c:24:
>> fs/nfsd/pnfs.h:94:46: warning: declaration of 'struct nfs4_layout_stateid' will not be visible outside of this function [-Wvisibility]
static inline void nfsd4_close_layout(struct nfs4_layout_stateid *ls)
^
fs/nfsd/export.c:1016:17: warning: variable 'inode' set but not used [-Wunused-but-set-variable]
struct inode *inode;
^
2 warnings generated.
vim +94 fs/nfsd/pnfs.h
86
87 static inline void nfsd4_return_all_client_layouts(struct nfs4_client *clp)
88 {
89 }
90 static inline void nfsd4_return_all_file_layouts(struct nfs4_client *clp,
91 struct nfs4_file *fp)
92 {
93 }
> 94 static inline void nfsd4_close_layout(struct nfs4_layout_stateid *ls)
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
On Fri, Nov 24, 2023 at 11:28:46AM +1100, NeilBrown wrote:
> When there is layout state on a filesystem that is being "unlocked" that
> is now revoked, which involves closing the nfsd_file and releasing the
> vfs lease.
>
> To avoid races, all users of ->ls_file - after the layout state has been
> successfully created - now need to take a counted reference under
> rcu_read_lock(). To support this, ->fence_client and
> nfsd4_cb_layout_fail() now take a second argument being the nfsd_file.
>
> Signed-off-by: NeilBrown <[email protected]>
Hi Neil, would you Cc: Christoph Hellwig <[email protected]> and Tom Haynes
<[email protected]> to this patch next time you post this series?
> ---
> fs/nfsd/blocklayout.c | 4 ++--
> fs/nfsd/nfs4layouts.c | 38 +++++++++++++++++++++++++++-----------
> fs/nfsd/nfs4state.c | 28 +++++++++++++++++++++-------
> fs/nfsd/pnfs.h | 7 ++++++-
> 4 files changed, 56 insertions(+), 21 deletions(-)
>
> diff --git a/fs/nfsd/blocklayout.c b/fs/nfsd/blocklayout.c
> index 46fd74d91ea9..3c040c81c77d 100644
> --- a/fs/nfsd/blocklayout.c
> +++ b/fs/nfsd/blocklayout.c
> @@ -328,10 +328,10 @@ nfsd4_scsi_proc_layoutcommit(struct inode *inode,
> }
>
> static void
> -nfsd4_scsi_fence_client(struct nfs4_layout_stateid *ls)
> +nfsd4_scsi_fence_client(struct nfs4_layout_stateid *ls, struct nfsd_file *file)
> {
> struct nfs4_client *clp = ls->ls_stid.sc_client;
> - struct block_device *bdev = ls->ls_file->nf_file->f_path.mnt->mnt_sb->s_bdev;
> + struct block_device *bdev = file->nf_file->f_path.mnt->mnt_sb->s_bdev;
>
> bdev->bd_disk->fops->pr_ops->pr_preempt(bdev, NFSD_MDS_PR_KEY,
> nfsd4_scsi_pr_key(clp), 0, true);
> diff --git a/fs/nfsd/nfs4layouts.c b/fs/nfsd/nfs4layouts.c
> index 77656126ad2a..dbc52413ce57 100644
> --- a/fs/nfsd/nfs4layouts.c
> +++ b/fs/nfsd/nfs4layouts.c
> @@ -152,6 +152,18 @@ void nfsd4_setup_layout_type(struct svc_export *exp)
> #endif
> }
>
> +void nfsd4_close_layout(struct nfs4_layout_stateid *ls)
> +{
> + struct nfsd_file *fl = xchg(&ls->ls_file, NULL);
> +
> + if (fl) {
> + if (!nfsd4_layout_ops[ls->ls_layout_type]->disable_recalls)
> + vfs_setlease(fl->nf_file, F_UNLCK, NULL,
> + (void **)&ls);
> + nfsd_file_put(fl);
> + }
> +}
> +
> static void
> nfsd4_free_layout_stateid(struct nfs4_stid *stid)
> {
> @@ -169,9 +181,7 @@ nfsd4_free_layout_stateid(struct nfs4_stid *stid)
> list_del_init(&ls->ls_perfile);
> spin_unlock(&fp->fi_lock);
>
> - if (!nfsd4_layout_ops[ls->ls_layout_type]->disable_recalls)
> - vfs_setlease(ls->ls_file->nf_file, F_UNLCK, NULL, (void **)&ls);
> - nfsd_file_put(ls->ls_file);
> + nfsd4_close_layout(ls);
>
> if (ls->ls_recalled)
> atomic_dec(&ls->ls_stid.sc_file->fi_lo_recalls);
> @@ -605,7 +615,7 @@ nfsd4_return_all_file_layouts(struct nfs4_client *clp, struct nfs4_file *fp)
> }
>
> static void
> -nfsd4_cb_layout_fail(struct nfs4_layout_stateid *ls)
> +nfsd4_cb_layout_fail(struct nfs4_layout_stateid *ls, struct nfsd_file *file)
> {
> struct nfs4_client *clp = ls->ls_stid.sc_client;
> char addr_str[INET6_ADDRSTRLEN];
> @@ -627,7 +637,7 @@ nfsd4_cb_layout_fail(struct nfs4_layout_stateid *ls)
>
> argv[0] = (char *)nfsd_recall_failed;
> argv[1] = addr_str;
> - argv[2] = ls->ls_file->nf_file->f_path.mnt->mnt_sb->s_id;
> + argv[2] = file->nf_file->f_path.mnt->mnt_sb->s_id;
> argv[3] = NULL;
>
> error = call_usermodehelper(nfsd_recall_failed, argv, envp,
> @@ -657,6 +667,7 @@ nfsd4_cb_layout_done(struct nfsd4_callback *cb, struct rpc_task *task)
> struct nfsd_net *nn;
> ktime_t now, cutoff;
> const struct nfsd4_layout_ops *ops;
> + struct nfsd_file *fl;
>
> trace_nfsd_cb_layout_done(&ls->ls_stid.sc_stateid, task);
> switch (task->tk_status) {
> @@ -688,12 +699,17 @@ nfsd4_cb_layout_done(struct nfsd4_callback *cb, struct rpc_task *task)
> * Unknown error or non-responding client, we'll need to fence.
> */
> trace_nfsd_layout_recall_fail(&ls->ls_stid.sc_stateid);
> -
> - ops = nfsd4_layout_ops[ls->ls_layout_type];
> - if (ops->fence_client)
> - ops->fence_client(ls);
> - else
> - nfsd4_cb_layout_fail(ls);
> + rcu_read_lock();
> + fl = nfsd_file_get(ls->ls_file);
> + rcu_read_unlock();
> + if (fl) {
> + ops = nfsd4_layout_ops[ls->ls_layout_type];
> + if (ops->fence_client)
> + ops->fence_client(ls, fl);
> + else
> + nfsd4_cb_layout_fail(ls, fl);
> + nfsd_file_put(fl);
> + }
> return 1;
> case -NFS4ERR_NOMATCHING_LAYOUT:
> trace_nfsd_layout_recall_done(&ls->ls_stid.sc_stateid);
> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> index 3d85c88ec4d7..d82ca209eb96 100644
> --- a/fs/nfsd/nfs4state.c
> +++ b/fs/nfsd/nfs4state.c
> @@ -1712,7 +1712,8 @@ void nfsd4_revoke_states(struct net *net, struct super_block *sb)
> unsigned int idhashval;
> unsigned int sc_types;
>
> - sc_types = NFS4_OPEN_STID | NFS4_LOCK_STID | NFS4_DELEG_STID;
> + sc_types = (NFS4_OPEN_STID | NFS4_LOCK_STID |
> + NFS4_DELEG_STID | NFS4_LAYOUT_STID);
>
> spin_lock(&nn->client_lock);
> for (idhashval = 0; idhashval < CLIENT_HASH_MASK; idhashval++) {
> @@ -1725,6 +1726,7 @@ void nfsd4_revoke_states(struct net *net, struct super_block *sb)
> if (stid) {
> struct nfs4_ol_stateid *stp;
> struct nfs4_delegation *dp;
> + struct nfs4_layout_stateid *ls;
>
> spin_unlock(&nn->client_lock);
> switch (stid->sc_type) {
> @@ -1780,6 +1782,10 @@ void nfsd4_revoke_states(struct net *net, struct super_block *sb)
> if (dp)
> revoke_delegation(dp);
> break;
> + case NFS4_LAYOUT_STID:
> + ls = layoutstateid(stid);
> + nfsd4_close_layout(ls);
> + break;
> }
> nfs4_put_stid(stid);
> spin_lock(&nn->client_lock);
> @@ -2859,17 +2865,25 @@ static int nfs4_show_layout(struct seq_file *s, struct nfs4_stid *st)
> struct nfsd_file *file;
>
> ls = container_of(st, struct nfs4_layout_stateid, ls_stid);
> - file = ls->ls_file;
> + rcu_read_lock();
> + file = nfsd_file_get(ls->ls_file);
> + rcu_read_unlock();
>
> - seq_printf(s, "- ");
> + seq_puts(s, "- ");
> nfs4_show_stateid(s, &st->sc_stateid);
> - seq_printf(s, ": { type: layout, ");
> + seq_puts(s, ": { type: layout");
>
> /* XXX: What else would be useful? */
>
> - nfs4_show_superblock(s, file);
> - seq_printf(s, ", ");
> - nfs4_show_fname(s, file);
> + if (file) {
> + seq_puts(s, ", ");
> + nfs4_show_superblock(s, file);
> + seq_puts(s, ", ");
> + nfs4_show_fname(s, file);
> + nfsd_file_put(file);
> + }
> + if (st->sc_status & NFS4_STID_ADMIN_REVOKED)
> + seq_puts(s, ", admin-revoked");
> seq_printf(s, " }\n");
>
> return 0;
> diff --git a/fs/nfsd/pnfs.h b/fs/nfsd/pnfs.h
> index de1e0dfed06a..f2777577865e 100644
> --- a/fs/nfsd/pnfs.h
> +++ b/fs/nfsd/pnfs.h
> @@ -37,7 +37,8 @@ struct nfsd4_layout_ops {
> __be32 (*proc_layoutcommit)(struct inode *inode,
> struct nfsd4_layoutcommit *lcp);
>
> - void (*fence_client)(struct nfs4_layout_stateid *ls);
> + void (*fence_client)(struct nfs4_layout_stateid *ls,
> + struct nfsd_file *file);
> };
>
> extern const struct nfsd4_layout_ops *nfsd4_layout_ops[];
> @@ -72,6 +73,7 @@ void nfsd4_setup_layout_type(struct svc_export *exp);
> void nfsd4_return_all_client_layouts(struct nfs4_client *);
> void nfsd4_return_all_file_layouts(struct nfs4_client *clp,
> struct nfs4_file *fp);
> +void nfsd4_close_layout(struct nfs4_layout_stateid *ls);
> int nfsd4_init_pnfs(void);
> void nfsd4_exit_pnfs(void);
> #else
> @@ -89,6 +91,9 @@ static inline void nfsd4_return_all_file_layouts(struct nfs4_client *clp,
> struct nfs4_file *fp)
> {
> }
> +static inline void nfsd4_close_layout(struct nfs4_layout_stateid *ls)
> +{
> +}
> static inline void nfsd4_exit_pnfs(void)
> {
> }
> --
> 2.42.1
>
--
Chuck Lever
On Mon, Nov 27, 2023 at 10:25:59AM -0500, Chuck Lever wrote:
> On Fri, Nov 24, 2023 at 11:28:46AM +1100, NeilBrown wrote:
> > When there is layout state on a filesystem that is being "unlocked" that
> > is now revoked, which involves closing the nfsd_file and releasing the
> > vfs lease.
> >
> > To avoid races, all users of ->ls_file - after the layout state has been
> > successfully created - now need to take a counted reference under
> > rcu_read_lock(). To support this, ->fence_client and
> > nfsd4_cb_layout_fail() now take a second argument being the nfsd_file.
> >
> > Signed-off-by: NeilBrown <[email protected]>
>
> Hi Neil, would you Cc: Christoph Hellwig <[email protected]> and Tom Haynes
> <[email protected]> to this patch next time you post this series?
Re-visiting. Did you send out a v5 of this series and I missed it?
> > ---
> > fs/nfsd/blocklayout.c | 4 ++--
> > fs/nfsd/nfs4layouts.c | 38 +++++++++++++++++++++++++++-----------
> > fs/nfsd/nfs4state.c | 28 +++++++++++++++++++++-------
> > fs/nfsd/pnfs.h | 7 ++++++-
> > 4 files changed, 56 insertions(+), 21 deletions(-)
> >
> > diff --git a/fs/nfsd/blocklayout.c b/fs/nfsd/blocklayout.c
> > index 46fd74d91ea9..3c040c81c77d 100644
> > --- a/fs/nfsd/blocklayout.c
> > +++ b/fs/nfsd/blocklayout.c
> > @@ -328,10 +328,10 @@ nfsd4_scsi_proc_layoutcommit(struct inode *inode,
> > }
> >
> > static void
> > -nfsd4_scsi_fence_client(struct nfs4_layout_stateid *ls)
> > +nfsd4_scsi_fence_client(struct nfs4_layout_stateid *ls, struct nfsd_file *file)
> > {
> > struct nfs4_client *clp = ls->ls_stid.sc_client;
> > - struct block_device *bdev = ls->ls_file->nf_file->f_path.mnt->mnt_sb->s_bdev;
> > + struct block_device *bdev = file->nf_file->f_path.mnt->mnt_sb->s_bdev;
> >
> > bdev->bd_disk->fops->pr_ops->pr_preempt(bdev, NFSD_MDS_PR_KEY,
> > nfsd4_scsi_pr_key(clp), 0, true);
> > diff --git a/fs/nfsd/nfs4layouts.c b/fs/nfsd/nfs4layouts.c
> > index 77656126ad2a..dbc52413ce57 100644
> > --- a/fs/nfsd/nfs4layouts.c
> > +++ b/fs/nfsd/nfs4layouts.c
> > @@ -152,6 +152,18 @@ void nfsd4_setup_layout_type(struct svc_export *exp)
> > #endif
> > }
> >
> > +void nfsd4_close_layout(struct nfs4_layout_stateid *ls)
> > +{
> > + struct nfsd_file *fl = xchg(&ls->ls_file, NULL);
> > +
> > + if (fl) {
> > + if (!nfsd4_layout_ops[ls->ls_layout_type]->disable_recalls)
> > + vfs_setlease(fl->nf_file, F_UNLCK, NULL,
> > + (void **)&ls);
> > + nfsd_file_put(fl);
> > + }
> > +}
> > +
> > static void
> > nfsd4_free_layout_stateid(struct nfs4_stid *stid)
> > {
> > @@ -169,9 +181,7 @@ nfsd4_free_layout_stateid(struct nfs4_stid *stid)
> > list_del_init(&ls->ls_perfile);
> > spin_unlock(&fp->fi_lock);
> >
> > - if (!nfsd4_layout_ops[ls->ls_layout_type]->disable_recalls)
> > - vfs_setlease(ls->ls_file->nf_file, F_UNLCK, NULL, (void **)&ls);
> > - nfsd_file_put(ls->ls_file);
> > + nfsd4_close_layout(ls);
> >
> > if (ls->ls_recalled)
> > atomic_dec(&ls->ls_stid.sc_file->fi_lo_recalls);
> > @@ -605,7 +615,7 @@ nfsd4_return_all_file_layouts(struct nfs4_client *clp, struct nfs4_file *fp)
> > }
> >
> > static void
> > -nfsd4_cb_layout_fail(struct nfs4_layout_stateid *ls)
> > +nfsd4_cb_layout_fail(struct nfs4_layout_stateid *ls, struct nfsd_file *file)
> > {
> > struct nfs4_client *clp = ls->ls_stid.sc_client;
> > char addr_str[INET6_ADDRSTRLEN];
> > @@ -627,7 +637,7 @@ nfsd4_cb_layout_fail(struct nfs4_layout_stateid *ls)
> >
> > argv[0] = (char *)nfsd_recall_failed;
> > argv[1] = addr_str;
> > - argv[2] = ls->ls_file->nf_file->f_path.mnt->mnt_sb->s_id;
> > + argv[2] = file->nf_file->f_path.mnt->mnt_sb->s_id;
> > argv[3] = NULL;
> >
> > error = call_usermodehelper(nfsd_recall_failed, argv, envp,
> > @@ -657,6 +667,7 @@ nfsd4_cb_layout_done(struct nfsd4_callback *cb, struct rpc_task *task)
> > struct nfsd_net *nn;
> > ktime_t now, cutoff;
> > const struct nfsd4_layout_ops *ops;
> > + struct nfsd_file *fl;
> >
> > trace_nfsd_cb_layout_done(&ls->ls_stid.sc_stateid, task);
> > switch (task->tk_status) {
> > @@ -688,12 +699,17 @@ nfsd4_cb_layout_done(struct nfsd4_callback *cb, struct rpc_task *task)
> > * Unknown error or non-responding client, we'll need to fence.
> > */
> > trace_nfsd_layout_recall_fail(&ls->ls_stid.sc_stateid);
> > -
> > - ops = nfsd4_layout_ops[ls->ls_layout_type];
> > - if (ops->fence_client)
> > - ops->fence_client(ls);
> > - else
> > - nfsd4_cb_layout_fail(ls);
> > + rcu_read_lock();
> > + fl = nfsd_file_get(ls->ls_file);
> > + rcu_read_unlock();
> > + if (fl) {
> > + ops = nfsd4_layout_ops[ls->ls_layout_type];
> > + if (ops->fence_client)
> > + ops->fence_client(ls, fl);
> > + else
> > + nfsd4_cb_layout_fail(ls, fl);
> > + nfsd_file_put(fl);
> > + }
> > return 1;
> > case -NFS4ERR_NOMATCHING_LAYOUT:
> > trace_nfsd_layout_recall_done(&ls->ls_stid.sc_stateid);
> > diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> > index 3d85c88ec4d7..d82ca209eb96 100644
> > --- a/fs/nfsd/nfs4state.c
> > +++ b/fs/nfsd/nfs4state.c
> > @@ -1712,7 +1712,8 @@ void nfsd4_revoke_states(struct net *net, struct super_block *sb)
> > unsigned int idhashval;
> > unsigned int sc_types;
> >
> > - sc_types = NFS4_OPEN_STID | NFS4_LOCK_STID | NFS4_DELEG_STID;
> > + sc_types = (NFS4_OPEN_STID | NFS4_LOCK_STID |
> > + NFS4_DELEG_STID | NFS4_LAYOUT_STID);
> >
> > spin_lock(&nn->client_lock);
> > for (idhashval = 0; idhashval < CLIENT_HASH_MASK; idhashval++) {
> > @@ -1725,6 +1726,7 @@ void nfsd4_revoke_states(struct net *net, struct super_block *sb)
> > if (stid) {
> > struct nfs4_ol_stateid *stp;
> > struct nfs4_delegation *dp;
> > + struct nfs4_layout_stateid *ls;
> >
> > spin_unlock(&nn->client_lock);
> > switch (stid->sc_type) {
> > @@ -1780,6 +1782,10 @@ void nfsd4_revoke_states(struct net *net, struct super_block *sb)
> > if (dp)
> > revoke_delegation(dp);
> > break;
> > + case NFS4_LAYOUT_STID:
> > + ls = layoutstateid(stid);
> > + nfsd4_close_layout(ls);
> > + break;
> > }
> > nfs4_put_stid(stid);
> > spin_lock(&nn->client_lock);
> > @@ -2859,17 +2865,25 @@ static int nfs4_show_layout(struct seq_file *s, struct nfs4_stid *st)
> > struct nfsd_file *file;
> >
> > ls = container_of(st, struct nfs4_layout_stateid, ls_stid);
> > - file = ls->ls_file;
> > + rcu_read_lock();
> > + file = nfsd_file_get(ls->ls_file);
> > + rcu_read_unlock();
> >
> > - seq_printf(s, "- ");
> > + seq_puts(s, "- ");
> > nfs4_show_stateid(s, &st->sc_stateid);
> > - seq_printf(s, ": { type: layout, ");
> > + seq_puts(s, ": { type: layout");
> >
> > /* XXX: What else would be useful? */
> >
> > - nfs4_show_superblock(s, file);
> > - seq_printf(s, ", ");
> > - nfs4_show_fname(s, file);
> > + if (file) {
> > + seq_puts(s, ", ");
> > + nfs4_show_superblock(s, file);
> > + seq_puts(s, ", ");
> > + nfs4_show_fname(s, file);
> > + nfsd_file_put(file);
> > + }
> > + if (st->sc_status & NFS4_STID_ADMIN_REVOKED)
> > + seq_puts(s, ", admin-revoked");
> > seq_printf(s, " }\n");
> >
> > return 0;
> > diff --git a/fs/nfsd/pnfs.h b/fs/nfsd/pnfs.h
> > index de1e0dfed06a..f2777577865e 100644
> > --- a/fs/nfsd/pnfs.h
> > +++ b/fs/nfsd/pnfs.h
> > @@ -37,7 +37,8 @@ struct nfsd4_layout_ops {
> > __be32 (*proc_layoutcommit)(struct inode *inode,
> > struct nfsd4_layoutcommit *lcp);
> >
> > - void (*fence_client)(struct nfs4_layout_stateid *ls);
> > + void (*fence_client)(struct nfs4_layout_stateid *ls,
> > + struct nfsd_file *file);
> > };
> >
> > extern const struct nfsd4_layout_ops *nfsd4_layout_ops[];
> > @@ -72,6 +73,7 @@ void nfsd4_setup_layout_type(struct svc_export *exp);
> > void nfsd4_return_all_client_layouts(struct nfs4_client *);
> > void nfsd4_return_all_file_layouts(struct nfs4_client *clp,
> > struct nfs4_file *fp);
> > +void nfsd4_close_layout(struct nfs4_layout_stateid *ls);
> > int nfsd4_init_pnfs(void);
> > void nfsd4_exit_pnfs(void);
> > #else
> > @@ -89,6 +91,9 @@ static inline void nfsd4_return_all_file_layouts(struct nfs4_client *clp,
> > struct nfs4_file *fp)
> > {
> > }
> > +static inline void nfsd4_close_layout(struct nfs4_layout_stateid *ls)
> > +{
> > +}
> > static inline void nfsd4_exit_pnfs(void)
> > {
> > }
> > --
> > 2.42.1
> >
>
> --
> Chuck Lever
>
--
Chuck Lever
On Sat, 23 Dec 2023, Chuck Lever wrote:
> On Mon, Nov 27, 2023 at 10:25:59AM -0500, Chuck Lever wrote:
> > On Fri, Nov 24, 2023 at 11:28:46AM +1100, NeilBrown wrote:
> > > When there is layout state on a filesystem that is being "unlocked" that
> > > is now revoked, which involves closing the nfsd_file and releasing the
> > > vfs lease.
> > >
> > > To avoid races, all users of ->ls_file - after the layout state has been
> > > successfully created - now need to take a counted reference under
> > > rcu_read_lock(). To support this, ->fence_client and
> > > nfsd4_cb_layout_fail() now take a second argument being the nfsd_file.
> > >
> > > Signed-off-by: NeilBrown <[email protected]>
> >
> > Hi Neil, would you Cc: Christoph Hellwig <[email protected]> and Tom Haynes
> > <[email protected]> to this patch next time you post this series?
>
> Re-visiting. Did you send out a v5 of this series and I missed it?
No you didn't miss anything. I think I got caught up with other
priorities. I'll hopefully repost early in the new year.
NeilBrown
>
>
> > > ---
> > > fs/nfsd/blocklayout.c | 4 ++--
> > > fs/nfsd/nfs4layouts.c | 38 +++++++++++++++++++++++++++-----------
> > > fs/nfsd/nfs4state.c | 28 +++++++++++++++++++++-------
> > > fs/nfsd/pnfs.h | 7 ++++++-
> > > 4 files changed, 56 insertions(+), 21 deletions(-)
> > >
> > > diff --git a/fs/nfsd/blocklayout.c b/fs/nfsd/blocklayout.c
> > > index 46fd74d91ea9..3c040c81c77d 100644
> > > --- a/fs/nfsd/blocklayout.c
> > > +++ b/fs/nfsd/blocklayout.c
> > > @@ -328,10 +328,10 @@ nfsd4_scsi_proc_layoutcommit(struct inode *inode,
> > > }
> > >
> > > static void
> > > -nfsd4_scsi_fence_client(struct nfs4_layout_stateid *ls)
> > > +nfsd4_scsi_fence_client(struct nfs4_layout_stateid *ls, struct nfsd_file *file)
> > > {
> > > struct nfs4_client *clp = ls->ls_stid.sc_client;
> > > - struct block_device *bdev = ls->ls_file->nf_file->f_path.mnt->mnt_sb->s_bdev;
> > > + struct block_device *bdev = file->nf_file->f_path.mnt->mnt_sb->s_bdev;
> > >
> > > bdev->bd_disk->fops->pr_ops->pr_preempt(bdev, NFSD_MDS_PR_KEY,
> > > nfsd4_scsi_pr_key(clp), 0, true);
> > > diff --git a/fs/nfsd/nfs4layouts.c b/fs/nfsd/nfs4layouts.c
> > > index 77656126ad2a..dbc52413ce57 100644
> > > --- a/fs/nfsd/nfs4layouts.c
> > > +++ b/fs/nfsd/nfs4layouts.c
> > > @@ -152,6 +152,18 @@ void nfsd4_setup_layout_type(struct svc_export *exp)
> > > #endif
> > > }
> > >
> > > +void nfsd4_close_layout(struct nfs4_layout_stateid *ls)
> > > +{
> > > + struct nfsd_file *fl = xchg(&ls->ls_file, NULL);
> > > +
> > > + if (fl) {
> > > + if (!nfsd4_layout_ops[ls->ls_layout_type]->disable_recalls)
> > > + vfs_setlease(fl->nf_file, F_UNLCK, NULL,
> > > + (void **)&ls);
> > > + nfsd_file_put(fl);
> > > + }
> > > +}
> > > +
> > > static void
> > > nfsd4_free_layout_stateid(struct nfs4_stid *stid)
> > > {
> > > @@ -169,9 +181,7 @@ nfsd4_free_layout_stateid(struct nfs4_stid *stid)
> > > list_del_init(&ls->ls_perfile);
> > > spin_unlock(&fp->fi_lock);
> > >
> > > - if (!nfsd4_layout_ops[ls->ls_layout_type]->disable_recalls)
> > > - vfs_setlease(ls->ls_file->nf_file, F_UNLCK, NULL, (void **)&ls);
> > > - nfsd_file_put(ls->ls_file);
> > > + nfsd4_close_layout(ls);
> > >
> > > if (ls->ls_recalled)
> > > atomic_dec(&ls->ls_stid.sc_file->fi_lo_recalls);
> > > @@ -605,7 +615,7 @@ nfsd4_return_all_file_layouts(struct nfs4_client *clp, struct nfs4_file *fp)
> > > }
> > >
> > > static void
> > > -nfsd4_cb_layout_fail(struct nfs4_layout_stateid *ls)
> > > +nfsd4_cb_layout_fail(struct nfs4_layout_stateid *ls, struct nfsd_file *file)
> > > {
> > > struct nfs4_client *clp = ls->ls_stid.sc_client;
> > > char addr_str[INET6_ADDRSTRLEN];
> > > @@ -627,7 +637,7 @@ nfsd4_cb_layout_fail(struct nfs4_layout_stateid *ls)
> > >
> > > argv[0] = (char *)nfsd_recall_failed;
> > > argv[1] = addr_str;
> > > - argv[2] = ls->ls_file->nf_file->f_path.mnt->mnt_sb->s_id;
> > > + argv[2] = file->nf_file->f_path.mnt->mnt_sb->s_id;
> > > argv[3] = NULL;
> > >
> > > error = call_usermodehelper(nfsd_recall_failed, argv, envp,
> > > @@ -657,6 +667,7 @@ nfsd4_cb_layout_done(struct nfsd4_callback *cb, struct rpc_task *task)
> > > struct nfsd_net *nn;
> > > ktime_t now, cutoff;
> > > const struct nfsd4_layout_ops *ops;
> > > + struct nfsd_file *fl;
> > >
> > > trace_nfsd_cb_layout_done(&ls->ls_stid.sc_stateid, task);
> > > switch (task->tk_status) {
> > > @@ -688,12 +699,17 @@ nfsd4_cb_layout_done(struct nfsd4_callback *cb, struct rpc_task *task)
> > > * Unknown error or non-responding client, we'll need to fence.
> > > */
> > > trace_nfsd_layout_recall_fail(&ls->ls_stid.sc_stateid);
> > > -
> > > - ops = nfsd4_layout_ops[ls->ls_layout_type];
> > > - if (ops->fence_client)
> > > - ops->fence_client(ls);
> > > - else
> > > - nfsd4_cb_layout_fail(ls);
> > > + rcu_read_lock();
> > > + fl = nfsd_file_get(ls->ls_file);
> > > + rcu_read_unlock();
> > > + if (fl) {
> > > + ops = nfsd4_layout_ops[ls->ls_layout_type];
> > > + if (ops->fence_client)
> > > + ops->fence_client(ls, fl);
> > > + else
> > > + nfsd4_cb_layout_fail(ls, fl);
> > > + nfsd_file_put(fl);
> > > + }
> > > return 1;
> > > case -NFS4ERR_NOMATCHING_LAYOUT:
> > > trace_nfsd_layout_recall_done(&ls->ls_stid.sc_stateid);
> > > diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> > > index 3d85c88ec4d7..d82ca209eb96 100644
> > > --- a/fs/nfsd/nfs4state.c
> > > +++ b/fs/nfsd/nfs4state.c
> > > @@ -1712,7 +1712,8 @@ void nfsd4_revoke_states(struct net *net, struct super_block *sb)
> > > unsigned int idhashval;
> > > unsigned int sc_types;
> > >
> > > - sc_types = NFS4_OPEN_STID | NFS4_LOCK_STID | NFS4_DELEG_STID;
> > > + sc_types = (NFS4_OPEN_STID | NFS4_LOCK_STID |
> > > + NFS4_DELEG_STID | NFS4_LAYOUT_STID);
> > >
> > > spin_lock(&nn->client_lock);
> > > for (idhashval = 0; idhashval < CLIENT_HASH_MASK; idhashval++) {
> > > @@ -1725,6 +1726,7 @@ void nfsd4_revoke_states(struct net *net, struct super_block *sb)
> > > if (stid) {
> > > struct nfs4_ol_stateid *stp;
> > > struct nfs4_delegation *dp;
> > > + struct nfs4_layout_stateid *ls;
> > >
> > > spin_unlock(&nn->client_lock);
> > > switch (stid->sc_type) {
> > > @@ -1780,6 +1782,10 @@ void nfsd4_revoke_states(struct net *net, struct super_block *sb)
> > > if (dp)
> > > revoke_delegation(dp);
> > > break;
> > > + case NFS4_LAYOUT_STID:
> > > + ls = layoutstateid(stid);
> > > + nfsd4_close_layout(ls);
> > > + break;
> > > }
> > > nfs4_put_stid(stid);
> > > spin_lock(&nn->client_lock);
> > > @@ -2859,17 +2865,25 @@ static int nfs4_show_layout(struct seq_file *s, struct nfs4_stid *st)
> > > struct nfsd_file *file;
> > >
> > > ls = container_of(st, struct nfs4_layout_stateid, ls_stid);
> > > - file = ls->ls_file;
> > > + rcu_read_lock();
> > > + file = nfsd_file_get(ls->ls_file);
> > > + rcu_read_unlock();
> > >
> > > - seq_printf(s, "- ");
> > > + seq_puts(s, "- ");
> > > nfs4_show_stateid(s, &st->sc_stateid);
> > > - seq_printf(s, ": { type: layout, ");
> > > + seq_puts(s, ": { type: layout");
> > >
> > > /* XXX: What else would be useful? */
> > >
> > > - nfs4_show_superblock(s, file);
> > > - seq_printf(s, ", ");
> > > - nfs4_show_fname(s, file);
> > > + if (file) {
> > > + seq_puts(s, ", ");
> > > + nfs4_show_superblock(s, file);
> > > + seq_puts(s, ", ");
> > > + nfs4_show_fname(s, file);
> > > + nfsd_file_put(file);
> > > + }
> > > + if (st->sc_status & NFS4_STID_ADMIN_REVOKED)
> > > + seq_puts(s, ", admin-revoked");
> > > seq_printf(s, " }\n");
> > >
> > > return 0;
> > > diff --git a/fs/nfsd/pnfs.h b/fs/nfsd/pnfs.h
> > > index de1e0dfed06a..f2777577865e 100644
> > > --- a/fs/nfsd/pnfs.h
> > > +++ b/fs/nfsd/pnfs.h
> > > @@ -37,7 +37,8 @@ struct nfsd4_layout_ops {
> > > __be32 (*proc_layoutcommit)(struct inode *inode,
> > > struct nfsd4_layoutcommit *lcp);
> > >
> > > - void (*fence_client)(struct nfs4_layout_stateid *ls);
> > > + void (*fence_client)(struct nfs4_layout_stateid *ls,
> > > + struct nfsd_file *file);
> > > };
> > >
> > > extern const struct nfsd4_layout_ops *nfsd4_layout_ops[];
> > > @@ -72,6 +73,7 @@ void nfsd4_setup_layout_type(struct svc_export *exp);
> > > void nfsd4_return_all_client_layouts(struct nfs4_client *);
> > > void nfsd4_return_all_file_layouts(struct nfs4_client *clp,
> > > struct nfs4_file *fp);
> > > +void nfsd4_close_layout(struct nfs4_layout_stateid *ls);
> > > int nfsd4_init_pnfs(void);
> > > void nfsd4_exit_pnfs(void);
> > > #else
> > > @@ -89,6 +91,9 @@ static inline void nfsd4_return_all_file_layouts(struct nfs4_client *clp,
> > > struct nfs4_file *fp)
> > > {
> > > }
> > > +static inline void nfsd4_close_layout(struct nfs4_layout_stateid *ls)
> > > +{
> > > +}
> > > static inline void nfsd4_exit_pnfs(void)
> > > {
> > > }
> > > --
> > > 2.42.1
> > >
> >
> > --
> > Chuck Lever
> >
>
> --
> Chuck Lever
>