2012-09-21 20:50:54

by Myklebust, Trond

[permalink] [raw]
Subject: [PATCH v2 01/26] NFS: Clean up the pNFS layoutget interface

Ensure that we do return errors from nfs4_proc_layoutget() and that we
don't mark the layout as having failed if the error was due to a
signal or resource problem on the client side.

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/nfs4proc.c | 14 +++++++++-----
fs/nfs/pnfs.c | 25 ++++++++++++++++---------
fs/nfs/pnfs.h | 4 ++--
include/linux/nfs_xdr.h | 1 -
4 files changed, 27 insertions(+), 17 deletions(-)

diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index 645c450..b738577 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -6286,7 +6286,8 @@ static const struct rpc_call_ops nfs4_layoutget_call_ops = {
.rpc_release = nfs4_layoutget_release,
};

-void nfs4_proc_layoutget(struct nfs4_layoutget *lgp, gfp_t gfp_flags)
+struct pnfs_layout_segment *
+nfs4_proc_layoutget(struct nfs4_layoutget *lgp, gfp_t gfp_flags)
{
struct nfs_server *server = NFS_SERVER(lgp->args.inode);
size_t max_pages = max_response_pages(server);
@@ -6303,6 +6304,7 @@ void nfs4_proc_layoutget(struct nfs4_layoutget *lgp, gfp_t gfp_flags)
.callback_data = lgp,
.flags = RPC_TASK_ASYNC,
};
+ struct pnfs_layout_segment *lseg = NULL;
int status = 0;

dprintk("--> %s\n", __func__);
@@ -6310,7 +6312,7 @@ void nfs4_proc_layoutget(struct nfs4_layoutget *lgp, gfp_t gfp_flags)
lgp->args.layout.pages = nfs4_alloc_pages(max_pages, gfp_flags);
if (!lgp->args.layout.pages) {
nfs4_layoutget_release(lgp);
- return;
+ return ERR_PTR(-ENOMEM);
}
lgp->args.layout.pglen = max_pages * PAGE_SIZE;

@@ -6319,15 +6321,17 @@ void nfs4_proc_layoutget(struct nfs4_layoutget *lgp, gfp_t gfp_flags)
nfs41_init_sequence(&lgp->args.seq_args, &lgp->res.seq_res, 0);
task = rpc_run_task(&task_setup_data);
if (IS_ERR(task))
- return;
+ return ERR_CAST(task);
status = nfs4_wait_for_completion_rpc_task(task);
if (status == 0)
status = task->tk_status;
if (status == 0)
- status = pnfs_layout_process(lgp);
+ lseg = pnfs_layout_process(lgp);
rpc_put_task(task);
dprintk("<-- %s status=%d\n", __func__, status);
- return;
+ if (status)
+ return ERR_PTR(status);
+ return lseg;
}

static void
diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 2e00fea..3a7ac97 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -582,7 +582,7 @@ send_layoutget(struct pnfs_layout_hdr *lo,
struct inode *ino = lo->plh_inode;
struct nfs_server *server = NFS_SERVER(ino);
struct nfs4_layoutget *lgp;
- struct pnfs_layout_segment *lseg = NULL;
+ struct pnfs_layout_segment *lseg;

dprintk("--> %s\n", __func__);

@@ -599,16 +599,22 @@ send_layoutget(struct pnfs_layout_hdr *lo,
lgp->args.type = server->pnfs_curr_ld->id;
lgp->args.inode = ino;
lgp->args.ctx = get_nfs_open_context(ctx);
- lgp->lsegpp = &lseg;
lgp->gfp_flags = gfp_flags;

/* Synchronously retrieve layout information from server and
* store in lseg.
*/
- nfs4_proc_layoutget(lgp, gfp_flags);
- if (!lseg) {
- /* remember that LAYOUTGET failed and suspend trying */
- set_bit(lo_fail_bit(range->iomode), &lo->plh_flags);
+ lseg = nfs4_proc_layoutget(lgp, gfp_flags);
+ if (IS_ERR(lseg)) {
+ switch (PTR_ERR(lseg)) {
+ case -ENOMEM:
+ case -ERESTARTSYS:
+ break;
+ default:
+ /* remember that LAYOUTGET failed and suspend trying */
+ set_bit(lo_fail_bit(range->iomode), &lo->plh_flags);
+ }
+ return NULL;
}

return lseg;
@@ -1096,7 +1102,7 @@ out_unlock:
}
EXPORT_SYMBOL_GPL(pnfs_update_layout);

-int
+struct pnfs_layout_segment *
pnfs_layout_process(struct nfs4_layoutget *lgp)
{
struct pnfs_layout_hdr *lo = NFS_I(lgp->args.inode)->layout;
@@ -1129,7 +1135,7 @@ pnfs_layout_process(struct nfs4_layoutget *lgp)
}
init_lseg(lo, lseg);
lseg->pls_range = res->range;
- *lgp->lsegpp = get_lseg(lseg);
+ get_lseg(lseg);
pnfs_insert_layout(lo, lseg);

if (res->return_on_close) {
@@ -1140,8 +1146,9 @@ pnfs_layout_process(struct nfs4_layoutget *lgp)
/* Done processing layoutget. Set the layout stateid */
pnfs_set_layout_stateid(lo, &res->stateid, false);
spin_unlock(&ino->i_lock);
+ return lseg;
out:
- return status;
+ return ERR_PTR(status);

out_forget_reply:
spin_unlock(&ino->i_lock);
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index 745aa1b..d51ef88 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -172,7 +172,7 @@ extern int nfs4_proc_getdevicelist(struct nfs_server *server,
struct pnfs_devicelist *devlist);
extern int nfs4_proc_getdeviceinfo(struct nfs_server *server,
struct pnfs_device *dev);
-extern void nfs4_proc_layoutget(struct nfs4_layoutget *lgp, gfp_t gfp_flags);
+extern struct pnfs_layout_segment* nfs4_proc_layoutget(struct nfs4_layoutget *lgp, gfp_t gfp_flags);
extern int nfs4_proc_layoutreturn(struct nfs4_layoutreturn *lrp);

/* pnfs.c */
@@ -192,7 +192,7 @@ void pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *, struct nfs_page
int pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc);
bool pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio, struct nfs_page *prev, struct nfs_page *req);
void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg);
-int pnfs_layout_process(struct nfs4_layoutget *lgp);
+struct pnfs_layout_segment *pnfs_layout_process(struct nfs4_layoutget *lgp);
void pnfs_free_lseg_list(struct list_head *tmp_list);
void pnfs_destroy_layout(struct nfs_inode *);
void pnfs_destroy_all_layouts(struct nfs_client *);
diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h
index be9cf3c..5da789f 100644
--- a/include/linux/nfs_xdr.h
+++ b/include/linux/nfs_xdr.h
@@ -251,7 +251,6 @@ struct nfs4_layoutget_res {
struct nfs4_layoutget {
struct nfs4_layoutget_args args;
struct nfs4_layoutget_res res;
- struct pnfs_layout_segment **lsegpp;
gfp_t gfp_flags;
};

--
1.7.11.4



2012-09-24 14:55:57

by Myklebust, Trond

[permalink] [raw]
Subject: RE: [PATCH v2 06/26] NFSv4.1: Add helpers for setting/reading the I/O fail bit

> -----Original Message-----
> From: Andy Adamson [mailto:[email protected]]
> Sent: Monday, September 24, 2012 10:48 AM
> To: Myklebust, Trond
> Cc: [email protected]
> Subject: Re: [PATCH v2 06/26] NFSv4.1: Add helpers for setting/reading the
> I/O fail bit
>
> On Fri, Sep 21, 2012 at 4:50 PM, Trond Myklebust
> <[email protected]> wrote:
> > ...and make them local to the pnfs.c file.
> >
> > Signed-off-by: Trond Myklebust <[email protected]>
> > ---
> > fs/nfs/pnfs.c | 37 ++++++++++++++++++++++++++-----------
> > fs/nfs/pnfs.h | 6 ------
> > 2 files changed, 26 insertions(+), 17 deletions(-)
> >
> > diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index 498af87..f85cc4c
> > 100644
> > --- a/fs/nfs/pnfs.c
> > +++ b/fs/nfs/pnfs.c
> > @@ -238,6 +238,27 @@ pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo)
> > }
> > }
> >
> > +static int
> > +pnfs_iomode_to_fail_bit(u32 iomode)
> > +{
> > + return iomode == IOMODE_RW ?
> > + NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED; }
> > +
> > +static void
> > +pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode) {
> > + set_bit(pnfs_iomode_to_fail_bit(iomode), &lo->plh_flags);
> > + dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__,
> > + iomode == IOMODE_RW ? "RW" : "READ"); }
> > +
> > +static bool
> > +pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode)
>
>
> Nit - this is a bad name. These bits test if LAYOUTGET succeeds or not. They
> do not test whether IO succeeded or failed. I suggest
> pnfs_layoutget_test_failed.

Blocks uses them to signal that the I/O actually failed. See the use of pnfs_set_lo_fail(). Then there is the use in _pnfs_return_layout(), where it simply signals an ENOMEM error.

> -->Andy
>
> > +{
> > + return test_bit(pnfs_iomode_to_fail_bit(iomode),
> > +&lo->plh_flags) != 0; }
> > +
> > static void
> > init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment
> > *lseg) { @@ -612,7 +633,7 @@ send_layoutget(struct pnfs_layout_hdr
> > *lo,
> > break;
> > default:
> > /* remember that LAYOUTGET failed and suspend trying */
> > - set_bit(lo_fail_bit(range->iomode), &lo->plh_flags);
> > + pnfs_layout_io_set_failed(lo, range->iomode);
> > }
> > return NULL;
> > }
> > @@ -669,8 +690,8 @@ _pnfs_return_layout(struct inode *ino)
> > lrp = kzalloc(sizeof(*lrp), GFP_KERNEL);
> > if (unlikely(lrp == NULL)) {
> > status = -ENOMEM;
> > - set_bit(NFS_LAYOUT_RW_FAILED, &lo->plh_flags);
> > - set_bit(NFS_LAYOUT_RO_FAILED, &lo->plh_flags);
> > + pnfs_layout_io_set_failed(lo, IOMODE_RW);
> > + pnfs_layout_io_set_failed(lo, IOMODE_READ);
> > pnfs_clear_layout_returned(lo);
> > pnfs_put_layout_hdr(lo);
> > goto out;
> > @@ -1046,7 +1067,7 @@ pnfs_update_layout(struct inode *ino,
> > }
> >
> > /* if LAYOUTGET already failed once we don't try again */
> > - if (test_bit(lo_fail_bit(iomode), &nfsi->layout->plh_flags))
> > + if (pnfs_layout_io_test_failed(nfsi->layout, iomode))
> > goto out_unlock;
> >
> > /* Check to see if the layout for the given range already
> > exists */ @@ -1581,13 +1602,7 @@ static void
> > pnfs_list_write_lseg(struct inode *inode, struct list_head *listp)
> >
> > void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg) {
> > - if (lseg->pls_range.iomode == IOMODE_RW) {
> > - dprintk("%s Setting layout IOMODE_RW fail bit\n", __func__);
> > - set_bit(lo_fail_bit(IOMODE_RW), &lseg->pls_layout->plh_flags);
> > - } else {
> > - dprintk("%s Setting layout IOMODE_READ fail bit\n", __func__);
> > - set_bit(lo_fail_bit(IOMODE_READ), &lseg->pls_layout->plh_flags);
> > - }
> > + pnfs_layout_io_set_failed(lseg->pls_layout,
> > + lseg->pls_range.iomode);
> > }
> > EXPORT_SYMBOL_GPL(pnfs_set_lo_fail);
> >
> > diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h index 0495879..e3eb7d1
> > 100644
> > --- a/fs/nfs/pnfs.h
> > +++ b/fs/nfs/pnfs.h
> > @@ -274,12 +274,6 @@ pnfs_test_layout_returned(struct
> pnfs_layout_hdr *lo)
> > return test_bit(NFS_LAYOUT_RETURNED, &lo->plh_flags); }
> >
> > -static inline int lo_fail_bit(u32 iomode) -{
> > - return iomode == IOMODE_RW ?
> > - NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED;
> > -}
> > -
> > static inline struct pnfs_layout_segment * pnfs_get_lseg(struct
> > pnfs_layout_segment *lseg) {
> > --
> > 1.7.11.4
> >
> > --
> > To unsubscribe from this list: send the line "unsubscribe linux-nfs"
> > in the body of a message to [email protected] More
> majordomo
> > info at http://vger.kernel.org/majordomo-info.html

2012-09-21 20:52:45

by Myklebust, Trond

[permalink] [raw]
Subject: [PATCH v2 20/26] NFSv4.1: Balance pnfs_layout_hdr refcount in pnfs_layout_(insert|remove)_lseg

Ensure that the reference count for pnfs_layout_hdr reverts to the
original value after a call to pnfs_layout_remove_lseg().

Note that the caller is expected to hold a reference to the struct
pnfs_layout_hdr.

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/pnfs.c | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index ed8d674..9c5320c 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -295,8 +295,6 @@ static void pnfs_free_lseg(struct pnfs_layout_segment *lseg)
struct inode *ino = lseg->pls_layout->plh_inode;

NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
- /* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */
- pnfs_put_layout_hdr(NFS_I(ino)->layout);
}

static void
@@ -307,6 +305,8 @@ pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo,

WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
list_del_init(&lseg->pls_list);
+ /* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */
+ atomic_dec(&lo->plh_refcount);
if (list_empty(&lo->plh_segs))
set_bit(NFS_LAYOUT_DESTROYED, &lo->plh_flags);
rpc_wake_up(&NFS_SERVER(inode)->roc_rpcwaitq);
@@ -327,9 +327,11 @@ pnfs_put_lseg(struct pnfs_layout_segment *lseg)
lo = lseg->pls_layout;
inode = lo->plh_inode;
if (atomic_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) {
+ pnfs_get_layout_hdr(lo);
pnfs_layout_remove_lseg(lo, lseg);
spin_unlock(&inode->i_lock);
pnfs_free_lseg(lseg);
+ pnfs_put_layout_hdr(lo);
}
}
EXPORT_SYMBOL_GPL(pnfs_put_lseg);
--
1.7.11.4


2012-09-21 20:53:01

by Myklebust, Trond

[permalink] [raw]
Subject: [PATCH v2 23/26] squash! NFSv4.1: Remove redundant reference to the pnfs_layout_hdr

Ensure that pnfs_find_alloc_layout() always returns a reference
to the pnfs_layout_hdr, which will be matched by the final call to
pnfs_put_layout_hdr() in pnfs_update_layout().

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/pnfs.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 196b8bb..bdb70cc 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -884,8 +884,8 @@ pnfs_find_alloc_layout(struct inode *ino,
if (nfsi->layout) {
if (test_bit(NFS_LAYOUT_DESTROYED, &nfsi->layout->plh_flags))
return NULL;
- else
- return nfsi->layout;
+ pnfs_get_layout_hdr(nfsi->layout);
+ return nfsi->layout;
}
spin_unlock(&ino->i_lock);
new = alloc_init_layout_hdr(ino, ctx, gfp_flags);
--
1.7.11.4


2012-09-24 14:47:31

by Andy Adamson

[permalink] [raw]
Subject: Re: [PATCH v2 06/26] NFSv4.1: Add helpers for setting/reading the I/O fail bit

On Fri, Sep 21, 2012 at 4:50 PM, Trond Myklebust
<[email protected]> wrote:
> ...and make them local to the pnfs.c file.
>
> Signed-off-by: Trond Myklebust <[email protected]>
> ---
> fs/nfs/pnfs.c | 37 ++++++++++++++++++++++++++-----------
> fs/nfs/pnfs.h | 6 ------
> 2 files changed, 26 insertions(+), 17 deletions(-)
>
> diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
> index 498af87..f85cc4c 100644
> --- a/fs/nfs/pnfs.c
> +++ b/fs/nfs/pnfs.c
> @@ -238,6 +238,27 @@ pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo)
> }
> }
>
> +static int
> +pnfs_iomode_to_fail_bit(u32 iomode)
> +{
> + return iomode == IOMODE_RW ?
> + NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED;
> +}
> +
> +static void
> +pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode)
> +{
> + set_bit(pnfs_iomode_to_fail_bit(iomode), &lo->plh_flags);
> + dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__,
> + iomode == IOMODE_RW ? "RW" : "READ");
> +}
> +
> +static bool
> +pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode)


Nit - this is a bad name. These bits test if LAYOUTGET succeeds or
not. They do not test whether IO succeeded or failed. I suggest
pnfs_layoutget_test_failed.

-->Andy

> +{
> + return test_bit(pnfs_iomode_to_fail_bit(iomode), &lo->plh_flags) != 0;
> +}
> +
> static void
> init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg)
> {
> @@ -612,7 +633,7 @@ send_layoutget(struct pnfs_layout_hdr *lo,
> break;
> default:
> /* remember that LAYOUTGET failed and suspend trying */
> - set_bit(lo_fail_bit(range->iomode), &lo->plh_flags);
> + pnfs_layout_io_set_failed(lo, range->iomode);
> }
> return NULL;
> }
> @@ -669,8 +690,8 @@ _pnfs_return_layout(struct inode *ino)
> lrp = kzalloc(sizeof(*lrp), GFP_KERNEL);
> if (unlikely(lrp == NULL)) {
> status = -ENOMEM;
> - set_bit(NFS_LAYOUT_RW_FAILED, &lo->plh_flags);
> - set_bit(NFS_LAYOUT_RO_FAILED, &lo->plh_flags);
> + pnfs_layout_io_set_failed(lo, IOMODE_RW);
> + pnfs_layout_io_set_failed(lo, IOMODE_READ);
> pnfs_clear_layout_returned(lo);
> pnfs_put_layout_hdr(lo);
> goto out;
> @@ -1046,7 +1067,7 @@ pnfs_update_layout(struct inode *ino,
> }
>
> /* if LAYOUTGET already failed once we don't try again */
> - if (test_bit(lo_fail_bit(iomode), &nfsi->layout->plh_flags))
> + if (pnfs_layout_io_test_failed(nfsi->layout, iomode))
> goto out_unlock;
>
> /* Check to see if the layout for the given range already exists */
> @@ -1581,13 +1602,7 @@ static void pnfs_list_write_lseg(struct inode *inode, struct list_head *listp)
>
> void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg)
> {
> - if (lseg->pls_range.iomode == IOMODE_RW) {
> - dprintk("%s Setting layout IOMODE_RW fail bit\n", __func__);
> - set_bit(lo_fail_bit(IOMODE_RW), &lseg->pls_layout->plh_flags);
> - } else {
> - dprintk("%s Setting layout IOMODE_READ fail bit\n", __func__);
> - set_bit(lo_fail_bit(IOMODE_READ), &lseg->pls_layout->plh_flags);
> - }
> + pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode);
> }
> EXPORT_SYMBOL_GPL(pnfs_set_lo_fail);
>
> diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
> index 0495879..e3eb7d1 100644
> --- a/fs/nfs/pnfs.h
> +++ b/fs/nfs/pnfs.h
> @@ -274,12 +274,6 @@ pnfs_test_layout_returned(struct pnfs_layout_hdr *lo)
> return test_bit(NFS_LAYOUT_RETURNED, &lo->plh_flags);
> }
>
> -static inline int lo_fail_bit(u32 iomode)
> -{
> - return iomode == IOMODE_RW ?
> - NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED;
> -}
> -
> static inline struct pnfs_layout_segment *
> pnfs_get_lseg(struct pnfs_layout_segment *lseg)
> {
> --
> 1.7.11.4
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
> the body of a message to [email protected]
> More majordomo info at http://vger.kernel.org/majordomo-info.html

2012-09-21 20:53:07

by Myklebust, Trond

[permalink] [raw]
Subject: [PATCH v2 24/26] NFSv4.1: Get rid of the NFS_LAYOUT_DESTROYED state

We already have a mechanism for blocking LAYOUTGET by means of the
plh_block_lgets counter. The only "service" that NFS_LAYOUT_DESTROYED
provides at this point is to block layoutget once the layout segment
list is empty, which basically means that you have to wait until
the pnfs_layout_hdr is destroyed before you can do pNFS on that file
again.

This patch enables the reuse of the pnfs_layout_hdr if the layout
segment list is empty.

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/nfs4filelayout.c | 3 +--
fs/nfs/pnfs.c | 9 +--------
fs/nfs/pnfs.h | 7 -------
3 files changed, 2 insertions(+), 17 deletions(-)

diff --git a/fs/nfs/nfs4filelayout.c b/fs/nfs/nfs4filelayout.c
index 6cce57e..52d8472 100644
--- a/fs/nfs/nfs4filelayout.c
+++ b/fs/nfs/nfs4filelayout.c
@@ -279,8 +279,7 @@ filelayout_reset_to_mds(struct pnfs_layout_segment *lseg)
{
struct nfs4_deviceid_node *node = FILELAYOUT_DEVID_NODE(lseg);

- return pnfs_test_layout_destroyed(lseg->pls_layout) ||
- filelayout_test_devid_unavailable(node);
+ return filelayout_test_devid_unavailable(node);
}

/*
diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index bdb70cc..8ba7c35 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -306,8 +306,6 @@ pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo,
list_del_init(&lseg->pls_list);
/* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */
atomic_dec(&lo->plh_refcount);
- if (list_empty(&lo->plh_segs))
- set_bit(NFS_LAYOUT_DESTROYED, &lo->plh_flags);
rpc_wake_up(&NFS_SERVER(inode)->roc_rpcwaitq);
}

@@ -438,10 +436,8 @@ pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,

dprintk("%s:Begin lo %p\n", __func__, lo);

- if (list_empty(&lo->plh_segs)) {
- set_bit(NFS_LAYOUT_DESTROYED, &lo->plh_flags);
+ if (list_empty(&lo->plh_segs))
return 0;
- }
list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
if (!recall_range ||
should_free_lseg(&lseg->pls_range, recall_range)) {
@@ -560,7 +556,6 @@ pnfs_layoutgets_blocked(struct pnfs_layout_hdr *lo, nfs4_stateid *stateid,
(int)(lo->plh_barrier - be32_to_cpu(stateid->seqid)) >= 0)
return true;
return lo->plh_block_lgets ||
- test_bit(NFS_LAYOUT_DESTROYED, &lo->plh_flags) ||
test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags) ||
(list_empty(&lo->plh_segs) &&
(atomic_read(&lo->plh_outstanding) > lget));
@@ -882,8 +877,6 @@ pnfs_find_alloc_layout(struct inode *ino,
dprintk("%s Begin ino=%p layout=%p\n", __func__, ino, nfsi->layout);

if (nfsi->layout) {
- if (test_bit(NFS_LAYOUT_DESTROYED, &nfsi->layout->plh_flags))
- return NULL;
pnfs_get_layout_hdr(nfsi->layout);
return nfsi->layout;
}
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index aacda7f..92f6ce6 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -62,7 +62,6 @@ enum {
NFS_LAYOUT_RW_FAILED, /* get rw layout failed stop trying */
NFS_LAYOUT_BULK_RECALL, /* bulk recall affecting layout */
NFS_LAYOUT_ROC, /* some lseg had roc bit set */
- NFS_LAYOUT_DESTROYED, /* no new use of layout allowed */
NFS_LAYOUT_RETURNED, /* layout has already been returned */
};

@@ -278,12 +277,6 @@ pnfs_test_layout_returned(struct pnfs_layout_hdr *lo)
return test_bit(NFS_LAYOUT_RETURNED, &lo->plh_flags);
}

-static inline bool
-pnfs_test_layout_destroyed(struct pnfs_layout_hdr *lo)
-{
- return test_bit(NFS_LAYOUT_DESTROYED, &lo->plh_flags);
-}
-
static inline struct pnfs_layout_segment *
pnfs_get_lseg(struct pnfs_layout_segment *lseg)
{
--
1.7.11.4


2012-09-21 20:52:04

by Myklebust, Trond

[permalink] [raw]
Subject: [PATCH v2 13/26] NFSv4.1: Get rid of pNFS layout state "NFS_LAYOUT_INVALID"

In all cases where we set NFS_LAYOUT_INVALID, we also set NFS_LAYOUT_DESTROYED.
Furthermore, in all cases where we test for NFS_LAYOUT_INVALID, we should
also be testing for NFS_LAYOUT_DESTROYED, since the latter means that
we hold no valid layout segments.
Ergo the two are redundant.

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/nfs4filelayout.c | 4 +---
fs/nfs/nfs4filelayout.h | 6 ------
fs/nfs/pnfs.h | 7 ++++++-
3 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/fs/nfs/nfs4filelayout.c b/fs/nfs/nfs4filelayout.c
index dac2162..6cce57e 100644
--- a/fs/nfs/nfs4filelayout.c
+++ b/fs/nfs/nfs4filelayout.c
@@ -190,8 +190,6 @@ static int filelayout_async_handle_error(struct rpc_task *task,
* i/o and all i/o waiting on the slot table to the MDS until
* layout is destroyed and a new valid layout is obtained.
*/
- set_bit(NFS_LAYOUT_INVALID,
- &NFS_I(inode)->layout->plh_flags);
pnfs_destroy_layout(NFS_I(inode));
rpc_wake_up(&tbl->slot_tbl_waitq);
goto reset;
@@ -281,7 +279,7 @@ filelayout_reset_to_mds(struct pnfs_layout_segment *lseg)
{
struct nfs4_deviceid_node *node = FILELAYOUT_DEVID_NODE(lseg);

- return filelayout_test_layout_invalid(lseg->pls_layout) ||
+ return pnfs_test_layout_destroyed(lseg->pls_layout) ||
filelayout_test_devid_unavailable(node);
}

diff --git a/fs/nfs/nfs4filelayout.h b/fs/nfs/nfs4filelayout.h
index 10b0f13..dca47d78 100644
--- a/fs/nfs/nfs4filelayout.h
+++ b/fs/nfs/nfs4filelayout.h
@@ -129,12 +129,6 @@ filelayout_mark_devid_invalid(struct nfs4_deviceid_node *node)
}

static inline bool
-filelayout_test_layout_invalid(struct pnfs_layout_hdr *lo)
-{
- return test_bit(NFS_LAYOUT_INVALID, &lo->plh_flags);
-}
-
-static inline bool
filelayout_test_devid_invalid(struct nfs4_deviceid_node *node)
{
return test_bit(NFS_DEVICEID_INVALID, &node->flags);
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index aa9fa1b..aacda7f 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -63,7 +63,6 @@ enum {
NFS_LAYOUT_BULK_RECALL, /* bulk recall affecting layout */
NFS_LAYOUT_ROC, /* some lseg had roc bit set */
NFS_LAYOUT_DESTROYED, /* no new use of layout allowed */
- NFS_LAYOUT_INVALID, /* layout is being destroyed */
NFS_LAYOUT_RETURNED, /* layout has already been returned */
};

@@ -279,6 +278,12 @@ pnfs_test_layout_returned(struct pnfs_layout_hdr *lo)
return test_bit(NFS_LAYOUT_RETURNED, &lo->plh_flags);
}

+static inline bool
+pnfs_test_layout_destroyed(struct pnfs_layout_hdr *lo)
+{
+ return test_bit(NFS_LAYOUT_DESTROYED, &lo->plh_flags);
+}
+
static inline struct pnfs_layout_segment *
pnfs_get_lseg(struct pnfs_layout_segment *lseg)
{
--
1.7.11.4


2012-09-21 20:51:58

by Myklebust, Trond

[permalink] [raw]
Subject: [PATCH v2 12/26] NFSv4.1: Simplify the pNFS return-on-close code

Confine it to the nfs4_do_close() code.

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/nfs4_fs.h | 2 +-
fs/nfs/nfs4proc.c | 6 ++----
fs/nfs/nfs4state.c | 7 ++-----
3 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/fs/nfs/nfs4_fs.h b/fs/nfs/nfs4_fs.h
index 71d407f..9cacc13 100644
--- a/fs/nfs/nfs4_fs.h
+++ b/fs/nfs/nfs4_fs.h
@@ -223,7 +223,7 @@ extern int nfs4_proc_exchange_id(struct nfs_client *clp, struct rpc_cred *cred);
extern int nfs4_destroy_clientid(struct nfs_client *clp);
extern int nfs4_init_clientid(struct nfs_client *, struct rpc_cred *);
extern int nfs41_init_clientid(struct nfs_client *, struct rpc_cred *);
-extern int nfs4_do_close(struct nfs4_state *state, gfp_t gfp_mask, int wait, bool roc);
+extern int nfs4_do_close(struct nfs4_state *state, gfp_t gfp_mask, int wait);
extern int nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *fhandle);
extern int nfs4_proc_fs_locations(struct rpc_clnt *, struct inode *, const struct qstr *,
struct nfs4_fs_locations *, struct page *);
diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index 49d5c7d..d9d67b3 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -2204,7 +2204,7 @@ static const struct rpc_call_ops nfs4_close_ops = {
*
* NOTE: Caller must be holding the sp->so_owner semaphore!
*/
-int nfs4_do_close(struct nfs4_state *state, gfp_t gfp_mask, int wait, bool roc)
+int nfs4_do_close(struct nfs4_state *state, gfp_t gfp_mask, int wait)
{
struct nfs_server *server = NFS_SERVER(state->inode);
struct nfs4_closedata *calldata;
@@ -2240,7 +2240,7 @@ int nfs4_do_close(struct nfs4_state *state, gfp_t gfp_mask, int wait, bool roc)
calldata->res.fattr = &calldata->fattr;
calldata->res.seqid = calldata->arg.seqid;
calldata->res.server = server;
- calldata->roc = roc;
+ calldata->roc = pnfs_roc(state->inode);
nfs_sb_active(calldata->inode->i_sb);

msg.rpc_argp = &calldata->arg;
@@ -2257,8 +2257,6 @@ int nfs4_do_close(struct nfs4_state *state, gfp_t gfp_mask, int wait, bool roc)
out_free_calldata:
kfree(calldata);
out:
- if (roc)
- pnfs_roc_release(state->inode);
nfs4_put_open_state(state);
nfs4_put_state_owner(sp);
return status;
diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c
index fc6cfe6..a5331ec 100644
--- a/fs/nfs/nfs4state.c
+++ b/fs/nfs/nfs4state.c
@@ -729,11 +729,8 @@ static void __nfs4_close(struct nfs4_state *state,
if (!call_close) {
nfs4_put_open_state(state);
nfs4_put_state_owner(owner);
- } else {
- bool roc = pnfs_roc(state->inode);
-
- nfs4_do_close(state, gfp_mask, wait, roc);
- }
+ } else
+ nfs4_do_close(state, gfp_mask, wait);
}

void nfs4_close_state(struct nfs4_state *state, fmode_t fmode)
--
1.7.11.4


2012-09-21 20:52:34

by Myklebust, Trond

[permalink] [raw]
Subject: [PATCH v2 18/26] NFSv4.1: Clean up the removal of pnfs_layout_hdr from the server list

Move the code into pnfs_free_layout_hdr(), and add checks to
get_layout_by_fh_locked to ensure that they don't reference a layout
that is being freed.

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/callback_proc.c | 19 ++++++++++++++++++-
fs/nfs/pnfs.c | 29 ++++++++++-------------------
2 files changed, 28 insertions(+), 20 deletions(-)

diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c
index 24252fe..76b4a7a 100644
--- a/fs/nfs/callback_proc.c
+++ b/fs/nfs/callback_proc.c
@@ -122,7 +122,15 @@ static struct pnfs_layout_hdr * get_layout_by_fh_locked(struct nfs_client *clp,
ino = igrab(lo->plh_inode);
if (!ino)
continue;
+ spin_lock(&ino->i_lock);
+ /* Is this layout in the process of being freed? */
+ if (NFS_I(ino)->layout != lo) {
+ spin_unlock(&ino->i_lock);
+ iput(ino);
+ continue;
+ }
pnfs_get_layout_hdr(lo);
+ spin_unlock(&ino->i_lock);
return lo;
}
}
@@ -196,9 +204,18 @@ static u32 initiate_bulk_draining(struct nfs_client *clp,
continue;

list_for_each_entry(lo, &server->layouts, plh_layouts) {
- if (!igrab(lo->plh_inode))
+ ino = igrab(lo->plh_inode);
+ if (ino)
+ continue;
+ spin_lock(&ino->i_lock);
+ /* Is this layout in the process of being freed? */
+ if (NFS_I(ino)->layout != lo) {
+ spin_unlock(&ino->i_lock);
+ iput(ino);
continue;
+ }
pnfs_get_layout_hdr(lo);
+ spin_unlock(&ino->i_lock);
BUG_ON(!list_empty(&lo->plh_bulk_recall));
list_add(&lo->plh_bulk_recall, &recall_list);
}
diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 1827cae..557faaf 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -207,7 +207,16 @@ pnfs_alloc_layout_hdr(struct inode *ino, gfp_t gfp_flags)
static void
pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo)
{
- struct pnfs_layoutdriver_type *ld = NFS_SERVER(lo->plh_inode)->pnfs_curr_ld;
+ struct nfs_server *server = NFS_SERVER(lo->plh_inode);
+ struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld;
+
+ if (!list_empty(&lo->plh_layouts)) {
+ struct nfs_client *clp = server->nfs_client;
+
+ spin_lock(&clp->cl_lock);
+ list_del_init(&lo->plh_layouts);
+ spin_unlock(&clp->cl_lock);
+ }
put_rpccred(lo->plh_lc_cred);
return ld->alloc_layout_hdr ? ld->free_layout_hdr(lo) : kfree(lo);
}
@@ -217,7 +226,6 @@ pnfs_detach_layout_hdr(struct pnfs_layout_hdr *lo)
{
struct nfs_inode *nfsi = NFS_I(lo->plh_inode);
dprintk("%s: freeing layout cache %p\n", __func__, lo);
- BUG_ON(!list_empty(&lo->plh_layouts));
nfsi->layout = NULL;
/* Reset MDS Threshold I/O counters */
nfsi->write_io = 0;
@@ -455,22 +463,10 @@ void
pnfs_free_lseg_list(struct list_head *free_me)
{
struct pnfs_layout_segment *lseg, *tmp;
- struct pnfs_layout_hdr *lo;

if (list_empty(free_me))
return;

- lo = list_first_entry(free_me, struct pnfs_layout_segment,
- pls_list)->pls_layout;
-
- if (test_bit(NFS_LAYOUT_DESTROYED, &lo->plh_flags)) {
- struct nfs_client *clp;
-
- clp = NFS_SERVER(lo->plh_inode)->nfs_client;
- spin_lock(&clp->cl_lock);
- list_del_init(&lo->plh_layouts);
- spin_unlock(&clp->cl_lock);
- }
list_for_each_entry_safe(lseg, tmp, free_me, pls_list) {
list_del(&lseg->pls_list);
free_lseg(lseg);
@@ -1121,11 +1117,6 @@ pnfs_update_layout(struct inode *ino,
arg.length = PAGE_CACHE_ALIGN(arg.length);

lseg = send_layoutget(lo, ctx, &arg, gfp_flags);
- if (!lseg && first) {
- spin_lock(&clp->cl_lock);
- list_del_init(&lo->plh_layouts);
- spin_unlock(&clp->cl_lock);
- }
atomic_dec(&lo->plh_outstanding);
out:
pnfs_free_lseg_list(&tmp_list);
--
1.7.11.4


2012-09-21 20:51:46

by Myklebust, Trond

[permalink] [raw]
Subject: [PATCH v2 10/26] NFSv4.1: pnfs_layout_io_test_failed must clear invalid lsegs before a retry

If pnfs_layout_io_test_failed() authorises a retry of the failed layoutgets,
it should first clear the existing layout segments so that we start afresh.

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/pnfs.c | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index cbbb0fc..80aaaba 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -256,7 +256,8 @@ pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode)
}

static bool
-pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode)
+pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode,
+ struct list_head *head)
{
unsigned long start, end;
if (test_bit(pnfs_iomode_to_fail_bit(iomode), &lo->plh_flags) == 0)
@@ -267,6 +268,7 @@ pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode)
/* It is time to retry the failed layoutgets */
clear_bit(NFS_LAYOUT_RW_FAILED, &lo->plh_flags);
clear_bit(NFS_LAYOUT_RO_FAILED, &lo->plh_flags);
+ pnfs_mark_matching_lsegs_invalid(lo, head, NULL);
return false;
}
return true;
@@ -1058,6 +1060,7 @@ pnfs_update_layout(struct inode *ino,
struct nfs_client *clp = server->nfs_client;
struct pnfs_layout_hdr *lo;
struct pnfs_layout_segment *lseg = NULL;
+ LIST_HEAD(tmp_list);
bool first = false;

if (!pnfs_enabled_sb(NFS_SERVER(ino)))
@@ -1081,7 +1084,7 @@ pnfs_update_layout(struct inode *ino,
}

/* if LAYOUTGET already failed once we don't try again */
- if (pnfs_layout_io_test_failed(nfsi->layout, iomode))
+ if (pnfs_layout_io_test_failed(nfsi->layout, iomode, &tmp_list))
goto out_unlock;

/* Check to see if the layout for the given range already exists */
@@ -1127,6 +1130,7 @@ pnfs_update_layout(struct inode *ino,
}
atomic_dec(&lo->plh_outstanding);
out:
+ pnfs_free_lseg_list(&tmp_list);
pnfs_put_layout_hdr(lo);
dprintk("%s end, state 0x%lx lseg %p\n", __func__,
nfsi->layout ? nfsi->layout->plh_flags : -1, lseg);
--
1.7.11.4


2012-09-24 19:50:04

by Myklebust, Trond

[permalink] [raw]
Subject: Re: [PATCH v2 10/26] NFSv4.1: pnfs_layout_io_test_failed must clear invalid lsegs before a retry

T24gTW9uLCAyMDEyLTA5LTI0IGF0IDEwOjQ3IC0wNDAwLCBBbmR5IEFkYW1zb24gd3JvdGU6DQo+
IE9uIEZyaSwgU2VwIDIxLCAyMDEyIGF0IDQ6NTAgUE0sIFRyb25kIE15a2xlYnVzdA0KPiA8VHJv
bmQuTXlrbGVidXN0QG5ldGFwcC5jb20+IHdyb3RlOg0KPiA+IElmIHBuZnNfbGF5b3V0X2lvX3Rl
c3RfZmFpbGVkKCkgYXV0aG9yaXNlcyBhIHJldHJ5IG9mIHRoZSBmYWlsZWQgbGF5b3V0Z2V0cywN
Cj4gPiBpdCBzaG91bGQgZmlyc3QgY2xlYXIgdGhlIGV4aXN0aW5nIGxheW91dCBzZWdtZW50cyBz
byB0aGF0IHdlIHN0YXJ0IGFmcmVzaC4NCj4gPg0KPiA+IFNpZ25lZC1vZmYtYnk6IFRyb25kIE15
a2xlYnVzdCA8VHJvbmQuTXlrbGVidXN0QG5ldGFwcC5jb20+DQo+ID4gLS0tDQo+ID4gIGZzL25m
cy9wbmZzLmMgfCA4ICsrKysrKy0tDQo+ID4gIDEgZmlsZSBjaGFuZ2VkLCA2IGluc2VydGlvbnMo
KyksIDIgZGVsZXRpb25zKC0pDQo+ID4NCj4gPiBkaWZmIC0tZ2l0IGEvZnMvbmZzL3BuZnMuYyBi
L2ZzL25mcy9wbmZzLmMNCj4gPiBpbmRleCBjYmJiMGZjLi44MGFhYWJhIDEwMDY0NA0KPiA+IC0t
LSBhL2ZzL25mcy9wbmZzLmMNCj4gPiArKysgYi9mcy9uZnMvcG5mcy5jDQo+ID4gQEAgLTI1Niw3
ICsyNTYsOCBAQCBwbmZzX2xheW91dF9pb19zZXRfZmFpbGVkKHN0cnVjdCBwbmZzX2xheW91dF9o
ZHIgKmxvLCB1MzIgaW9tb2RlKQ0KPiA+ICB9DQo+ID4NCj4gPiAgc3RhdGljIGJvb2wNCj4gPiAt
cG5mc19sYXlvdXRfaW9fdGVzdF9mYWlsZWQoc3RydWN0IHBuZnNfbGF5b3V0X2hkciAqbG8sIHUz
MiBpb21vZGUpDQo+ID4gK3BuZnNfbGF5b3V0X2lvX3Rlc3RfZmFpbGVkKHN0cnVjdCBwbmZzX2xh
eW91dF9oZHIgKmxvLCB1MzIgaW9tb2RlLA0KPiA+ICsgICAgICAgICAgICAgICBzdHJ1Y3QgbGlz
dF9oZWFkICpoZWFkKQ0KPiA+ICB7DQo+ID4gICAgICAgICB1bnNpZ25lZCBsb25nIHN0YXJ0LCBl
bmQ7DQo+ID4gICAgICAgICBpZiAodGVzdF9iaXQocG5mc19pb21vZGVfdG9fZmFpbF9iaXQoaW9t
b2RlKSwgJmxvLT5wbGhfZmxhZ3MpID09IDApDQo+ID4gQEAgLTI2Nyw2ICsyNjgsNyBAQCBwbmZz
X2xheW91dF9pb190ZXN0X2ZhaWxlZChzdHJ1Y3QgcG5mc19sYXlvdXRfaGRyICpsbywgdTMyIGlv
bW9kZSkNCj4gPiAgICAgICAgICAgICAgICAgLyogSXQgaXMgdGltZSB0byByZXRyeSB0aGUgZmFp
bGVkIGxheW91dGdldHMgKi8NCj4gPiAgICAgICAgICAgICAgICAgY2xlYXJfYml0KE5GU19MQVlP
VVRfUldfRkFJTEVELCAmbG8tPnBsaF9mbGFncyk7DQo+ID4gICAgICAgICAgICAgICAgIGNsZWFy
X2JpdChORlNfTEFZT1VUX1JPX0ZBSUxFRCwgJmxvLT5wbGhfZmxhZ3MpOw0KPiA+ICsgICAgICAg
ICAgICAgICBwbmZzX21hcmtfbWF0Y2hpbmdfbHNlZ3NfaW52YWxpZChsbywgaGVhZCwgTlVMTCk7
DQo+IA0KPiBCeSBjbGVhcmluZyBhbGwgaW9tb2RlIGZhaWxlZCBiaXRzIGFuZCBwYXNzaW5nIGEg
TlVMTCBhcyB0aGUNCj4gcmVjYWxsX3JhbmdlIHRvIHBuZnNfbWFya19tYXRjaGluZ19sc2Vncywg
eW91IGFyZSBkaXNjYXJkaW5nIGxheW91dHMNCj4gbm90IGF1dGhvcml6ZWQgYnkgdGhlIGlvbW9k
ZSBwYXNzZWQgaW50byBwbmZzX2xheW91dF9pb190ZXN0X2ZhaWxlZC4NCj4gV2h5IG5vdCBjbGVh
ciBqdXN0IHRoZSBmYWlsZWQgYml0IGFuZCBzZXR1cCBhIHBuZnNfbGF5b3V0X3JhbmdlDQo+IHN0
cnVjdHVyZSAoZm9yIHRoZSB3aG9sZSBmaWxlKSBhbmQgcGFzcyB0aGUgYXV0aG9yaXplZCBmYWls
ZWQgaW9tb2RlDQo+IHRvIHBuZnNfbWFya19tYXRjaGluZ19sc2VncyBzbyBhcyB0byByZW1vdmUg
b25seSB0aGUgZmFpbGVkIGlvbW9kZQ0KPiBsYXlvdXQgc2VnbWVudHM/DQoNClRoZXJlIHdlcmUg
b3RoZXIgcmVhc29ucyBmb3IgY2hhbmdpbmcgdGhpcyBwYXJ0IG9mIHRoZSBjb2RlIHRvby4gVGhl
DQptYWluIHByb2JsZW0gaXMgdGhhdCBpZiB0aGUgZmlyc3QgbGF5b3V0Z2V0IGZhaWxzLCBzbyB0
aGF0IHdlIG5ldmVyDQphc3NvY2lhdGUgYW4gbHNlZyB0byB0aGUgcG5mc19sYXlvdXRfaGRyLCB0
aGVuIHRoZSBwbGhfcmVmY291bnQgd2lsbCBnbw0KdG8gemVybywgYW5kIHdlIGVuZCB1cCBub3Qg
Y2FjaGluZyB0aGUgZmFpbCBiaXQuDQoNClZlcnNpb24gMyBvZiB0aGVzZSBwYXRjaGVzIHRoZXJl
Zm9yZSBjaGFuZ2UgdGhlIGNvZGUgaW4gdGhpcyByZWdpb24gc28NCnRoYXQgd2Uga2VlcCBhIHJl
ZmVyZW5jZSB0byB0aGUgcG5mc19sYXlvdXRfaGRyIHdpdGggdGhlIGZhaWwgYml0LiBUaGVuDQpp
dCBjbGVhcnMgb3V0IHRoZSBleGlzdGluZyBsc2VncyAobWF0Y2hpbmcgaW9tb2RlcykgaW4NCnBu
ZnNfbGF5b3V0X2lvX3NldF9mYWlsZWQuDQoNCj4gLS0+QW5keQ0KPiANCj4gDQo+ID4gICAgICAg
ICAgICAgICAgIHJldHVybiBmYWxzZTsNCj4gPiAgICAgICAgIH0NCj4gPiAgICAgICAgIHJldHVy
biB0cnVlOw0KPiA+IEBAIC0xMDU4LDYgKzEwNjAsNyBAQCBwbmZzX3VwZGF0ZV9sYXlvdXQoc3Ry
dWN0IGlub2RlICppbm8sDQo+ID4gICAgICAgICBzdHJ1Y3QgbmZzX2NsaWVudCAqY2xwID0gc2Vy
dmVyLT5uZnNfY2xpZW50Ow0KPiA+ICAgICAgICAgc3RydWN0IHBuZnNfbGF5b3V0X2hkciAqbG87
DQo+ID4gICAgICAgICBzdHJ1Y3QgcG5mc19sYXlvdXRfc2VnbWVudCAqbHNlZyA9IE5VTEw7DQo+
ID4gKyAgICAgICBMSVNUX0hFQUQodG1wX2xpc3QpOw0KPiA+ICAgICAgICAgYm9vbCBmaXJzdCA9
IGZhbHNlOw0KPiA+DQo+ID4gICAgICAgICBpZiAoIXBuZnNfZW5hYmxlZF9zYihORlNfU0VSVkVS
KGlubykpKQ0KPiA+IEBAIC0xMDgxLDcgKzEwODQsNyBAQCBwbmZzX3VwZGF0ZV9sYXlvdXQoc3Ry
dWN0IGlub2RlICppbm8sDQo+ID4gICAgICAgICB9DQo+ID4NCj4gPiAgICAgICAgIC8qIGlmIExB
WU9VVEdFVCBhbHJlYWR5IGZhaWxlZCBvbmNlIHdlIGRvbid0IHRyeSBhZ2FpbiAqLw0KPiA+IC0g
ICAgICAgaWYgKHBuZnNfbGF5b3V0X2lvX3Rlc3RfZmFpbGVkKG5mc2ktPmxheW91dCwgaW9tb2Rl
KSkNCj4gPiArICAgICAgIGlmIChwbmZzX2xheW91dF9pb190ZXN0X2ZhaWxlZChuZnNpLT5sYXlv
dXQsIGlvbW9kZSwgJnRtcF9saXN0KSkNCj4gPiAgICAgICAgICAgICAgICAgZ290byBvdXRfdW5s
b2NrOw0KPiA+DQo+ID4gICAgICAgICAvKiBDaGVjayB0byBzZWUgaWYgdGhlIGxheW91dCBmb3Ig
dGhlIGdpdmVuIHJhbmdlIGFscmVhZHkgZXhpc3RzICovDQo+ID4gQEAgLTExMjcsNiArMTEzMCw3
IEBAIHBuZnNfdXBkYXRlX2xheW91dChzdHJ1Y3QgaW5vZGUgKmlubywNCj4gPiAgICAgICAgIH0N
Cj4gPiAgICAgICAgIGF0b21pY19kZWMoJmxvLT5wbGhfb3V0c3RhbmRpbmcpOw0KPiA+ICBvdXQ6
DQo+ID4gKyAgICAgICBwbmZzX2ZyZWVfbHNlZ19saXN0KCZ0bXBfbGlzdCk7DQo+ID4gICAgICAg
ICBwbmZzX3B1dF9sYXlvdXRfaGRyKGxvKTsNCj4gPiAgICAgICAgIGRwcmludGsoIiVzIGVuZCwg
c3RhdGUgMHglbHggbHNlZyAlcFxuIiwgX19mdW5jX18sDQo+ID4gICAgICAgICAgICAgICAgIG5m
c2ktPmxheW91dCA/IG5mc2ktPmxheW91dC0+cGxoX2ZsYWdzIDogLTEsIGxzZWcpOw0KPiA+IC0t
DQo+ID4gMS43LjExLjQNCj4gPg0KPiA+IC0tDQo+ID4gVG8gdW5zdWJzY3JpYmUgZnJvbSB0aGlz
IGxpc3Q6IHNlbmQgdGhlIGxpbmUgInVuc3Vic2NyaWJlIGxpbnV4LW5mcyIgaW4NCj4gPiB0aGUg
Ym9keSBvZiBhIG1lc3NhZ2UgdG8gbWFqb3Jkb21vQHZnZXIua2VybmVsLm9yZw0KPiA+IE1vcmUg
bWFqb3Jkb21vIGluZm8gYXQgIGh0dHA6Ly92Z2VyLmtlcm5lbC5vcmcvbWFqb3Jkb21vLWluZm8u
aHRtbA0KDQotLSANClRyb25kIE15a2xlYnVzdA0KTGludXggTkZTIGNsaWVudCBtYWludGFpbmVy
DQoNCk5ldEFwcA0KVHJvbmQuTXlrbGVidXN0QG5ldGFwcC5jb20NCnd3dy5uZXRhcHAuY29tDQo=

2012-09-21 20:51:52

by Myklebust, Trond

[permalink] [raw]
Subject: [PATCH v2 11/26] NFSv4.1: Fix a race in the pNFS return-on-close code

If we sleep after dropping the inode->i_lock, then we are no longer
atomic with respect to the rpc_wake_up() call in pnfs_layout_remove_lseg().

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/nfs4proc.c | 8 +++-----
fs/nfs/pnfs.c | 22 ++++++++++++----------
fs/nfs/pnfs.h | 4 ++--
3 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index 87702a0..49d5c7d 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -2137,6 +2137,7 @@ static void nfs4_close_prepare(struct rpc_task *task, void *data)
{
struct nfs4_closedata *calldata = data;
struct nfs4_state *state = calldata->state;
+ struct inode *inode = calldata->inode;
int call_close = 0;

dprintk("%s: begin!\n", __func__);
@@ -2170,16 +2171,13 @@ static void nfs4_close_prepare(struct rpc_task *task, void *data)
if (calldata->arg.fmode == 0) {
task->tk_msg.rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_CLOSE];
if (calldata->roc &&
- pnfs_roc_drain(calldata->inode, &calldata->roc_barrier)) {
- rpc_sleep_on(&NFS_SERVER(calldata->inode)->roc_rpcwaitq,
- task, NULL);
+ pnfs_roc_drain(inode, &calldata->roc_barrier, task))
goto out;
- }
}

nfs_fattr_init(calldata->res.fattr);
calldata->timestamp = jiffies;
- if (nfs4_setup_sequence(NFS_SERVER(calldata->inode),
+ if (nfs4_setup_sequence(NFS_SERVER(inode),
&calldata->arg.seq_args,
&calldata->res.seq_res,
task))
diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 80aaaba..f3acb68 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -777,27 +777,29 @@ void pnfs_roc_set_barrier(struct inode *ino, u32 barrier)
spin_unlock(&ino->i_lock);
}

-bool pnfs_roc_drain(struct inode *ino, u32 *barrier)
+bool pnfs_roc_drain(struct inode *ino, u32 *barrier, struct rpc_task *task)
{
struct nfs_inode *nfsi = NFS_I(ino);
+ struct pnfs_layout_hdr *lo;
struct pnfs_layout_segment *lseg;
+ u32 current_seqid;
bool found = false;

spin_lock(&ino->i_lock);
list_for_each_entry(lseg, &nfsi->layout->plh_segs, pls_list)
if (test_bit(NFS_LSEG_ROC, &lseg->pls_flags)) {
+ rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL);
found = true;
- break;
+ goto out;
}
- if (!found) {
- struct pnfs_layout_hdr *lo = nfsi->layout;
- u32 current_seqid = be32_to_cpu(lo->plh_stateid.seqid);
+ lo = nfsi->layout;
+ current_seqid = be32_to_cpu(lo->plh_stateid.seqid);

- /* Since close does not return a layout stateid for use as
- * a barrier, we choose the worst-case barrier.
- */
- *barrier = current_seqid + atomic_read(&lo->plh_outstanding);
- }
+ /* Since close does not return a layout stateid for use as
+ * a barrier, we choose the worst-case barrier.
+ */
+ *barrier = current_seqid + atomic_read(&lo->plh_outstanding);
+out:
spin_unlock(&ino->i_lock);
return found;
}
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index 9735031..aa9fa1b 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -210,7 +210,7 @@ int pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
bool pnfs_roc(struct inode *ino);
void pnfs_roc_release(struct inode *ino);
void pnfs_roc_set_barrier(struct inode *ino, u32 barrier);
-bool pnfs_roc_drain(struct inode *ino, u32 *barrier);
+bool pnfs_roc_drain(struct inode *ino, u32 *barrier, struct rpc_task *task);
void pnfs_set_layoutcommit(struct nfs_write_data *wdata);
void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data);
int pnfs_layoutcommit_inode(struct inode *inode, bool sync);
@@ -442,7 +442,7 @@ pnfs_roc_set_barrier(struct inode *ino, u32 barrier)
}

static inline bool
-pnfs_roc_drain(struct inode *ino, u32 *barrier)
+pnfs_roc_drain(struct inode *ino, u32 *barrier, struct rpc_task *task)
{
return false;
}
--
1.7.11.4


2012-09-21 20:52:09

by Myklebust, Trond

[permalink] [raw]
Subject: [PATCH v2 14/26] NFSv4.1: reset the inode MDS threshold counters on layout destruction

Instead of resetting the inode MDS threshold counters when we mark
the layout for destruction, do it as part of freeing the layout.

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/pnfs.c | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index f3acb68..8633b78 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -215,9 +215,13 @@ pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo)
static void
destroy_layout_hdr(struct pnfs_layout_hdr *lo)
{
+ struct nfs_inode *nfsi = NFS_I(lo->plh_inode);
dprintk("%s: freeing layout cache %p\n", __func__, lo);
BUG_ON(!list_empty(&lo->plh_layouts));
- NFS_I(lo->plh_inode)->layout = NULL;
+ nfsi->layout = NULL;
+ /* Reset MDS Threshold I/O counters */
+ nfsi->write_io = 0;
+ nfsi->read_io = 0;
pnfs_free_layout_hdr(lo);
}

@@ -436,9 +440,6 @@ pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
dprintk("%s:Begin lo %p\n", __func__, lo);

if (list_empty(&lo->plh_segs)) {
- /* Reset MDS Threshold I/O counters */
- NFS_I(lo->plh_inode)->write_io = 0;
- NFS_I(lo->plh_inode)->read_io = 0;
if (!test_and_set_bit(NFS_LAYOUT_DESTROYED, &lo->plh_flags))
pnfs_put_layout_hdr_locked(lo);
return 0;
--
1.7.11.4


2012-09-21 20:51:28

by Myklebust, Trond

[permalink] [raw]
Subject: [PATCH v2 07/26] NFSv4.1: Retry pNFS after a 2 minute timeout

If we had to fall back to read/write through MDS, then assume that we should
retry pNFS after a suitable timeout period.
The following patch sets a timeout of 2 minutes.

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/pnfs.c | 15 ++++++++++++++-
fs/nfs/pnfs.h | 1 +
2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index f85cc4c..6656cb4 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -35,6 +35,7 @@
#include "iostat.h"

#define NFSDBG_FACILITY NFSDBG_PNFS
+#define PNFS_LAYOUTGET_RETRY_TIMEOUT (120*HZ)

/* Locking:
*
@@ -248,6 +249,7 @@ pnfs_iomode_to_fail_bit(u32 iomode)
static void
pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode)
{
+ lo->plh_retry_timestamp = jiffies;
set_bit(pnfs_iomode_to_fail_bit(iomode), &lo->plh_flags);
dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__,
iomode == IOMODE_RW ? "RW" : "READ");
@@ -256,7 +258,18 @@ pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode)
static bool
pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode)
{
- return test_bit(pnfs_iomode_to_fail_bit(iomode), &lo->plh_flags) != 0;
+ unsigned long start, end;
+ if (test_bit(pnfs_iomode_to_fail_bit(iomode), &lo->plh_flags) == 0)
+ return false;
+ end = jiffies;
+ start = end - PNFS_LAYOUTGET_RETRY_TIMEOUT;
+ if (!time_in_range(lo->plh_retry_timestamp, start, end)) {
+ /* It is time to retry the failed layoutgets */
+ clear_bit(NFS_LAYOUT_RW_FAILED, &lo->plh_flags);
+ clear_bit(NFS_LAYOUT_RO_FAILED, &lo->plh_flags);
+ return false;
+ }
+ return true;
}

static void
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index e3eb7d1..bc8e500 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -140,6 +140,7 @@ struct pnfs_layout_hdr {
atomic_t plh_outstanding; /* number of RPCs out */
unsigned long plh_block_lgets; /* block LAYOUTGET if >0 */
u32 plh_barrier; /* ignore lower seqids */
+ unsigned long plh_retry_timestamp;
unsigned long plh_flags;
loff_t plh_lwb; /* last write byte for layoutcommit */
struct rpc_cred *plh_lc_cred; /* layoutcommit cred */
--
1.7.11.4


2012-09-21 20:53:19

by Myklebust, Trond

[permalink] [raw]
Subject: [PATCH v2 26/26] NFSv4.1: Remove the NFS_LAYOUT_RETURNED state

It serves no purpose that the test for whether or not we have valid
layout segments doesn't already serve.

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/pnfs.c | 7 +------
fs/nfs/pnfs.h | 19 -------------------
2 files changed, 1 insertion(+), 25 deletions(-)

diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index a5edf27..f0872f5 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -662,7 +662,7 @@ _pnfs_return_layout(struct inode *ino)

spin_lock(&ino->i_lock);
lo = nfsi->layout;
- if (!lo || pnfs_test_layout_returned(lo)) {
+ if (!lo) {
spin_unlock(&ino->i_lock);
dprintk("NFS: %s no layout to return\n", __func__);
goto out;
@@ -680,7 +680,6 @@ _pnfs_return_layout(struct inode *ino)
goto out;
}
lo->plh_block_lgets++;
- pnfs_mark_layout_returned(lo);
spin_unlock(&ino->i_lock);
pnfs_free_lseg_list(&tmp_list);

@@ -691,7 +690,6 @@ _pnfs_return_layout(struct inode *ino)
status = -ENOMEM;
pnfs_layout_io_set_failed(lo, IOMODE_RW);
pnfs_layout_io_set_failed(lo, IOMODE_READ);
- pnfs_clear_layout_returned(lo);
pnfs_put_layout_hdr(lo);
goto out;
}
@@ -1084,9 +1082,6 @@ pnfs_update_layout(struct inode *ino,
if (list_empty(&lo->plh_segs))
first = true;

- /* Enable LAYOUTRETURNs */
- pnfs_clear_layout_returned(lo);
-
spin_unlock(&ino->i_lock);
if (first) {
/* The lo must be on the clp list if there is any
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index 92f6ce6..6cede2c 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -62,7 +62,6 @@ enum {
NFS_LAYOUT_RW_FAILED, /* get rw layout failed stop trying */
NFS_LAYOUT_BULK_RECALL, /* bulk recall affecting layout */
NFS_LAYOUT_ROC, /* some lseg had roc bit set */
- NFS_LAYOUT_RETURNED, /* layout has already been returned */
};

enum layoutdriver_policy_flags {
@@ -259,24 +258,6 @@ void nfs4_mark_deviceid_unavailable(struct nfs4_deviceid_node *node);
bool nfs4_test_deviceid_unavailable(struct nfs4_deviceid_node *node);
void nfs4_deviceid_purge_client(const struct nfs_client *);

-static inline void
-pnfs_mark_layout_returned(struct pnfs_layout_hdr *lo)
-{
- set_bit(NFS_LAYOUT_RETURNED, &lo->plh_flags);
-}
-
-static inline void
-pnfs_clear_layout_returned(struct pnfs_layout_hdr *lo)
-{
- clear_bit(NFS_LAYOUT_RETURNED, &lo->plh_flags);
-}
-
-static inline bool
-pnfs_test_layout_returned(struct pnfs_layout_hdr *lo)
-{
- return test_bit(NFS_LAYOUT_RETURNED, &lo->plh_flags);
-}
-
static inline struct pnfs_layout_segment *
pnfs_get_lseg(struct pnfs_layout_segment *lseg)
{
--
1.7.11.4


2012-09-21 20:51:05

by Myklebust, Trond

[permalink] [raw]
Subject: [PATCH v2 03/26] NFSv4.1: Cleanup; add "pnfs_" prefix to get_layout_hdr() and put_layout_hdr()

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/callback_proc.c | 8 ++++----
fs/nfs/nfs4proc.c | 2 +-
fs/nfs/pnfs.c | 30 +++++++++++++++---------------
fs/nfs/pnfs.h | 4 ++--
4 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c
index 57b8bda..24252fe 100644
--- a/fs/nfs/callback_proc.c
+++ b/fs/nfs/callback_proc.c
@@ -122,7 +122,7 @@ static struct pnfs_layout_hdr * get_layout_by_fh_locked(struct nfs_client *clp,
ino = igrab(lo->plh_inode);
if (!ino)
continue;
- get_layout_hdr(lo);
+ pnfs_get_layout_hdr(lo);
return lo;
}
}
@@ -166,7 +166,7 @@ static u32 initiate_file_draining(struct nfs_client *clp,
pnfs_set_layout_stateid(lo, &args->cbl_stateid, true);
spin_unlock(&ino->i_lock);
pnfs_free_lseg_list(&free_me_list);
- put_layout_hdr(lo);
+ pnfs_put_layout_hdr(lo);
iput(ino);
return rv;
}
@@ -198,7 +198,7 @@ static u32 initiate_bulk_draining(struct nfs_client *clp,
list_for_each_entry(lo, &server->layouts, plh_layouts) {
if (!igrab(lo->plh_inode))
continue;
- get_layout_hdr(lo);
+ pnfs_get_layout_hdr(lo);
BUG_ON(!list_empty(&lo->plh_bulk_recall));
list_add(&lo->plh_bulk_recall, &recall_list);
}
@@ -216,7 +216,7 @@ static u32 initiate_bulk_draining(struct nfs_client *clp,
list_del_init(&lo->plh_bulk_recall);
spin_unlock(&ino->i_lock);
pnfs_free_lseg_list(&free_me_list);
- put_layout_hdr(lo);
+ pnfs_put_layout_hdr(lo);
iput(ino);
}
return rv;
diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index b738577..20f1229 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -6375,7 +6375,7 @@ static void nfs4_layoutreturn_release(void *calldata)
struct nfs4_layoutreturn *lrp = calldata;

dprintk("--> %s\n", __func__);
- put_layout_hdr(lrp->args.layout);
+ pnfs_put_layout_hdr(lrp->args.layout);
kfree(calldata);
dprintk("<-- %s\n", __func__);
}
diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index aea2e52..512c863 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -190,7 +190,7 @@ EXPORT_SYMBOL_GPL(pnfs_unregister_layoutdriver);

/* Need to hold i_lock if caller does not already hold reference */
void
-get_layout_hdr(struct pnfs_layout_hdr *lo)
+pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo)
{
atomic_inc(&lo->plh_refcount);
}
@@ -221,14 +221,14 @@ destroy_layout_hdr(struct pnfs_layout_hdr *lo)
}

static void
-put_layout_hdr_locked(struct pnfs_layout_hdr *lo)
+pnfs_put_layout_hdr_locked(struct pnfs_layout_hdr *lo)
{
if (atomic_dec_and_test(&lo->plh_refcount))
destroy_layout_hdr(lo);
}

void
-put_layout_hdr(struct pnfs_layout_hdr *lo)
+pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo)
{
struct inode *inode = lo->plh_inode;

@@ -254,8 +254,8 @@ static void free_lseg(struct pnfs_layout_segment *lseg)
struct inode *ino = lseg->pls_layout->plh_inode;

NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
- /* Matched by get_layout_hdr in pnfs_insert_layout */
- put_layout_hdr(NFS_I(ino)->layout);
+ /* Matched by pnfs_get_layout_hdr in pnfs_insert_layout */
+ pnfs_put_layout_hdr(NFS_I(ino)->layout);
}

static void
@@ -268,7 +268,7 @@ put_lseg_common(struct pnfs_layout_segment *lseg)
if (list_empty(&lseg->pls_layout->plh_segs)) {
set_bit(NFS_LAYOUT_DESTROYED, &lseg->pls_layout->plh_flags);
/* Matched by initial refcount set in alloc_init_layout_hdr */
- put_layout_hdr_locked(lseg->pls_layout);
+ pnfs_put_layout_hdr_locked(lseg->pls_layout);
}
rpc_wake_up(&NFS_SERVER(inode)->roc_rpcwaitq);
}
@@ -404,7 +404,7 @@ pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
NFS_I(lo->plh_inode)->write_io = 0;
NFS_I(lo->plh_inode)->read_io = 0;
if (!test_and_set_bit(NFS_LAYOUT_DESTROYED, &lo->plh_flags))
- put_layout_hdr_locked(lo);
+ pnfs_put_layout_hdr_locked(lo);
return 0;
}
list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
@@ -649,13 +649,13 @@ _pnfs_return_layout(struct inode *ino)
}
stateid = nfsi->layout->plh_stateid;
/* Reference matched in nfs4_layoutreturn_release */
- get_layout_hdr(lo);
+ pnfs_get_layout_hdr(lo);
empty = list_empty(&lo->plh_segs);
pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL);
/* Don't send a LAYOUTRETURN if list was initially empty */
if (empty) {
spin_unlock(&ino->i_lock);
- put_layout_hdr(lo);
+ pnfs_put_layout_hdr(lo);
dprintk("NFS: %s no layout segments to return\n", __func__);
goto out;
}
@@ -672,7 +672,7 @@ _pnfs_return_layout(struct inode *ino)
set_bit(NFS_LAYOUT_RW_FAILED, &lo->plh_flags);
set_bit(NFS_LAYOUT_RO_FAILED, &lo->plh_flags);
pnfs_clear_layout_returned(lo);
- put_layout_hdr(lo);
+ pnfs_put_layout_hdr(lo);
goto out;
}

@@ -709,7 +709,7 @@ bool pnfs_roc(struct inode *ino)
if (!found)
goto out_nolayout;
lo->plh_block_lgets++;
- get_layout_hdr(lo); /* matched in pnfs_roc_release */
+ pnfs_get_layout_hdr(lo); /* matched in pnfs_roc_release */
spin_unlock(&ino->i_lock);
pnfs_free_lseg_list(&tmp_list);
return true;
@@ -726,7 +726,7 @@ void pnfs_roc_release(struct inode *ino)
spin_lock(&ino->i_lock);
lo = NFS_I(ino)->layout;
lo->plh_block_lgets--;
- put_layout_hdr_locked(lo);
+ pnfs_put_layout_hdr_locked(lo);
spin_unlock(&ino->i_lock);
}

@@ -819,7 +819,7 @@ pnfs_insert_layout(struct pnfs_layout_hdr *lo,
__func__, lseg, lseg->pls_range.iomode,
lseg->pls_range.offset, lseg->pls_range.length);
out:
- get_layout_hdr(lo);
+ pnfs_get_layout_hdr(lo);

dprintk("%s:Return\n", __func__);
}
@@ -1058,7 +1058,7 @@ pnfs_update_layout(struct inode *ino,
goto out_unlock;
atomic_inc(&lo->plh_outstanding);

- get_layout_hdr(lo);
+ pnfs_get_layout_hdr(lo);
if (list_empty(&lo->plh_segs))
first = true;

@@ -1091,7 +1091,7 @@ pnfs_update_layout(struct inode *ino,
spin_unlock(&clp->cl_lock);
}
atomic_dec(&lo->plh_outstanding);
- put_layout_hdr(lo);
+ pnfs_put_layout_hdr(lo);
out:
dprintk("%s end, state 0x%lx lseg %p\n", __func__,
nfsi->layout ? nfsi->layout->plh_flags : -1, lseg);
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index 6af5189..2af681f 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -176,7 +176,7 @@ extern struct pnfs_layout_segment* nfs4_proc_layoutget(struct nfs4_layoutget *lg
extern int nfs4_proc_layoutreturn(struct nfs4_layoutreturn *lrp);

/* pnfs.c */
-void get_layout_hdr(struct pnfs_layout_hdr *lo);
+void pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo);
void put_lseg(struct pnfs_layout_segment *lseg);

void pnfs_pageio_init_read(struct nfs_pageio_descriptor *, struct inode *,
@@ -196,7 +196,7 @@ struct pnfs_layout_segment *pnfs_layout_process(struct nfs4_layoutget *lgp);
void pnfs_free_lseg_list(struct list_head *tmp_list);
void pnfs_destroy_layout(struct nfs_inode *);
void pnfs_destroy_all_layouts(struct nfs_client *);
-void put_layout_hdr(struct pnfs_layout_hdr *lo);
+void pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo);
void pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo,
const nfs4_stateid *new,
bool update_barrier);
--
1.7.11.4


2012-09-21 20:52:39

by Myklebust, Trond

[permalink] [raw]
Subject: [PATCH v2 19/26] NFSv4.1: Clean up pnfs_put_lseg()

There is no longer a need to use pnfs_free_lseg_list(). Just call
pnfs_free_lseg() directly.

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/pnfs.c | 9 +++------
1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 557faaf..ed8d674 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -290,7 +290,7 @@ init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg)
lseg->pls_layout = lo;
}

-static void free_lseg(struct pnfs_layout_segment *lseg)
+static void pnfs_free_lseg(struct pnfs_layout_segment *lseg)
{
struct inode *ino = lseg->pls_layout->plh_inode;

@@ -327,12 +327,9 @@ pnfs_put_lseg(struct pnfs_layout_segment *lseg)
lo = lseg->pls_layout;
inode = lo->plh_inode;
if (atomic_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) {
- LIST_HEAD(free_me);
-
pnfs_layout_remove_lseg(lo, lseg);
spin_unlock(&inode->i_lock);
- list_add(&lseg->pls_list, &free_me);
- pnfs_free_lseg_list(&free_me);
+ pnfs_free_lseg(lseg);
}
}
EXPORT_SYMBOL_GPL(pnfs_put_lseg);
@@ -469,7 +466,7 @@ pnfs_free_lseg_list(struct list_head *free_me)

list_for_each_entry_safe(lseg, tmp, free_me, pls_list) {
list_del(&lseg->pls_list);
- free_lseg(lseg);
+ pnfs_free_lseg(lseg);
}
}

--
1.7.11.4


2012-09-21 20:51:34

by Myklebust, Trond

[permalink] [raw]
Subject: [PATCH v2 08/26] NFSv4.1: pNFS data servers may be temporarily offline

In cases where the pNFS data server is just temporarily out of service,
we want to mark it as such, and then try again later. Typically that will
be in cases of network connection errors etc.
This patch allows us to mark the devices as being "unavailable" for such
transient errors, and will make them available for retries after a
2 minute timeout period.

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/nfs4filelayout.c | 22 +++++++++++++++++++---
fs/nfs/nfs4filelayout.h | 8 ++------
fs/nfs/nfs4filelayoutdev.c | 15 +++++++--------
fs/nfs/pnfs.h | 4 ++++
fs/nfs/pnfs_dev.c | 27 +++++++++++++++++++++++++++
5 files changed, 59 insertions(+), 17 deletions(-)

diff --git a/fs/nfs/nfs4filelayout.c b/fs/nfs/nfs4filelayout.c
index af6ee4a..dac2162 100644
--- a/fs/nfs/nfs4filelayout.c
+++ b/fs/nfs/nfs4filelayout.c
@@ -205,7 +205,7 @@ static int filelayout_async_handle_error(struct rpc_task *task,
case -EPIPE:
dprintk("%s DS connection error %d\n", __func__,
task->tk_status);
- filelayout_mark_devid_invalid(devid);
+ nfs4_mark_deviceid_unavailable(devid);
clear_bit(NFS_INO_LAYOUTCOMMIT, &NFS_I(inode)->flags);
_pnfs_return_layout(inode);
rpc_wake_up(&tbl->slot_tbl_waitq);
@@ -269,6 +269,22 @@ filelayout_set_layoutcommit(struct nfs_write_data *wdata)
(unsigned long) NFS_I(hdr->inode)->layout->plh_lwb);
}

+bool
+filelayout_test_devid_unavailable(struct nfs4_deviceid_node *node)
+{
+ return filelayout_test_devid_invalid(node) ||
+ nfs4_test_deviceid_unavailable(node);
+}
+
+static bool
+filelayout_reset_to_mds(struct pnfs_layout_segment *lseg)
+{
+ struct nfs4_deviceid_node *node = FILELAYOUT_DEVID_NODE(lseg);
+
+ return filelayout_test_layout_invalid(lseg->pls_layout) ||
+ filelayout_test_devid_unavailable(node);
+}
+
/*
* Call ops for the async read/write cases
* In the case of dense layouts, the offset needs to be reset to its
@@ -613,8 +629,8 @@ filelayout_check_layout(struct pnfs_layout_hdr *lo,
goto out;
} else
dsaddr = container_of(d, struct nfs4_file_layout_dsaddr, id_node);
- /* Found deviceid is being reaped */
- if (test_bit(NFS_DEVICEID_INVALID, &dsaddr->id_node.flags))
+ /* Found deviceid is unavailable */
+ if (filelayout_test_devid_unavailable(&dsaddr->id_node))
goto out_put;

fl->dsaddr = dsaddr;
diff --git a/fs/nfs/nfs4filelayout.h b/fs/nfs/nfs4filelayout.h
index 11053c4..10b0f13 100644
--- a/fs/nfs/nfs4filelayout.h
+++ b/fs/nfs/nfs4filelayout.h
@@ -140,12 +140,8 @@ filelayout_test_devid_invalid(struct nfs4_deviceid_node *node)
return test_bit(NFS_DEVICEID_INVALID, &node->flags);
}

-static inline bool
-filelayout_reset_to_mds(struct pnfs_layout_segment *lseg)
-{
- return filelayout_test_devid_invalid(FILELAYOUT_DEVID_NODE(lseg)) ||
- filelayout_test_layout_invalid(lseg->pls_layout);
-}
+extern bool
+filelayout_test_devid_unavailable(struct nfs4_deviceid_node *node);

extern struct nfs_fh *
nfs4_fl_select_ds_fh(struct pnfs_layout_segment *lseg, u32 j);
diff --git a/fs/nfs/nfs4filelayoutdev.c b/fs/nfs/nfs4filelayoutdev.c
index b85a29d..3336d5e 100644
--- a/fs/nfs/nfs4filelayoutdev.c
+++ b/fs/nfs/nfs4filelayoutdev.c
@@ -804,13 +804,14 @@ nfs4_fl_prepare_ds(struct pnfs_layout_segment *lseg, u32 ds_idx)
struct nfs4_pnfs_ds *ds = dsaddr->ds_list[ds_idx];
struct nfs4_deviceid_node *devid = FILELAYOUT_DEVID_NODE(lseg);

- if (filelayout_test_devid_invalid(devid))
+ if (filelayout_test_devid_unavailable(devid))
return NULL;

if (ds == NULL) {
printk(KERN_ERR "NFS: %s: No data server for offset index %d\n",
__func__, ds_idx);
- goto mark_dev_invalid;
+ filelayout_mark_devid_invalid(devid);
+ return NULL;
}

if (!ds->ds_clp) {
@@ -818,14 +819,12 @@ nfs4_fl_prepare_ds(struct pnfs_layout_segment *lseg, u32 ds_idx)
int err;

err = nfs4_ds_connect(s, ds);
- if (err)
- goto mark_dev_invalid;
+ if (err) {
+ nfs4_mark_deviceid_unavailable(devid);
+ return NULL;
+ }
}
return ds;
-
-mark_dev_invalid:
- filelayout_mark_devid_invalid(devid);
- return NULL;
}

module_param(dataserver_retrans, uint, 0644);
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index bc8e500..9735031 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -234,6 +234,7 @@ struct nfs4_threshold *pnfs_mdsthreshold_alloc(void);
/* nfs4_deviceid_flags */
enum {
NFS_DEVICEID_INVALID = 0, /* set when MDS clientid recalled */
+ NFS_DEVICEID_UNAVAILABLE, /* device temporarily unavailable */
};

/* pnfs_dev.c */
@@ -243,6 +244,7 @@ struct nfs4_deviceid_node {
const struct pnfs_layoutdriver_type *ld;
const struct nfs_client *nfs_client;
unsigned long flags;
+ unsigned long timestamp_unavailable;
struct nfs4_deviceid deviceid;
atomic_t ref;
};
@@ -255,6 +257,8 @@ void nfs4_init_deviceid_node(struct nfs4_deviceid_node *,
const struct nfs4_deviceid *);
struct nfs4_deviceid_node *nfs4_insert_deviceid_node(struct nfs4_deviceid_node *);
bool nfs4_put_deviceid_node(struct nfs4_deviceid_node *);
+void nfs4_mark_deviceid_unavailable(struct nfs4_deviceid_node *node);
+bool nfs4_test_deviceid_unavailable(struct nfs4_deviceid_node *node);
void nfs4_deviceid_purge_client(const struct nfs_client *);

static inline void
diff --git a/fs/nfs/pnfs_dev.c b/fs/nfs/pnfs_dev.c
index 73f701f..d35b62e 100644
--- a/fs/nfs/pnfs_dev.c
+++ b/fs/nfs/pnfs_dev.c
@@ -40,6 +40,8 @@
#define NFS4_DEVICE_ID_HASH_SIZE (1 << NFS4_DEVICE_ID_HASH_BITS)
#define NFS4_DEVICE_ID_HASH_MASK (NFS4_DEVICE_ID_HASH_SIZE - 1)

+#define PNFS_DEVICE_RETRY_TIMEOUT (120*HZ)
+
static struct hlist_head nfs4_deviceid_cache[NFS4_DEVICE_ID_HASH_SIZE];
static DEFINE_SPINLOCK(nfs4_deviceid_lock);

@@ -218,6 +220,30 @@ nfs4_put_deviceid_node(struct nfs4_deviceid_node *d)
}
EXPORT_SYMBOL_GPL(nfs4_put_deviceid_node);

+void
+nfs4_mark_deviceid_unavailable(struct nfs4_deviceid_node *node)
+{
+ node->timestamp_unavailable = jiffies;
+ set_bit(NFS_DEVICEID_UNAVAILABLE, &node->flags);
+}
+EXPORT_SYMBOL_GPL(nfs4_mark_deviceid_unavailable);
+
+bool
+nfs4_test_deviceid_unavailable(struct nfs4_deviceid_node *node)
+{
+ if (test_bit(NFS_DEVICEID_UNAVAILABLE, &node->flags)) {
+ unsigned long start, end;
+
+ end = jiffies;
+ start = end - PNFS_DEVICE_RETRY_TIMEOUT;
+ if (time_in_range(node->timestamp_unavailable, start, end))
+ return true;
+ clear_bit(NFS_DEVICEID_UNAVAILABLE, &node->flags);
+ }
+ return false;
+}
+EXPORT_SYMBOL_GPL(nfs4_test_deviceid_unavailable);
+
static void
_deviceid_purge_client(const struct nfs_client *clp, long hash)
{
@@ -276,3 +302,4 @@ nfs4_deviceid_mark_client_invalid(struct nfs_client *clp)
}
rcu_read_unlock();
}
+
--
1.7.11.4


2012-09-21 20:50:59

by Myklebust, Trond

[permalink] [raw]
Subject: [PATCH v2 02/26] NFSv4.1: Cleanup add a "pnfs_" prefix to mark_matching_lsegs_invalid

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/callback_proc.c | 4 ++--
fs/nfs/pnfs.c | 6 +++---
fs/nfs/pnfs.h | 2 +-
3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c
index 1b5d809..57b8bda 100644
--- a/fs/nfs/callback_proc.c
+++ b/fs/nfs/callback_proc.c
@@ -158,7 +158,7 @@ static u32 initiate_file_draining(struct nfs_client *clp,
ino = lo->plh_inode;
spin_lock(&ino->i_lock);
if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags) ||
- mark_matching_lsegs_invalid(lo, &free_me_list,
+ pnfs_mark_matching_lsegs_invalid(lo, &free_me_list,
&args->cbl_range))
rv = NFS4ERR_DELAY;
else
@@ -211,7 +211,7 @@ static u32 initiate_bulk_draining(struct nfs_client *clp,
ino = lo->plh_inode;
spin_lock(&ino->i_lock);
set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
- if (mark_matching_lsegs_invalid(lo, &free_me_list, &range))
+ if (pnfs_mark_matching_lsegs_invalid(lo, &free_me_list, &range))
rv = NFS4ERR_DELAY;
list_del_init(&lo->plh_bulk_recall);
spin_unlock(&ino->i_lock);
diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 3a7ac97..aea2e52 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -390,7 +390,7 @@ static int mark_lseg_invalid(struct pnfs_layout_segment *lseg,
* after call.
*/
int
-mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
+pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
struct list_head *tmp_list,
struct pnfs_layout_range *recall_range)
{
@@ -458,7 +458,7 @@ pnfs_destroy_layout(struct nfs_inode *nfsi)
lo = nfsi->layout;
if (lo) {
lo->plh_block_lgets++; /* permanently block new LAYOUTGETs */
- mark_matching_lsegs_invalid(lo, &tmp_list, NULL);
+ pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL);
}
spin_unlock(&nfsi->vfs_inode.i_lock);
pnfs_free_lseg_list(&tmp_list);
@@ -651,7 +651,7 @@ _pnfs_return_layout(struct inode *ino)
/* Reference matched in nfs4_layoutreturn_release */
get_layout_hdr(lo);
empty = list_empty(&lo->plh_segs);
- mark_matching_lsegs_invalid(lo, &tmp_list, NULL);
+ pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL);
/* Don't send a LAYOUTRETURN if list was initially empty */
if (empty) {
spin_unlock(&ino->i_lock);
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index d51ef88..6af5189 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -203,7 +203,7 @@ void pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo,
int pnfs_choose_layoutget_stateid(nfs4_stateid *dst,
struct pnfs_layout_hdr *lo,
struct nfs4_state *open_state);
-int mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
+int pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
struct list_head *tmp_list,
struct pnfs_layout_range *recall_range);
bool pnfs_roc(struct inode *ino);
--
1.7.11.4


2012-09-21 20:52:50

by Myklebust, Trond

[permalink] [raw]
Subject: [PATCH v2 21/26] NFSv4.1: Get rid of pNFS spin lock debugging asserts...

These are all in static declared functions that are called only once.

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/pnfs.c | 3 ---
1 file changed, 3 deletions(-)

diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 9c5320c..bbdce87 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -829,7 +829,6 @@ pnfs_layout_insert_lseg(struct pnfs_layout_hdr *lo,

dprintk("%s:Begin\n", __func__);

- assert_spin_locked(&lo->plh_inode->i_lock);
list_for_each_entry(lp, &lo->plh_segs, pls_list) {
if (cmp_layout(&lseg->pls_range, &lp->pls_range) > 0)
continue;
@@ -883,7 +882,6 @@ pnfs_find_alloc_layout(struct inode *ino,

dprintk("%s Begin ino=%p layout=%p\n", __func__, ino, nfsi->layout);

- assert_spin_locked(&ino->i_lock);
if (nfsi->layout) {
if (test_bit(NFS_LAYOUT_DESTROYED, &nfsi->layout->plh_flags))
return NULL;
@@ -940,7 +938,6 @@ pnfs_find_lseg(struct pnfs_layout_hdr *lo,

dprintk("%s:Begin\n", __func__);

- assert_spin_locked(&lo->plh_inode->i_lock);
list_for_each_entry(lseg, &lo->plh_segs, pls_list) {
if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) &&
is_matching_lseg(&lseg->pls_range, range)) {
--
1.7.11.4


2012-09-21 20:51:23

by Myklebust, Trond

[permalink] [raw]
Subject: [PATCH v2 06/26] NFSv4.1: Add helpers for setting/reading the I/O fail bit

...and make them local to the pnfs.c file.

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/pnfs.c | 37 ++++++++++++++++++++++++++-----------
fs/nfs/pnfs.h | 6 ------
2 files changed, 26 insertions(+), 17 deletions(-)

diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 498af87..f85cc4c 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -238,6 +238,27 @@ pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo)
}
}

+static int
+pnfs_iomode_to_fail_bit(u32 iomode)
+{
+ return iomode == IOMODE_RW ?
+ NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED;
+}
+
+static void
+pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode)
+{
+ set_bit(pnfs_iomode_to_fail_bit(iomode), &lo->plh_flags);
+ dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__,
+ iomode == IOMODE_RW ? "RW" : "READ");
+}
+
+static bool
+pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode)
+{
+ return test_bit(pnfs_iomode_to_fail_bit(iomode), &lo->plh_flags) != 0;
+}
+
static void
init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg)
{
@@ -612,7 +633,7 @@ send_layoutget(struct pnfs_layout_hdr *lo,
break;
default:
/* remember that LAYOUTGET failed and suspend trying */
- set_bit(lo_fail_bit(range->iomode), &lo->plh_flags);
+ pnfs_layout_io_set_failed(lo, range->iomode);
}
return NULL;
}
@@ -669,8 +690,8 @@ _pnfs_return_layout(struct inode *ino)
lrp = kzalloc(sizeof(*lrp), GFP_KERNEL);
if (unlikely(lrp == NULL)) {
status = -ENOMEM;
- set_bit(NFS_LAYOUT_RW_FAILED, &lo->plh_flags);
- set_bit(NFS_LAYOUT_RO_FAILED, &lo->plh_flags);
+ pnfs_layout_io_set_failed(lo, IOMODE_RW);
+ pnfs_layout_io_set_failed(lo, IOMODE_READ);
pnfs_clear_layout_returned(lo);
pnfs_put_layout_hdr(lo);
goto out;
@@ -1046,7 +1067,7 @@ pnfs_update_layout(struct inode *ino,
}

/* if LAYOUTGET already failed once we don't try again */
- if (test_bit(lo_fail_bit(iomode), &nfsi->layout->plh_flags))
+ if (pnfs_layout_io_test_failed(nfsi->layout, iomode))
goto out_unlock;

/* Check to see if the layout for the given range already exists */
@@ -1581,13 +1602,7 @@ static void pnfs_list_write_lseg(struct inode *inode, struct list_head *listp)

void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg)
{
- if (lseg->pls_range.iomode == IOMODE_RW) {
- dprintk("%s Setting layout IOMODE_RW fail bit\n", __func__);
- set_bit(lo_fail_bit(IOMODE_RW), &lseg->pls_layout->plh_flags);
- } else {
- dprintk("%s Setting layout IOMODE_READ fail bit\n", __func__);
- set_bit(lo_fail_bit(IOMODE_READ), &lseg->pls_layout->plh_flags);
- }
+ pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode);
}
EXPORT_SYMBOL_GPL(pnfs_set_lo_fail);

diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index 0495879..e3eb7d1 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -274,12 +274,6 @@ pnfs_test_layout_returned(struct pnfs_layout_hdr *lo)
return test_bit(NFS_LAYOUT_RETURNED, &lo->plh_flags);
}

-static inline int lo_fail_bit(u32 iomode)
-{
- return iomode == IOMODE_RW ?
- NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED;
-}
-
static inline struct pnfs_layout_segment *
pnfs_get_lseg(struct pnfs_layout_segment *lseg)
{
--
1.7.11.4


2012-09-21 20:52:21

by Myklebust, Trond

[permalink] [raw]
Subject: [PATCH v2 16/26] NFSv4.1: Remove redundant reference to the pnfs_layout_hdr

Each layout segment already holds a reference to the pnfs_layout_hdr,
so there is no need to hold an extra reference that is released once
the last layout segment is freed.

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/pnfs.c | 9 ++-------
1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 142d0f4..767c748 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -306,11 +306,8 @@ pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo,

WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
list_del_init(&lseg->pls_list);
- if (list_empty(&lo->plh_segs)) {
+ if (list_empty(&lo->plh_segs))
set_bit(NFS_LAYOUT_DESTROYED, &lo->plh_flags);
- /* Matched by initial refcount set in alloc_init_layout_hdr */
- pnfs_put_layout_hdr_locked(lo);
- }
rpc_wake_up(&NFS_SERVER(inode)->roc_rpcwaitq);
}

@@ -443,8 +440,7 @@ pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
dprintk("%s:Begin lo %p\n", __func__, lo);

if (list_empty(&lo->plh_segs)) {
- if (!test_and_set_bit(NFS_LAYOUT_DESTROYED, &lo->plh_flags))
- pnfs_put_layout_hdr_locked(lo);
+ set_bit(NFS_LAYOUT_DESTROYED, &lo->plh_flags);
return 0;
}
list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
@@ -1102,7 +1098,6 @@ pnfs_update_layout(struct inode *ino,
goto out_unlock;
atomic_inc(&lo->plh_outstanding);

- pnfs_get_layout_hdr(lo);
if (list_empty(&lo->plh_segs))
first = true;

--
1.7.11.4


2012-09-21 20:51:11

by Myklebust, Trond

[permalink] [raw]
Subject: [PATCH v2 04/26] NFSv4.1: Cleanup; add "pnfs_" prefix to put_lseg() and get_lseg()

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/nfs4filelayout.c | 16 ++++++++--------
fs/nfs/nfs4proc.c | 2 +-
fs/nfs/pnfs.c | 36 ++++++++++++++++++------------------
fs/nfs/pnfs.h | 8 ++++----
4 files changed, 31 insertions(+), 31 deletions(-)

diff --git a/fs/nfs/nfs4filelayout.c b/fs/nfs/nfs4filelayout.c
index 53f94d9..77cd115 100644
--- a/fs/nfs/nfs4filelayout.c
+++ b/fs/nfs/nfs4filelayout.c
@@ -453,7 +453,7 @@ static void filelayout_commit_release(void *calldata)
struct nfs_commit_data *data = calldata;

data->completion_ops->completion(data);
- put_lseg(data->lseg);
+ pnfs_put_lseg(data->lseg);
nfs_put_client(data->ds_clp);
nfs_commitdata_release(data);
}
@@ -931,7 +931,7 @@ filelayout_pg_init_write(struct nfs_pageio_descriptor *pgio,
nfs_init_cinfo(&cinfo, pgio->pg_inode, pgio->pg_dreq);
status = filelayout_alloc_commit_info(pgio->pg_lseg, &cinfo, GFP_NOFS);
if (status < 0) {
- put_lseg(pgio->pg_lseg);
+ pnfs_put_lseg(pgio->pg_lseg);
pgio->pg_lseg = NULL;
goto out_mds;
}
@@ -985,7 +985,7 @@ filelayout_clear_request_commit(struct nfs_page *req,
out:
nfs_request_remove_commit_list(req, cinfo);
spin_unlock(cinfo->lock);
- put_lseg(freeme);
+ pnfs_put_lseg(freeme);
}

static struct list_head *
@@ -1018,7 +1018,7 @@ filelayout_choose_commit_list(struct nfs_page *req,
* off due to a rewrite, in which case it will be done in
* filelayout_clear_request_commit
*/
- buckets[i].wlseg = get_lseg(lseg);
+ buckets[i].wlseg = pnfs_get_lseg(lseg);
}
set_bit(PG_COMMIT_TO_DS, &req->wb_flags);
cinfo->ds->nwritten++;
@@ -1128,7 +1128,7 @@ filelayout_scan_ds_commit_list(struct pnfs_commit_bucket *bucket,
if (list_empty(src))
bucket->wlseg = NULL;
else
- get_lseg(bucket->clseg);
+ pnfs_get_lseg(bucket->clseg);
}
return ret;
}
@@ -1159,12 +1159,12 @@ static void filelayout_recover_commit_reqs(struct list_head *dst,

/* NOTE cinfo->lock is NOT held, relying on fact that this is
* only called on single thread per dreq.
- * Can't take the lock because need to do put_lseg
+ * Can't take the lock because need to do pnfs_put_lseg
*/
for (i = 0, b = cinfo->ds->buckets; i < cinfo->ds->nbuckets; i++, b++) {
if (transfer_commit_list(&b->written, dst, cinfo, 0)) {
BUG_ON(!list_empty(&b->written));
- put_lseg(b->wlseg);
+ pnfs_put_lseg(b->wlseg);
b->wlseg = NULL;
}
}
@@ -1200,7 +1200,7 @@ alloc_ds_commits(struct nfs_commit_info *cinfo, struct list_head *list)
if (list_empty(&bucket->committing))
continue;
nfs_retry_commit(&bucket->committing, bucket->clseg, cinfo);
- put_lseg(bucket->clseg);
+ pnfs_put_lseg(bucket->clseg);
bucket->clseg = NULL;
}
/* Caller will clean up entries put on list */
diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index 20f1229..87702a0 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -6549,7 +6549,7 @@ static void nfs4_layoutcommit_release(void *calldata)
list_del_init(&lseg->pls_lc_list);
if (test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT,
&lseg->pls_flags))
- put_lseg(lseg);
+ pnfs_put_lseg(lseg);
}

clear_bit_unlock(NFS_INO_LAYOUTCOMMITTING, bitlock);
diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 512c863..498af87 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -259,7 +259,7 @@ static void free_lseg(struct pnfs_layout_segment *lseg)
}

static void
-put_lseg_common(struct pnfs_layout_segment *lseg)
+pnfs_put_lseg_common(struct pnfs_layout_segment *lseg)
{
struct inode *inode = lseg->pls_layout->plh_inode;

@@ -274,7 +274,7 @@ put_lseg_common(struct pnfs_layout_segment *lseg)
}

void
-put_lseg(struct pnfs_layout_segment *lseg)
+pnfs_put_lseg(struct pnfs_layout_segment *lseg)
{
struct inode *inode;

@@ -288,13 +288,13 @@ put_lseg(struct pnfs_layout_segment *lseg)
if (atomic_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) {
LIST_HEAD(free_me);

- put_lseg_common(lseg);
+ pnfs_put_lseg_common(lseg);
list_add(&lseg->pls_list, &free_me);
spin_unlock(&inode->i_lock);
pnfs_free_lseg_list(&free_me);
}
}
-EXPORT_SYMBOL_GPL(put_lseg);
+EXPORT_SYMBOL_GPL(pnfs_put_lseg);

static inline u64
end_offset(u64 start, u64 len)
@@ -378,7 +378,7 @@ static int mark_lseg_invalid(struct pnfs_layout_segment *lseg,
dprintk("%s: lseg %p ref %d\n", __func__, lseg,
atomic_read(&lseg->pls_refcount));
if (atomic_dec_and_test(&lseg->pls_refcount)) {
- put_lseg_common(lseg);
+ pnfs_put_lseg_common(lseg);
list_add(&lseg->pls_list, tmp_list);
rv = 1;
}
@@ -914,7 +914,7 @@ pnfs_find_lseg(struct pnfs_layout_hdr *lo,
list_for_each_entry(lseg, &lo->plh_segs, pls_list) {
if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) &&
is_matching_lseg(&lseg->pls_range, range)) {
- ret = get_lseg(lseg);
+ ret = pnfs_get_lseg(lseg);
break;
}
if (lseg->pls_range.offset > range->offset)
@@ -1135,7 +1135,7 @@ pnfs_layout_process(struct nfs4_layoutget *lgp)
}
init_lseg(lo, lseg);
lseg->pls_range = res->range;
- get_lseg(lseg);
+ pnfs_get_lseg(lseg);
pnfs_insert_layout(lo, lseg);

if (res->return_on_close) {
@@ -1369,12 +1369,12 @@ pnfs_do_multiple_writes(struct nfs_pageio_descriptor *desc, struct list_head *he
if (trypnfs == PNFS_NOT_ATTEMPTED)
pnfs_write_through_mds(desc, data);
}
- put_lseg(lseg);
+ pnfs_put_lseg(lseg);
}

static void pnfs_writehdr_free(struct nfs_pgio_header *hdr)
{
- put_lseg(hdr->lseg);
+ pnfs_put_lseg(hdr->lseg);
nfs_writehdr_free(hdr);
}
EXPORT_SYMBOL_GPL(pnfs_writehdr_free);
@@ -1389,17 +1389,17 @@ pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc)
whdr = nfs_writehdr_alloc();
if (!whdr) {
desc->pg_completion_ops->error_cleanup(&desc->pg_list);
- put_lseg(desc->pg_lseg);
+ pnfs_put_lseg(desc->pg_lseg);
desc->pg_lseg = NULL;
return -ENOMEM;
}
hdr = &whdr->header;
nfs_pgheader_init(desc, hdr, pnfs_writehdr_free);
- hdr->lseg = get_lseg(desc->pg_lseg);
+ hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
atomic_inc(&hdr->refcnt);
ret = nfs_generic_flush(desc, hdr);
if (ret != 0) {
- put_lseg(desc->pg_lseg);
+ pnfs_put_lseg(desc->pg_lseg);
desc->pg_lseg = NULL;
} else
pnfs_do_multiple_writes(desc, &hdr->rpc_list, desc->pg_ioflags);
@@ -1524,12 +1524,12 @@ pnfs_do_multiple_reads(struct nfs_pageio_descriptor *desc, struct list_head *hea
if (trypnfs == PNFS_NOT_ATTEMPTED)
pnfs_read_through_mds(desc, data);
}
- put_lseg(lseg);
+ pnfs_put_lseg(lseg);
}

static void pnfs_readhdr_free(struct nfs_pgio_header *hdr)
{
- put_lseg(hdr->lseg);
+ pnfs_put_lseg(hdr->lseg);
nfs_readhdr_free(hdr);
}
EXPORT_SYMBOL_GPL(pnfs_readhdr_free);
@@ -1545,17 +1545,17 @@ pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc)
if (!rhdr) {
desc->pg_completion_ops->error_cleanup(&desc->pg_list);
ret = -ENOMEM;
- put_lseg(desc->pg_lseg);
+ pnfs_put_lseg(desc->pg_lseg);
desc->pg_lseg = NULL;
return ret;
}
hdr = &rhdr->header;
nfs_pgheader_init(desc, hdr, pnfs_readhdr_free);
- hdr->lseg = get_lseg(desc->pg_lseg);
+ hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
atomic_inc(&hdr->refcnt);
ret = nfs_generic_pagein(desc, hdr);
if (ret != 0) {
- put_lseg(desc->pg_lseg);
+ pnfs_put_lseg(desc->pg_lseg);
desc->pg_lseg = NULL;
} else
pnfs_do_multiple_reads(desc, &hdr->rpc_list);
@@ -1608,7 +1608,7 @@ pnfs_set_layoutcommit(struct nfs_write_data *wdata)
}
if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &hdr->lseg->pls_flags)) {
/* references matched in nfs4_layoutcommit_release */
- get_lseg(hdr->lseg);
+ pnfs_get_lseg(hdr->lseg);
}
if (end_pos > nfsi->layout->plh_lwb)
nfsi->layout->plh_lwb = end_pos;
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index 2af681f..0495879 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -177,7 +177,7 @@ extern int nfs4_proc_layoutreturn(struct nfs4_layoutreturn *lrp);

/* pnfs.c */
void pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo);
-void put_lseg(struct pnfs_layout_segment *lseg);
+void pnfs_put_lseg(struct pnfs_layout_segment *lseg);

void pnfs_pageio_init_read(struct nfs_pageio_descriptor *, struct inode *,
const struct nfs_pgio_completion_ops *);
@@ -281,7 +281,7 @@ static inline int lo_fail_bit(u32 iomode)
}

static inline struct pnfs_layout_segment *
-get_lseg(struct pnfs_layout_segment *lseg)
+pnfs_get_lseg(struct pnfs_layout_segment *lseg)
{
if (lseg) {
atomic_inc(&lseg->pls_refcount);
@@ -406,12 +406,12 @@ static inline void pnfs_destroy_layout(struct nfs_inode *nfsi)
}

static inline struct pnfs_layout_segment *
-get_lseg(struct pnfs_layout_segment *lseg)
+pnfs_get_lseg(struct pnfs_layout_segment *lseg)
{
return NULL;
}

-static inline void put_lseg(struct pnfs_layout_segment *lseg)
+static inline void pnfs_put_lseg(struct pnfs_layout_segment *lseg)
{
}

--
1.7.11.4


2012-09-21 20:52:56

by Myklebust, Trond

[permalink] [raw]
Subject: [PATCH v2 22/26] NFSv4.1: Remove unused 'default allocation' for pnfs_alloc_layout_hdr()

...and ditto for pnfs_free_layout_hdr()

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/pnfs.c | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index bbdce87..196b8bb 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -200,8 +200,7 @@ static struct pnfs_layout_hdr *
pnfs_alloc_layout_hdr(struct inode *ino, gfp_t gfp_flags)
{
struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld;
- return ld->alloc_layout_hdr ? ld->alloc_layout_hdr(ino, gfp_flags) :
- kzalloc(sizeof(struct pnfs_layout_hdr), gfp_flags);
+ return ld->alloc_layout_hdr(ino, gfp_flags);
}

static void
@@ -218,7 +217,7 @@ pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo)
spin_unlock(&clp->cl_lock);
}
put_rpccred(lo->plh_lc_cred);
- return ld->alloc_layout_hdr ? ld->free_layout_hdr(lo) : kfree(lo);
+ return ld->free_layout_hdr(lo);
}

static void
--
1.7.11.4


2012-09-21 20:52:16

by Myklebust, Trond

[permalink] [raw]
Subject: [PATCH v2 15/26] NFSv4.1: Rename the pnfs_put_lseg_common to pnfs_layout_remove_lseg

The latter name is more descriptive of the actual function.
Also rename pnfs_insert_layout to pnfs_layout_insert_lseg.

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/pnfs.c | 27 +++++++++++++++------------
1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 8633b78..142d0f4 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -294,21 +294,22 @@ static void free_lseg(struct pnfs_layout_segment *lseg)
struct inode *ino = lseg->pls_layout->plh_inode;

NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
- /* Matched by pnfs_get_layout_hdr in pnfs_insert_layout */
+ /* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */
pnfs_put_layout_hdr(NFS_I(ino)->layout);
}

static void
-pnfs_put_lseg_common(struct pnfs_layout_segment *lseg)
+pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo,
+ struct pnfs_layout_segment *lseg)
{
- struct inode *inode = lseg->pls_layout->plh_inode;
+ struct inode *inode = lo->plh_inode;

WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
list_del_init(&lseg->pls_list);
- if (list_empty(&lseg->pls_layout->plh_segs)) {
- set_bit(NFS_LAYOUT_DESTROYED, &lseg->pls_layout->plh_flags);
+ if (list_empty(&lo->plh_segs)) {
+ set_bit(NFS_LAYOUT_DESTROYED, &lo->plh_flags);
/* Matched by initial refcount set in alloc_init_layout_hdr */
- pnfs_put_layout_hdr_locked(lseg->pls_layout);
+ pnfs_put_layout_hdr_locked(lo);
}
rpc_wake_up(&NFS_SERVER(inode)->roc_rpcwaitq);
}
@@ -316,6 +317,7 @@ pnfs_put_lseg_common(struct pnfs_layout_segment *lseg)
void
pnfs_put_lseg(struct pnfs_layout_segment *lseg)
{
+ struct pnfs_layout_hdr *lo;
struct inode *inode;

if (!lseg)
@@ -324,13 +326,14 @@ pnfs_put_lseg(struct pnfs_layout_segment *lseg)
dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg,
atomic_read(&lseg->pls_refcount),
test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
- inode = lseg->pls_layout->plh_inode;
+ lo = lseg->pls_layout;
+ inode = lo->plh_inode;
if (atomic_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) {
LIST_HEAD(free_me);

- pnfs_put_lseg_common(lseg);
- list_add(&lseg->pls_list, &free_me);
+ pnfs_layout_remove_lseg(lo, lseg);
spin_unlock(&inode->i_lock);
+ list_add(&lseg->pls_list, &free_me);
pnfs_free_lseg_list(&free_me);
}
}
@@ -418,7 +421,7 @@ static int mark_lseg_invalid(struct pnfs_layout_segment *lseg,
dprintk("%s: lseg %p ref %d\n", __func__, lseg,
atomic_read(&lseg->pls_refcount));
if (atomic_dec_and_test(&lseg->pls_refcount)) {
- pnfs_put_lseg_common(lseg);
+ pnfs_layout_remove_lseg(lseg->pls_layout, lseg);
list_add(&lseg->pls_list, tmp_list);
rv = 1;
}
@@ -831,7 +834,7 @@ cmp_layout(struct pnfs_layout_range *l1,
}

static void
-pnfs_insert_layout(struct pnfs_layout_hdr *lo,
+pnfs_layout_insert_lseg(struct pnfs_layout_hdr *lo,
struct pnfs_layout_segment *lseg)
{
struct pnfs_layout_segment *lp;
@@ -1178,7 +1181,7 @@ pnfs_layout_process(struct nfs4_layoutget *lgp)
init_lseg(lo, lseg);
lseg->pls_range = res->range;
pnfs_get_lseg(lseg);
- pnfs_insert_layout(lo, lseg);
+ pnfs_layout_insert_lseg(lo, lseg);

if (res->return_on_close) {
set_bit(NFS_LSEG_ROC, &lseg->pls_flags);
--
1.7.11.4


2012-09-21 20:52:27

by Myklebust, Trond

[permalink] [raw]
Subject: [PATCH v2 17/26] NFSv4.1: Free the pnfs_layout_hdr outside the inode->i_lock

None of the existing pNFS layout drivers seem to require the inode
to be locked while they free the layout header.

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/pnfs.c | 21 +++++++++------------
1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 767c748..1827cae 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -213,7 +213,7 @@ pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo)
}

static void
-destroy_layout_hdr(struct pnfs_layout_hdr *lo)
+pnfs_detach_layout_hdr(struct pnfs_layout_hdr *lo)
{
struct nfs_inode *nfsi = NFS_I(lo->plh_inode);
dprintk("%s: freeing layout cache %p\n", __func__, lo);
@@ -222,14 +222,6 @@ destroy_layout_hdr(struct pnfs_layout_hdr *lo)
/* Reset MDS Threshold I/O counters */
nfsi->write_io = 0;
nfsi->read_io = 0;
- pnfs_free_layout_hdr(lo);
-}
-
-static void
-pnfs_put_layout_hdr_locked(struct pnfs_layout_hdr *lo)
-{
- if (atomic_dec_and_test(&lo->plh_refcount))
- destroy_layout_hdr(lo);
}

void
@@ -238,8 +230,9 @@ pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo)
struct inode *inode = lo->plh_inode;

if (atomic_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) {
- destroy_layout_hdr(lo);
+ pnfs_detach_layout_hdr(lo);
spin_unlock(&inode->i_lock);
+ pnfs_free_layout_hdr(lo);
}
}

@@ -762,8 +755,12 @@ void pnfs_roc_release(struct inode *ino)
spin_lock(&ino->i_lock);
lo = NFS_I(ino)->layout;
lo->plh_block_lgets--;
- pnfs_put_layout_hdr_locked(lo);
- spin_unlock(&ino->i_lock);
+ if (atomic_dec_and_test(&lo->plh_refcount)) {
+ pnfs_detach_layout_hdr(lo);
+ spin_unlock(&ino->i_lock);
+ pnfs_free_layout_hdr(lo);
+ } else
+ spin_unlock(&ino->i_lock);
}

void pnfs_roc_set_barrier(struct inode *ino, u32 barrier)
--
1.7.11.4


2012-09-24 14:47:42

by Andy Adamson

[permalink] [raw]
Subject: Re: [PATCH v2 10/26] NFSv4.1: pnfs_layout_io_test_failed must clear invalid lsegs before a retry

On Fri, Sep 21, 2012 at 4:50 PM, Trond Myklebust
<[email protected]> wrote:
> If pnfs_layout_io_test_failed() authorises a retry of the failed layoutgets,
> it should first clear the existing layout segments so that we start afresh.
>
> Signed-off-by: Trond Myklebust <[email protected]>
> ---
> fs/nfs/pnfs.c | 8 ++++++--
> 1 file changed, 6 insertions(+), 2 deletions(-)
>
> diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
> index cbbb0fc..80aaaba 100644
> --- a/fs/nfs/pnfs.c
> +++ b/fs/nfs/pnfs.c
> @@ -256,7 +256,8 @@ pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode)
> }
>
> static bool
> -pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode)
> +pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode,
> + struct list_head *head)
> {
> unsigned long start, end;
> if (test_bit(pnfs_iomode_to_fail_bit(iomode), &lo->plh_flags) == 0)
> @@ -267,6 +268,7 @@ pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode)
> /* It is time to retry the failed layoutgets */
> clear_bit(NFS_LAYOUT_RW_FAILED, &lo->plh_flags);
> clear_bit(NFS_LAYOUT_RO_FAILED, &lo->plh_flags);
> + pnfs_mark_matching_lsegs_invalid(lo, head, NULL);

By clearing all iomode failed bits and passing a NULL as the
recall_range to pnfs_mark_matching_lsegs, you are discarding layouts
not authorized by the iomode passed into pnfs_layout_io_test_failed.
Why not clear just the failed bit and setup a pnfs_layout_range
structure (for the whole file) and pass the authorized failed iomode
to pnfs_mark_matching_lsegs so as to remove only the failed iomode
layout segments?

-->Andy


> return false;
> }
> return true;
> @@ -1058,6 +1060,7 @@ pnfs_update_layout(struct inode *ino,
> struct nfs_client *clp = server->nfs_client;
> struct pnfs_layout_hdr *lo;
> struct pnfs_layout_segment *lseg = NULL;
> + LIST_HEAD(tmp_list);
> bool first = false;
>
> if (!pnfs_enabled_sb(NFS_SERVER(ino)))
> @@ -1081,7 +1084,7 @@ pnfs_update_layout(struct inode *ino,
> }
>
> /* if LAYOUTGET already failed once we don't try again */
> - if (pnfs_layout_io_test_failed(nfsi->layout, iomode))
> + if (pnfs_layout_io_test_failed(nfsi->layout, iomode, &tmp_list))
> goto out_unlock;
>
> /* Check to see if the layout for the given range already exists */
> @@ -1127,6 +1130,7 @@ pnfs_update_layout(struct inode *ino,
> }
> atomic_dec(&lo->plh_outstanding);
> out:
> + pnfs_free_lseg_list(&tmp_list);
> pnfs_put_layout_hdr(lo);
> dprintk("%s end, state 0x%lx lseg %p\n", __func__,
> nfsi->layout ? nfsi->layout->plh_flags : -1, lseg);
> --
> 1.7.11.4
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
> the body of a message to [email protected]
> More majordomo info at http://vger.kernel.org/majordomo-info.html

2012-09-21 20:51:40

by Myklebust, Trond

[permalink] [raw]
Subject: [PATCH v2 09/26] NFSv4.1: Fix a reference leak in pnfs_update_layout

If we exit after the call to pnfs_find_alloc_layout(), we have to ensure
that we put the struct pnfs_layout_hdr.

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/pnfs.c | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 6656cb4..cbbb0fc 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -1070,7 +1070,8 @@ pnfs_update_layout(struct inode *ino,
lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags);
if (lo == NULL) {
dprintk("%s ERROR: can't get pnfs_layout_hdr\n", __func__);
- goto out_unlock;
+ spin_unlock(&ino->i_lock);
+ return NULL;
}

/* Do we even need to bother with this? */
@@ -1125,8 +1126,8 @@ pnfs_update_layout(struct inode *ino,
spin_unlock(&clp->cl_lock);
}
atomic_dec(&lo->plh_outstanding);
- pnfs_put_layout_hdr(lo);
out:
+ pnfs_put_layout_hdr(lo);
dprintk("%s end, state 0x%lx lseg %p\n", __func__,
nfsi->layout ? nfsi->layout->plh_flags : -1, lseg);
return lseg;
--
1.7.11.4


2012-09-21 20:51:17

by Myklebust, Trond

[permalink] [raw]
Subject: [PATCH v2 05/26] NFSv4.1: Replace get_device_info() with filelayout_get_device_info()

Fix the namespace pollution issue.

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/blocklayout/blocklayout.c | 2 +-
fs/nfs/nfs4filelayout.c | 2 +-
fs/nfs/nfs4filelayout.h | 2 +-
fs/nfs/nfs4filelayoutdev.c | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/fs/nfs/blocklayout/blocklayout.c b/fs/nfs/blocklayout/blocklayout.c
index dd392ed..329bfbf 100644
--- a/fs/nfs/blocklayout/blocklayout.c
+++ b/fs/nfs/blocklayout/blocklayout.c
@@ -874,7 +874,7 @@ static void free_blk_mountid(struct block_mount_id *mid)
}
}

-/* This is mostly copied from the filelayout's get_device_info function.
+/* This is mostly copied from the filelayout_get_device_info function.
* It seems much of this should be at the generic pnfs level.
*/
static struct pnfs_block_dev *
diff --git a/fs/nfs/nfs4filelayout.c b/fs/nfs/nfs4filelayout.c
index 77cd115..af6ee4a 100644
--- a/fs/nfs/nfs4filelayout.c
+++ b/fs/nfs/nfs4filelayout.c
@@ -608,7 +608,7 @@ filelayout_check_layout(struct pnfs_layout_hdr *lo,
d = nfs4_find_get_deviceid(NFS_SERVER(lo->plh_inode)->pnfs_curr_ld,
NFS_SERVER(lo->plh_inode)->nfs_client, id);
if (d == NULL) {
- dsaddr = get_device_info(lo->plh_inode, id, gfp_flags);
+ dsaddr = filelayout_get_device_info(lo->plh_inode, id, gfp_flags);
if (dsaddr == NULL)
goto out;
} else
diff --git a/fs/nfs/nfs4filelayout.h b/fs/nfs/nfs4filelayout.h
index 43fe802..11053c4 100644
--- a/fs/nfs/nfs4filelayout.h
+++ b/fs/nfs/nfs4filelayout.h
@@ -158,7 +158,7 @@ struct nfs4_pnfs_ds *nfs4_fl_prepare_ds(struct pnfs_layout_segment *lseg,
extern void nfs4_fl_put_deviceid(struct nfs4_file_layout_dsaddr *dsaddr);
extern void nfs4_fl_free_deviceid(struct nfs4_file_layout_dsaddr *dsaddr);
struct nfs4_file_layout_dsaddr *
-get_device_info(struct inode *inode, struct nfs4_deviceid *dev_id, gfp_t gfp_flags);
+filelayout_get_device_info(struct inode *inode, struct nfs4_deviceid *dev_id, gfp_t gfp_flags);
void nfs4_ds_disconnect(struct nfs_client *clp);

#endif /* FS_NFS_NFS4FILELAYOUT_H */
diff --git a/fs/nfs/nfs4filelayoutdev.c b/fs/nfs/nfs4filelayoutdev.c
index f81231f..b85a29d 100644
--- a/fs/nfs/nfs4filelayoutdev.c
+++ b/fs/nfs/nfs4filelayoutdev.c
@@ -690,7 +690,7 @@ decode_and_add_device(struct inode *inode, struct pnfs_device *dev, gfp_t gfp_fl
* of available devices, and return it.
*/
struct nfs4_file_layout_dsaddr *
-get_device_info(struct inode *inode, struct nfs4_deviceid *dev_id, gfp_t gfp_flags)
+filelayout_get_device_info(struct inode *inode, struct nfs4_deviceid *dev_id, gfp_t gfp_flags)
{
struct pnfs_device *pdev = NULL;
u32 max_resp_sz;
--
1.7.11.4


2012-09-21 20:53:13

by Myklebust, Trond

[permalink] [raw]
Subject: [PATCH v2 25/26] NFSv4.1: Clear NFS_LAYOUT_BULK_RECALL when the layout segments are freed

Once all the affected layout segments have been freed up, clear the
NFS_LAYOUT_BULK_RECALL flag so that we can reuse the pnfs_layout_hdr

Signed-off-by: Trond Myklebust <[email protected]>
---
fs/nfs/pnfs.c | 2 ++
1 file changed, 2 insertions(+)

diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 8ba7c35..a5edf27 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -306,6 +306,8 @@ pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo,
list_del_init(&lseg->pls_list);
/* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */
atomic_dec(&lo->plh_refcount);
+ if (list_empty(&lo->plh_segs))
+ clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
rpc_wake_up(&NFS_SERVER(inode)->roc_rpcwaitq);
}

--
1.7.11.4