The following patches fix bugs on the layout recall callback path both at
the server an client
[PATCH 1/9] pnfsd: Initialize cb_layout clr_recall.cb_work
[PATCH 2/9] pnfsd: add default error case in nfsd4_cb_layout_done
These two are for pnfs-all-2.6.37 only,
The rest are for both 37 and 38 (pnfs-all-latest)
[PATCH 3/9] pnfsd: do not clear rpc_resp in nfsd4_cb_done_sequence
[PATCH 4/9] pnfs: fix missing unlock in nfs_client_return_layouts
[PATCH 5/9] pnfs: return bool status from nfs4_asynch_forget_layouts
[PATCH 6/9] pnfs: return NFS4ERR_DELAY from layoutrecall while waiting on lsegs
[PATCH 7/9] pnfs: fix cl_drain_notification indexing
[PATCH 8/9] pnfs: fix nfs4_asynch_forget_layouts accounting
[PATCH 9/9] pnfs: improve nfs4_layoutget_done debug printouts
I've pushed them to a git tree I set up for the Connectathon:
git://linux-nfs.org/~bhalevy/linux-pnfs-cthon.git
or
ssh://linux-nfs.org/~bhalevy/exports/linux-pnfs-cthon.git
if you don't have git access.
Benny
Signed-off-by: Benny Halevy <[email protected]>
---
fs/nfs/nfs4proc.c | 3 ++-
1 files changed, 2 insertions(+), 1 deletions(-)
diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index 10d46c4..785c460 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -5446,7 +5446,7 @@ static void nfs4_layoutget_done(struct rpc_task *task, void *calldata)
struct nfs4_layoutget *lgp = calldata;
struct nfs_server *server = NFS_SERVER(lgp->args.inode);
- dprintk("--> %s\n", __func__);
+ dprintk("--> %s: tk_status=%d\n", __func__, task->tk_status);
if (!nfs4_sequence_done(task, &lgp->res.seq_res)) {
/* layout code relies on fact that in this case
@@ -5467,6 +5467,7 @@ static void nfs4_layoutget_done(struct rpc_task *task, void *calldata)
if (nfs4_async_handle_error(task, server, NULL, NULL) == -EAGAIN) {
struct inode *ino = lgp->args.inode;
+ dprintk("<-- %s retrying\n", __func__);
spin_lock(&ino->i_lock);
atomic_dec(&NFS_I(ino)->layout->plh_outstanding);
spin_unlock(&ino->i_lock);
--
1.7.3.4
increment notify_count only the first time is it associated with a callback.
Signed-off-by: Benny Halevy <[email protected]>
---
fs/nfs/pnfs.c | 15 +++++++++++----
1 files changed, 11 insertions(+), 4 deletions(-)
diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 8c5f4b4..a10e8e8 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -441,17 +441,22 @@ should_free_lseg(struct pnfs_layout_range *lseg_range,
lo_seg_intersecting(lseg_range, recall_range);
}
-static void mark_lseg_invalid(struct pnfs_layout_segment *lseg,
+static bool mark_lseg_invalid(struct pnfs_layout_segment *lseg,
struct list_head *tmp_list)
{
+ bool rv;
+
assert_spin_locked(&lseg->layout->inode->i_lock);
- if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) {
+ rv = test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags);
+ if (rv) {
/* Remove the reference keeping the lseg in the
* list. It will now be removed when all
* outstanding io is finished.
*/
put_lseg_locked(lseg, tmp_list);
}
+
+ return rv;
}
/* Returns false if there was nothing to do, true otherwise */
@@ -652,8 +657,10 @@ bool nfs4_asynch_forget_layouts(struct pnfs_layout_hdr *lo,
dprintk("%s\n", __func__);
list_for_each_entry_safe(lseg, tmp, &lo->segs, fi_list)
if (should_free_lseg(&lseg->range, range)) {
- lseg->pls_notify_mask |= (1 << notify_idx);
- atomic_inc(notify_count);
+ if (lseg->pls_notify_mask & (1 << notify_idx)) {
+ lseg->pls_notify_mask |= (1 << notify_idx);
+ atomic_inc(notify_count);
+ }
mark_lseg_invalid(lseg, tmp_list);
rv = true;
}
--
1.7.3.4
Signed-off-by: Benny Halevy <[email protected]>
---
fs/nfs/callback_proc.c | 1 +
1 files changed, 1 insertions(+), 0 deletions(-)
diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c
index 48c61c2..9547c34 100644
--- a/fs/nfs/callback_proc.c
+++ b/fs/nfs/callback_proc.c
@@ -209,6 +209,7 @@ void nfs_client_return_layouts(struct nfs_client *clp)
list_del(&cb_info->pcl_list);
clp->cl_cb_lrecall_count--;
clp->cl_drain_notification[1 << cb_info->pcl_notify_bit] = NULL;
+ spin_unlock(&clp->cl_lock);
rpc_wake_up(&clp->cl_rpcwaitq_recall);
kfree(cb_info);
}
--
1.7.3.4
return true if segments were marked invalid
Signed-off-by: Benny Halevy <[email protected]>
---
fs/nfs/pnfs.c | 10 ++++++++--
fs/nfs/pnfs.h | 2 +-
2 files changed, 9 insertions(+), 3 deletions(-)
diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index beebc85..7507054 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -475,7 +475,7 @@ pnfs_clear_lseg_list(struct pnfs_layout_hdr *lo, struct list_head *tmp_list,
mark_lseg_invalid(lseg, tmp_list);
rv = true;
}
- dprintk("%s:Return\n", __func__);
+ dprintk("%s:Return %d\n", __func__, rv);
return rv;
}
@@ -640,20 +640,26 @@ send_layoutget(struct pnfs_layout_hdr *lo,
return lseg;
}
-void nfs4_asynch_forget_layouts(struct pnfs_layout_hdr *lo,
+bool nfs4_asynch_forget_layouts(struct pnfs_layout_hdr *lo,
struct pnfs_layout_range *range,
int notify_bit, atomic_t *notify_count,
struct list_head *tmp_list)
{
+ bool rv = false;
struct pnfs_layout_segment *lseg, *tmp;
assert_spin_locked(&lo->inode->i_lock);
+ dprintk("%s\n", __func__);
list_for_each_entry_safe(lseg, tmp, &lo->segs, fi_list)
if (should_free_lseg(&lseg->range, range)) {
lseg->pls_notify_mask |= (1 << notify_bit);
atomic_inc(notify_count);
mark_lseg_invalid(lseg, tmp_list);
+ rv = true;
}
+
+ dprintk("%s:Return %d\n", __func__, rv);
+ return rv;
}
/* Return true if there is layout based io in progress in the given range.
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index 1ee59c2..11c6ee4 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -298,7 +298,7 @@ void pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo,
int pnfs_choose_layoutget_stateid(nfs4_stateid *dst,
struct pnfs_layout_hdr *lo,
struct nfs4_state *open_state);
-void nfs4_asynch_forget_layouts(struct pnfs_layout_hdr *lo,
+bool nfs4_asynch_forget_layouts(struct pnfs_layout_hdr *lo,
struct pnfs_layout_range *range,
int notify_bit, atomic_t *notify_count,
struct list_head *tmp_list);
--
1.7.3.4
Signed-off-by: Benny Halevy <[email protected]>
---
fs/nfs/callback_proc.c | 23 +++++++++++++++--------
1 files changed, 15 insertions(+), 8 deletions(-)
diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c
index 9547c34..5e4b5d1 100644
--- a/fs/nfs/callback_proc.c
+++ b/fs/nfs/callback_proc.c
@@ -186,6 +186,7 @@ void nfs_client_return_layouts(struct nfs_client *clp)
{
struct pnfs_cb_lrecall_info *cb_info;
+ dprintk("%s\n", __func__);
spin_lock(&clp->cl_lock);
while (true) {
if (list_empty(&clp->cl_layoutrecalls)) {
@@ -270,10 +271,13 @@ static int initiate_layout_draining(struct pnfs_cb_lrecall_info *cb_info)
spin_lock(&lo->inode->i_lock);
if (rv == NFS4_OK) {
lo->plh_block_lgets++;
- nfs4_asynch_forget_layouts(lo, &args->cbl_range,
- cb_info->pcl_notify_bit,
- &cb_info->pcl_count,
- &free_me_list);
+ if (nfs4_asynch_forget_layouts(lo, &args->cbl_range,
+ cb_info->pcl_notify_bit,
+ &cb_info->pcl_count,
+ &free_me_list))
+ rv = NFS4ERR_DELAY;
+ else
+ rv = NFS4ERR_NOMATCHING_LAYOUT;
}
pnfs_set_layout_stateid(lo, &args->cbl_stateid, true);
spin_unlock(&lo->inode->i_lock);
@@ -309,10 +313,13 @@ static int initiate_layout_draining(struct pnfs_cb_lrecall_info *cb_info)
&recall_list, plh_bulk_recall) {
spin_lock(&lo->inode->i_lock);
set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
- nfs4_asynch_forget_layouts(lo, &range,
- cb_info->pcl_notify_bit,
- &cb_info->pcl_count,
- &free_me_list);
+ if (nfs4_asynch_forget_layouts(lo, &range,
+ cb_info->pcl_notify_bit,
+ &cb_info->pcl_count,
+ &free_me_list))
+ rv = NFS4ERR_DELAY;
+ else
+ rv = NFS4ERR_NOMATCHING_LAYOUT;
list_del_init(&lo->plh_bulk_recall);
spin_unlock(&lo->inode->i_lock);
put_layout_hdr(lo);
--
1.7.3.4
Fixes a 2,6,37-only bug
Signed-off-by: Benny Halevy <[email protected]>
---
fs/nfsd/nfs4pnfsd.c | 1 +
1 files changed, 1 insertions(+), 0 deletions(-)
diff --git a/fs/nfsd/nfs4pnfsd.c b/fs/nfsd/nfs4pnfsd.c
index 4aa7999..507e3e2 100644
--- a/fs/nfsd/nfs4pnfsd.c
+++ b/fs/nfsd/nfs4pnfsd.c
@@ -579,6 +579,7 @@ alloc_init_layoutrecall(struct nfsd4_pnfs_cb_layout *cbl,
kref_init(&clr->clr_ref);
INIT_LIST_HEAD(&clr->clr_perclnt);
+ INIT_WORK(&clr->clr_recall.cb_work, nfsd4_do_callback_rpc);
dprintk("NFSD %s return %p\n", __func__, clr);
return clr;
--
1.7.3.4
Signed-off-by: Benny Halevy <[email protected]>
---
fs/nfsd/nfs4callback.c | 1 +
1 files changed, 1 insertions(+), 0 deletions(-)
diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c
index 55f27d1..8351f67 100644
--- a/fs/nfsd/nfs4callback.c
+++ b/fs/nfsd/nfs4callback.c
@@ -1072,6 +1072,7 @@ static void nfsd4_cb_layout_done(struct rpc_task *task, void *calldata)
switch (task->tk_status) {
case -EIO:
+ default:
/* Network partition? */
atomic_set(&clp->cl_cb_set, 0);
warn_no_callback_path(clp, task->tk_status);
--
1.7.3.4
cl_drain_notification is an actual index into cl_drain_notification
and a bit index into pls_notify_mask.
Signed-off-by: Benny Halevy <[email protected]>
---
fs/nfs/callback_proc.c | 27 ++++++++++++++++-----------
fs/nfs/pnfs.c | 4 ++--
fs/nfs/pnfs.h | 2 +-
3 files changed, 19 insertions(+), 14 deletions(-)
diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c
index 5e4b5d1..dd295e2 100644
--- a/fs/nfs/callback_proc.c
+++ b/fs/nfs/callback_proc.c
@@ -197,8 +197,10 @@ void nfs_client_return_layouts(struct nfs_client *clp)
struct pnfs_cb_lrecall_info,
pcl_list);
spin_unlock(&clp->cl_lock);
+ /* Were all recalled lsegs already forgotten */
if (atomic_read(&cb_info->pcl_count) != 0)
break;
+
/* What do on error return? These layoutreturns are
* required by the protocol. So if do not get
* successful reply, probably have to do something
@@ -209,7 +211,7 @@ void nfs_client_return_layouts(struct nfs_client *clp)
/* Removing from the list unblocks LAYOUTGETs */
list_del(&cb_info->pcl_list);
clp->cl_cb_lrecall_count--;
- clp->cl_drain_notification[1 << cb_info->pcl_notify_bit] = NULL;
+ clp->cl_drain_notification[cb_info->pcl_notify_idx] = NULL;
spin_unlock(&clp->cl_lock);
rpc_wake_up(&clp->cl_rpcwaitq_recall);
kfree(cb_info);
@@ -224,7 +226,7 @@ void notify_drained(struct nfs_client *clp, u64 mask)
/* clp lock not needed except to remove used up entries */
/* Should probably use functions defined in bitmap.h */
while (mask) {
- if ((mask & 1) && (atomic_dec_and_test(*ptr)))
+ if ((mask & 1) && atomic_dec_and_test(*ptr))
done = true;
mask >>= 1;
ptr++;
@@ -272,7 +274,7 @@ static int initiate_layout_draining(struct pnfs_cb_lrecall_info *cb_info)
if (rv == NFS4_OK) {
lo->plh_block_lgets++;
if (nfs4_asynch_forget_layouts(lo, &args->cbl_range,
- cb_info->pcl_notify_bit,
+ cb_info->pcl_notify_idx,
&cb_info->pcl_count,
&free_me_list))
rv = NFS4ERR_DELAY;
@@ -314,7 +316,7 @@ static int initiate_layout_draining(struct pnfs_cb_lrecall_info *cb_info)
spin_lock(&lo->inode->i_lock);
set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
if (nfs4_asynch_forget_layouts(lo, &range,
- cb_info->pcl_notify_bit,
+ cb_info->pcl_notify_idx,
&cb_info->pcl_count,
&free_me_list))
rv = NFS4ERR_DELAY;
@@ -334,8 +336,7 @@ static u32 do_callback_layoutrecall(struct nfs_client *clp,
struct cb_layoutrecallargs *args)
{
struct pnfs_cb_lrecall_info *new;
- atomic_t **ptr;
- int bit_num;
+ int i;
u32 res;
dprintk("%s enter, type=%i\n", __func__, args->cbl_recall_type);
@@ -353,22 +354,26 @@ static u32 do_callback_layoutrecall(struct nfs_client *clp,
kfree(new);
res = NFS4ERR_DELAY;
spin_unlock(&clp->cl_lock);
+ dprintk("%s: too many layout recalls\n", __func__);
goto out;
}
clp->cl_cb_lrecall_count++;
/* Adding to the list will block conflicting LGET activity */
list_add_tail(&new->pcl_list, &clp->cl_layoutrecalls);
- for (bit_num = 0, ptr = clp->cl_drain_notification; *ptr; ptr++)
- bit_num++;
- *ptr = &new->pcl_count;
- new->pcl_notify_bit = bit_num;
+ for (i = 0; i < PNFS_MAX_CB_LRECALLS; i++)
+ if (!clp->cl_drain_notification[i]) {
+ clp->cl_drain_notification[i] = &new->pcl_count;
+ break;
+ }
+ BUG_ON(i >= PNFS_MAX_CB_LRECALLS);
+ new->pcl_notify_idx = i;
spin_unlock(&clp->cl_lock);
res = initiate_layout_draining(new);
if (res || atomic_dec_and_test(&new->pcl_count)) {
spin_lock(&clp->cl_lock);
list_del(&new->pcl_list);
clp->cl_cb_lrecall_count--;
- clp->cl_drain_notification[1 << bit_num] = NULL;
+ clp->cl_drain_notification[new->pcl_notify_idx] = NULL;
rpc_wake_up(&clp->cl_rpcwaitq_recall);
spin_unlock(&clp->cl_lock);
if (res == NFS4_OK) {
diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 7507054..8c5f4b4 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -642,7 +642,7 @@ send_layoutget(struct pnfs_layout_hdr *lo,
bool nfs4_asynch_forget_layouts(struct pnfs_layout_hdr *lo,
struct pnfs_layout_range *range,
- int notify_bit, atomic_t *notify_count,
+ int notify_idx, atomic_t *notify_count,
struct list_head *tmp_list)
{
bool rv = false;
@@ -652,7 +652,7 @@ bool nfs4_asynch_forget_layouts(struct pnfs_layout_hdr *lo,
dprintk("%s\n", __func__);
list_for_each_entry_safe(lseg, tmp, &lo->segs, fi_list)
if (should_free_lseg(&lseg->range, range)) {
- lseg->pls_notify_mask |= (1 << notify_bit);
+ lseg->pls_notify_mask |= (1 << notify_idx);
atomic_inc(notify_count);
mark_lseg_invalid(lseg, tmp_list);
rv = true;
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index 11c6ee4..092559b 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -179,7 +179,7 @@ struct pnfs_device {
struct pnfs_cb_lrecall_info {
struct list_head pcl_list; /* hook into cl_layoutrecalls list */
atomic_t pcl_count;
- int pcl_notify_bit;
+ int pcl_notify_idx;
struct nfs_client *pcl_clp;
struct inode *pcl_ino;
struct cb_layoutrecallargs pcl_args;
--
1.7.3.4
This is not needed anymore and is causes bug on the retry path
after getting NFS4ERR_DELAY when nfs4_xdr_dec_cb_layout sees cb == NULL
Signed-off-by: Benny Halevy <[email protected]>
---
fs/nfsd/nfs4callback.c | 3 ---
1 files changed, 0 insertions(+), 3 deletions(-)
diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c
index 8351f67..3fb989a 100644
--- a/fs/nfsd/nfs4callback.c
+++ b/fs/nfsd/nfs4callback.c
@@ -876,9 +876,6 @@ static void nfsd4_cb_done_sequence(struct rpc_task *task,
rpc_wake_up_next(&clp->cl_cb_waitq);
dprintk("%s: freed slot, new seqid=%d\n", __func__,
clp->cl_cb_session->se_cb_seq_nr);
-
- /* We're done looking into the sequence information */
- task->tk_msg.rpc_resp = NULL;
}
}
--
1.7.3.4