* Create and use svc_rdma_wq instead of using the system workqueue and
flush_scheduled_work(). This workqueue is necessary to serve as
flushing domain for rdma->sc_work which is used to destroy itself
and thus can't be flushed explicitly.
* Replace cancel_delayed_work() + flush_scheduled_work() with
cancel_delayed_work_sync().
* Implement synchronous connect in xprt_rdma_connect() using
flush_delayed_work() on the rdma_connect work instead of using
flush_scheduled_work().
This is to prepare for the deprecation and removal of
flush_scheduled_work().
Signed-off-by: Tejun Heo <[email protected]>
---
net/sunrpc/xprtrdma/svc_rdma.c | 11 ++++++++++-
net/sunrpc/xprtrdma/svc_rdma_transport.c | 6 +++++-
net/sunrpc/xprtrdma/transport.c | 5 ++---
3 files changed, 17 insertions(+), 5 deletions(-)
Index: work/net/sunrpc/xprtrdma/svc_rdma.c
===================================================================
--- work.orig/net/sunrpc/xprtrdma/svc_rdma.c
+++ work/net/sunrpc/xprtrdma/svc_rdma.c
@@ -43,6 +43,7 @@
#include <linux/slab.h>
#include <linux/fs.h>
#include <linux/sysctl.h>
+#include <linux/workqueue.h>
#include <linux/sunrpc/clnt.h>
#include <linux/sunrpc/sched.h>
#include <linux/sunrpc/svc_rdma.h>
@@ -74,6 +75,8 @@ atomic_t rdma_stat_sq_prod;
struct kmem_cache *svc_rdma_map_cachep;
struct kmem_cache *svc_rdma_ctxt_cachep;
+struct workqueue_struct *svc_rdma_wq;
+
/*
* This function implements reading and resetting an atomic_t stat
* variable through read/write to a proc file. Any write to the file
@@ -231,7 +234,7 @@ static ctl_table svcrdma_root_table[] =
void svc_rdma_cleanup(void)
{
dprintk("SVCRDMA Module Removed, deregister RPC RDMA transport\n");
- flush_scheduled_work();
+ destroy_workqueue(svc_rdma_wq);
if (svcrdma_table_header) {
unregister_sysctl_table(svcrdma_table_header);
svcrdma_table_header = NULL;
@@ -249,6 +252,11 @@ int svc_rdma_init(void)
dprintk("\tsq_depth : %d\n",
svcrdma_max_requests * RPCRDMA_SQ_DEPTH_MULT);
dprintk("\tmax_inline : %d\n", svcrdma_max_req_size);
+
+ svc_rdma_wq = alloc_workqueue("svc_rdma", 0, 0);
+ if (!svc_rdma_wq)
+ return -ENOMEM;
+
if (!svcrdma_table_header)
svcrdma_table_header =
register_sysctl_table(svcrdma_root_table);
@@ -283,6 +291,7 @@ int svc_rdma_init(void)
kmem_cache_destroy(svc_rdma_map_cachep);
err0:
unregister_sysctl_table(svcrdma_table_header);
+ destroy_workqueue(svc_rdma_wq);
return -ENOMEM;
}
MODULE_AUTHOR("Tom Tucker <[email protected]>");
Index: work/net/sunrpc/xprtrdma/svc_rdma_transport.c
===================================================================
--- work.orig/net/sunrpc/xprtrdma/svc_rdma_transport.c
+++ work/net/sunrpc/xprtrdma/svc_rdma_transport.c
@@ -45,6 +45,7 @@
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
+#include <linux/workqueue.h>
#include <rdma/ib_verbs.h>
#include <rdma/rdma_cm.h>
#include <linux/sunrpc/svc_rdma.h>
@@ -89,6 +90,9 @@ struct svc_xprt_class svc_rdma_class = {
/* WR context cache. Created in svc_rdma.c */
extern struct kmem_cache *svc_rdma_ctxt_cachep;
+/* Workqueue created in svc_rdma.c */
+extern struct workqueue_struct *svc_rdma_wq;
+
struct svc_rdma_op_ctxt *svc_rdma_get_context(struct svcxprt_rdma *xprt)
{
struct svc_rdma_op_ctxt *ctxt;
@@ -1184,7 +1188,7 @@ static void svc_rdma_free(struct svc_xpr
struct svcxprt_rdma *rdma =
container_of(xprt, struct svcxprt_rdma, sc_xprt);
INIT_WORK(&rdma->sc_work, __svc_rdma_free);
- schedule_work(&rdma->sc_work);
+ queue_work(svc_rdma_wq, &rdma->sc_work);
}
static int svc_rdma_has_wspace(struct svc_xprt *xprt)
Index: work/net/sunrpc/xprtrdma/transport.c
===================================================================
--- work.orig/net/sunrpc/xprtrdma/transport.c
+++ work/net/sunrpc/xprtrdma/transport.c
@@ -237,8 +237,7 @@ xprt_rdma_destroy(struct rpc_xprt *xprt)
dprintk("RPC: %s: called\n", __func__);
- cancel_delayed_work(&r_xprt->rdma_connect);
- flush_scheduled_work();
+ cancel_delayed_work_sync(&r_xprt->rdma_connect);
xprt_clear_connected(xprt);
@@ -460,7 +459,7 @@ xprt_rdma_connect(struct rpc_task *task)
} else {
schedule_delayed_work(&r_xprt->rdma_connect, 0);
if (!RPC_IS_ASYNC(task))
- flush_scheduled_work();
+ flush_delayed_work(&r_xprt->rdma_connect);
}
}
On 10/15/10 2:11 PM, Tejun Heo wrote:
> On 10/15/2010 07:46 PM, Tom Tucker wrote:
>> On 10/15/10 12:11 PM, J. Bruce Fields wrote:
>>> Tom, could you take a look?
>>>
>> This looks reasonable to me.
>>
>> My bet is that it wasn't tested though due to hardware
>> availability. I'll test it this weekend and let you know how it
>> goes.
> Yeap, only compile tested. Testing would be much appreciated.
>
Tejun,
I've tested this now with iozone etc... works like a charm ;-)
Tom
> Thanks.
>
On 10/15/10 12:11 PM, J. Bruce Fields wrote:
> Tom, could you take a look?
>
This looks reasonable to me.
My bet is that it wasn't tested though due to hardware availability. I'll
test it this weekend and let you know how it goes.
Tom
> --b.
>
> On Fri, Oct 15, 2010 at 05:49:27PM +0200, Tejun Heo wrote:
>> * Create and use svc_rdma_wq instead of using the system workqueue and
>> flush_scheduled_work(). This workqueue is necessary to serve as
>> flushing domain for rdma->sc_work which is used to destroy itself
>> and thus can't be flushed explicitly.
>>
>> * Replace cancel_delayed_work() + flush_scheduled_work() with
>> cancel_delayed_work_sync().
>>
>> * Implement synchronous connect in xprt_rdma_connect() using
>> flush_delayed_work() on the rdma_connect work instead of using
>> flush_scheduled_work().
>>
>> This is to prepare for the deprecation and removal of
>> flush_scheduled_work().
>>
>> Signed-off-by: Tejun Heo<[email protected]>
>> ---
>> net/sunrpc/xprtrdma/svc_rdma.c | 11 ++++++++++-
>> net/sunrpc/xprtrdma/svc_rdma_transport.c | 6 +++++-
>> net/sunrpc/xprtrdma/transport.c | 5 ++---
>> 3 files changed, 17 insertions(+), 5 deletions(-)
>>
>> Index: work/net/sunrpc/xprtrdma/svc_rdma.c
>> ===================================================================
>> --- work.orig/net/sunrpc/xprtrdma/svc_rdma.c
>> +++ work/net/sunrpc/xprtrdma/svc_rdma.c
>> @@ -43,6 +43,7 @@
>> #include<linux/slab.h>
>> #include<linux/fs.h>
>> #include<linux/sysctl.h>
>> +#include<linux/workqueue.h>
>> #include<linux/sunrpc/clnt.h>
>> #include<linux/sunrpc/sched.h>
>> #include<linux/sunrpc/svc_rdma.h>
>> @@ -74,6 +75,8 @@ atomic_t rdma_stat_sq_prod;
>> struct kmem_cache *svc_rdma_map_cachep;
>> struct kmem_cache *svc_rdma_ctxt_cachep;
>>
>> +struct workqueue_struct *svc_rdma_wq;
>> +
>> /*
>> * This function implements reading and resetting an atomic_t stat
>> * variable through read/write to a proc file. Any write to the file
>> @@ -231,7 +234,7 @@ static ctl_table svcrdma_root_table[] =
>> void svc_rdma_cleanup(void)
>> {
>> dprintk("SVCRDMA Module Removed, deregister RPC RDMA transport\n");
>> - flush_scheduled_work();
>> + destroy_workqueue(svc_rdma_wq);
>> if (svcrdma_table_header) {
>> unregister_sysctl_table(svcrdma_table_header);
>> svcrdma_table_header = NULL;
>> @@ -249,6 +252,11 @@ int svc_rdma_init(void)
>> dprintk("\tsq_depth : %d\n",
>> svcrdma_max_requests * RPCRDMA_SQ_DEPTH_MULT);
>> dprintk("\tmax_inline : %d\n", svcrdma_max_req_size);
>> +
>> + svc_rdma_wq = alloc_workqueue("svc_rdma", 0, 0);
>> + if (!svc_rdma_wq)
>> + return -ENOMEM;
>> +
>> if (!svcrdma_table_header)
>> svcrdma_table_header =
>> register_sysctl_table(svcrdma_root_table);
>> @@ -283,6 +291,7 @@ int svc_rdma_init(void)
>> kmem_cache_destroy(svc_rdma_map_cachep);
>> err0:
>> unregister_sysctl_table(svcrdma_table_header);
>> + destroy_workqueue(svc_rdma_wq);
>> return -ENOMEM;
>> }
>> MODULE_AUTHOR("Tom Tucker<[email protected]>");
>> Index: work/net/sunrpc/xprtrdma/svc_rdma_transport.c
>> ===================================================================
>> --- work.orig/net/sunrpc/xprtrdma/svc_rdma_transport.c
>> +++ work/net/sunrpc/xprtrdma/svc_rdma_transport.c
>> @@ -45,6 +45,7 @@
>> #include<linux/sched.h>
>> #include<linux/slab.h>
>> #include<linux/spinlock.h>
>> +#include<linux/workqueue.h>
>> #include<rdma/ib_verbs.h>
>> #include<rdma/rdma_cm.h>
>> #include<linux/sunrpc/svc_rdma.h>
>> @@ -89,6 +90,9 @@ struct svc_xprt_class svc_rdma_class = {
>> /* WR context cache. Created in svc_rdma.c */
>> extern struct kmem_cache *svc_rdma_ctxt_cachep;
>>
>> +/* Workqueue created in svc_rdma.c */
>> +extern struct workqueue_struct *svc_rdma_wq;
>> +
>> struct svc_rdma_op_ctxt *svc_rdma_get_context(struct svcxprt_rdma *xprt)
>> {
>> struct svc_rdma_op_ctxt *ctxt;
>> @@ -1184,7 +1188,7 @@ static void svc_rdma_free(struct svc_xpr
>> struct svcxprt_rdma *rdma =
>> container_of(xprt, struct svcxprt_rdma, sc_xprt);
>> INIT_WORK(&rdma->sc_work, __svc_rdma_free);
>> - schedule_work(&rdma->sc_work);
>> + queue_work(svc_rdma_wq,&rdma->sc_work);
>> }
>>
>> static int svc_rdma_has_wspace(struct svc_xprt *xprt)
>> Index: work/net/sunrpc/xprtrdma/transport.c
>> ===================================================================
>> --- work.orig/net/sunrpc/xprtrdma/transport.c
>> +++ work/net/sunrpc/xprtrdma/transport.c
>> @@ -237,8 +237,7 @@ xprt_rdma_destroy(struct rpc_xprt *xprt)
>>
>> dprintk("RPC: %s: called\n", __func__);
>>
>> - cancel_delayed_work(&r_xprt->rdma_connect);
>> - flush_scheduled_work();
>> + cancel_delayed_work_sync(&r_xprt->rdma_connect);
>>
>> xprt_clear_connected(xprt);
>>
>> @@ -460,7 +459,7 @@ xprt_rdma_connect(struct rpc_task *task)
>> } else {
>> schedule_delayed_work(&r_xprt->rdma_connect, 0);
>> if (!RPC_IS_ASYNC(task))
>> - flush_scheduled_work();
>> + flush_delayed_work(&r_xprt->rdma_connect);
>> }
>> }
>>
>> --
>> To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
>> the body of a message to [email protected]
>> More majordomo info at http://vger.kernel.org/majordomo-info.html
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to [email protected]
> More majordomo info at http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at http://www.tux.org/lkml/
On 10/15/2010 07:46 PM, Tom Tucker wrote:
> On 10/15/10 12:11 PM, J. Bruce Fields wrote:
>> Tom, could you take a look?
>>
> This looks reasonable to me.
>
> My bet is that it wasn't tested though due to hardware
> availability. I'll test it this weekend and let you know how it
> goes.
Yeap, only compile tested. Testing would be much appreciated.
Thanks.
--
tejun
On Wed, Oct 20, 2010 at 11:05:05AM -0500, Tom Tucker wrote:
> On 10/15/10 2:11 PM, Tejun Heo wrote:
> >On 10/15/2010 07:46 PM, Tom Tucker wrote:
> >> On 10/15/10 12:11 PM, J. Bruce Fields wrote:
> >>>Tom, could you take a look?
> >>>
> >>This looks reasonable to me.
> >>
> >>My bet is that it wasn't tested though due to hardware
> >>availability. I'll test it this weekend and let you know how it
> >>goes.
> >Yeap, only compile tested. Testing would be much appreciated.
> >
> Tejun,
>
> I've tested this now with iozone etc... works like a charm ;-)
Applied, thanks!
--b.
On 10/21/2010 05:16 PM, J. Bruce Fields wrote:
>> I've tested this now with iozone etc... works like a charm ;-)
>
> Applied, thanks!
Great, thank you. :-)
--
tejun
Tom, could you take a look?
--b.
On Fri, Oct 15, 2010 at 05:49:27PM +0200, Tejun Heo wrote:
> * Create and use svc_rdma_wq instead of using the system workqueue and
> flush_scheduled_work(). This workqueue is necessary to serve as
> flushing domain for rdma->sc_work which is used to destroy itself
> and thus can't be flushed explicitly.
>
> * Replace cancel_delayed_work() + flush_scheduled_work() with
> cancel_delayed_work_sync().
>
> * Implement synchronous connect in xprt_rdma_connect() using
> flush_delayed_work() on the rdma_connect work instead of using
> flush_scheduled_work().
>
> This is to prepare for the deprecation and removal of
> flush_scheduled_work().
>
> Signed-off-by: Tejun Heo <[email protected]>
> ---
> net/sunrpc/xprtrdma/svc_rdma.c | 11 ++++++++++-
> net/sunrpc/xprtrdma/svc_rdma_transport.c | 6 +++++-
> net/sunrpc/xprtrdma/transport.c | 5 ++---
> 3 files changed, 17 insertions(+), 5 deletions(-)
>
> Index: work/net/sunrpc/xprtrdma/svc_rdma.c
> ===================================================================
> --- work.orig/net/sunrpc/xprtrdma/svc_rdma.c
> +++ work/net/sunrpc/xprtrdma/svc_rdma.c
> @@ -43,6 +43,7 @@
> #include <linux/slab.h>
> #include <linux/fs.h>
> #include <linux/sysctl.h>
> +#include <linux/workqueue.h>
> #include <linux/sunrpc/clnt.h>
> #include <linux/sunrpc/sched.h>
> #include <linux/sunrpc/svc_rdma.h>
> @@ -74,6 +75,8 @@ atomic_t rdma_stat_sq_prod;
> struct kmem_cache *svc_rdma_map_cachep;
> struct kmem_cache *svc_rdma_ctxt_cachep;
>
> +struct workqueue_struct *svc_rdma_wq;
> +
> /*
> * This function implements reading and resetting an atomic_t stat
> * variable through read/write to a proc file. Any write to the file
> @@ -231,7 +234,7 @@ static ctl_table svcrdma_root_table[] =
> void svc_rdma_cleanup(void)
> {
> dprintk("SVCRDMA Module Removed, deregister RPC RDMA transport\n");
> - flush_scheduled_work();
> + destroy_workqueue(svc_rdma_wq);
> if (svcrdma_table_header) {
> unregister_sysctl_table(svcrdma_table_header);
> svcrdma_table_header = NULL;
> @@ -249,6 +252,11 @@ int svc_rdma_init(void)
> dprintk("\tsq_depth : %d\n",
> svcrdma_max_requests * RPCRDMA_SQ_DEPTH_MULT);
> dprintk("\tmax_inline : %d\n", svcrdma_max_req_size);
> +
> + svc_rdma_wq = alloc_workqueue("svc_rdma", 0, 0);
> + if (!svc_rdma_wq)
> + return -ENOMEM;
> +
> if (!svcrdma_table_header)
> svcrdma_table_header =
> register_sysctl_table(svcrdma_root_table);
> @@ -283,6 +291,7 @@ int svc_rdma_init(void)
> kmem_cache_destroy(svc_rdma_map_cachep);
> err0:
> unregister_sysctl_table(svcrdma_table_header);
> + destroy_workqueue(svc_rdma_wq);
> return -ENOMEM;
> }
> MODULE_AUTHOR("Tom Tucker <[email protected]>");
> Index: work/net/sunrpc/xprtrdma/svc_rdma_transport.c
> ===================================================================
> --- work.orig/net/sunrpc/xprtrdma/svc_rdma_transport.c
> +++ work/net/sunrpc/xprtrdma/svc_rdma_transport.c
> @@ -45,6 +45,7 @@
> #include <linux/sched.h>
> #include <linux/slab.h>
> #include <linux/spinlock.h>
> +#include <linux/workqueue.h>
> #include <rdma/ib_verbs.h>
> #include <rdma/rdma_cm.h>
> #include <linux/sunrpc/svc_rdma.h>
> @@ -89,6 +90,9 @@ struct svc_xprt_class svc_rdma_class = {
> /* WR context cache. Created in svc_rdma.c */
> extern struct kmem_cache *svc_rdma_ctxt_cachep;
>
> +/* Workqueue created in svc_rdma.c */
> +extern struct workqueue_struct *svc_rdma_wq;
> +
> struct svc_rdma_op_ctxt *svc_rdma_get_context(struct svcxprt_rdma *xprt)
> {
> struct svc_rdma_op_ctxt *ctxt;
> @@ -1184,7 +1188,7 @@ static void svc_rdma_free(struct svc_xpr
> struct svcxprt_rdma *rdma =
> container_of(xprt, struct svcxprt_rdma, sc_xprt);
> INIT_WORK(&rdma->sc_work, __svc_rdma_free);
> - schedule_work(&rdma->sc_work);
> + queue_work(svc_rdma_wq, &rdma->sc_work);
> }
>
> static int svc_rdma_has_wspace(struct svc_xprt *xprt)
> Index: work/net/sunrpc/xprtrdma/transport.c
> ===================================================================
> --- work.orig/net/sunrpc/xprtrdma/transport.c
> +++ work/net/sunrpc/xprtrdma/transport.c
> @@ -237,8 +237,7 @@ xprt_rdma_destroy(struct rpc_xprt *xprt)
>
> dprintk("RPC: %s: called\n", __func__);
>
> - cancel_delayed_work(&r_xprt->rdma_connect);
> - flush_scheduled_work();
> + cancel_delayed_work_sync(&r_xprt->rdma_connect);
>
> xprt_clear_connected(xprt);
>
> @@ -460,7 +459,7 @@ xprt_rdma_connect(struct rpc_task *task)
> } else {
> schedule_delayed_work(&r_xprt->rdma_connect, 0);
> if (!RPC_IS_ASYNC(task))
> - flush_scheduled_work();
> + flush_delayed_work(&r_xprt->rdma_connect);
> }
> }
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
> the body of a message to [email protected]
> More majordomo info at http://vger.kernel.org/majordomo-info.html