From: Greg Banks Subject: [PATCH 008 of 11] knfsd: add svc_set_num_threads Date: Tue, 25 Jul 2006 15:14:47 +1000 Message-ID: <1153804486.21040.20.camel@hole.melbourne.sgi.com> Mime-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Cc: Linux NFS Mailing List Return-path: Received: from sc8-sf-mx1-b.sourceforge.net ([10.3.1.91] helo=mail.sourceforge.net) by sc8-sf-list2-new.sourceforge.net with esmtp (Exim 4.43) id 1G5FFt-0003pG-CT for nfs@lists.sourceforge.net; Mon, 24 Jul 2006 22:14:53 -0700 Received: from omx2-ext.sgi.com ([192.48.171.19] helo=omx2.sgi.com) by mail.sourceforge.net with esmtp (Exim 4.44) id 1G5FFt-0004Cj-EE for nfs@lists.sourceforge.net; Mon, 24 Jul 2006 22:14:53 -0700 To: Neil Brown List-Id: "Discussion of NFS under Linux development, interoperability, and testing." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Sender: nfs-bounces@lists.sourceforge.net Errors-To: nfs-bounces@lists.sourceforge.net knfsd: Currently knfsd keeps its own list of all nfsd threads in nfssvc.c; add a new way of managing the list of all threads in a svc_serv. Add svc_create_pooled() to allow creation of a svc_serv whose threads are managed by the sunrpc code. Add svc_set_num_threads() to manage the number of threads in a service, either per-pool or globally across the service. Signed-off-by: Greg Banks --- include/linux/sunrpc/svc.h | 19 +++- net/sunrpc/sunrpc_syms.c | 2 net/sunrpc/svc.c | 137 +++++++++++++++++++++++++++++++++- 3 files changed, 150 insertions(+), 8 deletions(-) Index: linus-git/include/linux/sunrpc/svc.h =================================================================== --- linus-git.orig/include/linux/sunrpc/svc.h 2006-07-24 20:54:04.638635111 +1000 +++ linus-git/include/linux/sunrpc/svc.h 2006-07-24 22:16:36.041218126 +1000 @@ -17,6 +17,10 @@ #include #include +/* + * This is the RPC server thread function prototype + */ +typedef void (*svc_thread_fn)(struct svc_rqst *); /* * @@ -34,6 +38,7 @@ struct svc_pool { struct list_head sp_threads; /* idle server threads */ struct list_head sp_sockets; /* pending sockets */ unsigned int sp_nrthreads; /* # of threads in pool */ + struct list_head sp_all_threads; /* all server threads */ } ____cacheline_aligned_in_smp; /* @@ -62,6 +67,10 @@ struct svc_serv { char * sv_name; /* service name */ unsigned int sv_nrpools; /* number of thread pools */ struct svc_pool * sv_pools; /* array of thread pools */ + struct module * sv_module; /* optional module to count when + * adding threads */ + svc_thread_fn sv_function; /* main function for threads */ + int sv_kill_signal; /* signal to kill threads */ }; /* @@ -141,6 +150,7 @@ static inline void svc_putu32(struct kve */ struct svc_rqst { struct list_head rq_list; /* idle list */ + struct list_head rq_all; /* all threads list */ struct svc_sock * rq_sock; /* socket */ struct sockaddr_in rq_addr; /* peer address */ int rq_addrlen; @@ -195,6 +205,7 @@ struct svc_rqst { * to prevent encrypting page * cache pages */ wait_queue_head_t rq_wait; /* synchronization */ + struct task_struct *rq_task; /* service thread */ }; /* @@ -336,16 +347,14 @@ struct svc_procedure { }; /* - * This is the RPC server thread function prototype - */ -typedef void (*svc_thread_fn)(struct svc_rqst *); - -/* * Function prototypes. */ struct svc_serv * svc_create(struct svc_program *, unsigned int); int svc_create_thread(svc_thread_fn, struct svc_serv *); void svc_exit_thread(struct svc_rqst *); +struct svc_serv * svc_create_pooled(struct svc_program *, unsigned int, + svc_thread_fn, int sig, struct module *); +int svc_set_num_threads(struct svc_serv *, struct svc_pool *, int); void svc_destroy(struct svc_serv *); int svc_process(struct svc_serv *, struct svc_rqst *); int svc_register(struct svc_serv *, int, unsigned short); Index: linus-git/net/sunrpc/sunrpc_syms.c =================================================================== --- linus-git.orig/net/sunrpc/sunrpc_syms.c 2006-07-24 20:43:35.596757996 +1000 +++ linus-git/net/sunrpc/sunrpc_syms.c 2006-07-24 22:16:36.153203582 +1000 @@ -73,6 +73,8 @@ EXPORT_SYMBOL(put_rpccred); /* RPC server stuff */ EXPORT_SYMBOL(svc_create); EXPORT_SYMBOL(svc_create_thread); +EXPORT_SYMBOL(svc_create_pooled); +EXPORT_SYMBOL(svc_set_num_threads); EXPORT_SYMBOL(svc_exit_thread); EXPORT_SYMBOL(svc_destroy); EXPORT_SYMBOL(svc_drop); Index: linus-git/net/sunrpc/svc.c =================================================================== --- linus-git.orig/net/sunrpc/svc.c 2006-07-24 20:54:04.874604317 +1000 +++ linus-git/net/sunrpc/svc.c 2006-07-24 22:16:36.157203063 +1000 @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include #include @@ -25,8 +27,8 @@ /* * Create an RPC service */ -struct svc_serv * -svc_create(struct svc_program *prog, unsigned int bufsize) +static struct svc_serv * +__svc_create(struct svc_program *prog, unsigned int bufsize, int npools) { struct svc_serv *serv; int vers; @@ -59,7 +61,7 @@ svc_create(struct svc_program *prog, uns init_timer(&serv->sv_temptimer); spin_lock_init(&serv->sv_lock); - serv->sv_nrpools = 1; + serv->sv_nrpools = npools; if (!(serv->sv_pools = (struct svc_pool *) kmalloc(sizeof(struct svc_pool) * serv->sv_nrpools, GFP_KERNEL))) { @@ -77,6 +79,7 @@ svc_create(struct svc_program *prog, uns pool->sp_id = i; INIT_LIST_HEAD(&pool->sp_threads); INIT_LIST_HEAD(&pool->sp_sockets); + INIT_LIST_HEAD(&pool->sp_all_threads); spin_lock_init(&pool->sp_lock); } @@ -87,6 +90,29 @@ svc_create(struct svc_program *prog, uns return serv; } +struct svc_serv * +svc_create(struct svc_program *prog, unsigned int bufsize) +{ + return __svc_create(prog, bufsize, /*npools*/1); +} + +struct svc_serv * +svc_create_pooled(struct svc_program *prog, unsigned int bufsize, + svc_thread_fn func, int sig, struct module *mod) +{ + struct svc_serv *serv; + + serv = __svc_create(prog, bufsize, /*npools*/1); + + if (serv != NULL) { + serv->sv_function = func; + serv->sv_kill_signal = sig; + serv->sv_module = mod; + } + + return serv; +} + /* * Destroy an RPC service. Should be called with the BKL held */ @@ -197,6 +223,7 @@ __svc_create_thread(svc_thread_fn func, serv->sv_nrthreads++; spin_lock_bh(&pool->sp_lock); pool->sp_nrthreads++; + list_add(&rqstp->rq_all, &pool->sp_all_threads); spin_unlock_bh(&pool->sp_lock); rqstp->rq_server = serv; rqstp->rq_pool = pool; @@ -223,6 +250,109 @@ svc_create_thread(svc_thread_fn func, st } /* + * Choose a pool in which to create a new thread, for svc_set_num_threads + */ +static inline struct svc_pool * +choose_pool(struct svc_serv *serv, struct svc_pool *pool, unsigned int *state) +{ + if (pool != NULL) + return pool; + + return &serv->sv_pools[(*state)++ % serv->sv_nrpools]; +} + +/* + * Choose a thread to kill, for svc_set_num_threads + */ +static inline struct task_struct * +choose_victim(struct svc_serv *serv, struct svc_pool *pool, unsigned int *state) +{ + unsigned int i; + struct task_struct *task = NULL; + + if (pool != NULL) { + spin_lock_bh(&pool->sp_lock); + } else { + /* choose a pool in round-robin fashion */ + for (i = 0 ; i < serv->sv_nrpools ; i++) { + pool = &serv->sv_pools[--(*state) % serv->sv_nrpools]; + spin_lock_bh(&pool->sp_lock); + if (!list_empty(&pool->sp_all_threads)) + goto found_pool; + spin_unlock_bh(&pool->sp_lock); + } + return NULL; + } + +found_pool: + if (!list_empty(&pool->sp_all_threads)) { + struct svc_rqst *rqstp; + + /* + * Remove from the pool->sp_all_threads list + * so we don't try to kill it again. + */ + rqstp = list_entry(pool->sp_all_threads.next, struct svc_rqst, rq_all); + list_del_init(&rqstp->rq_all); + task = rqstp->rq_task; + } + spin_unlock_bh(&pool->sp_lock); + + return task; +} + +/* + * Create or destroy enough new threads to make the number + * of threads the given number. If `pool' is non-NULL, applies + * only to threads in that pool, otherwise round-robins between + * all pools. Must be called with a svc_get() reference and + * the BKL held. + * + * Destroying threads relies on the service threads filling in + * rqstp->rq_task, which only the nfs ones do. Assumes the serv + * has been created using svc_create_pooled(). + * + * Based on code that used to be in nfsd_svc() but tweaked + * to be pool-aware. + */ +int +svc_set_num_threads(struct svc_serv *serv, struct svc_pool *pool, int nrservs) +{ + struct task_struct *victim; + int error = 0; + unsigned int state = serv->sv_nrthreads-1; + + if (pool == NULL) { + /* The -1 assumes caller has done a svc_get() */ + nrservs -= (serv->sv_nrthreads-1); + } else { + spin_lock_bh(&pool->sp_lock); + nrservs -= pool->sp_nrthreads; + spin_unlock_bh(&pool->sp_lock); + } + + /* create new threads */ + while (nrservs > 0) { + nrservs--; + __module_get(serv->sv_module); + error = __svc_create_thread(serv->sv_function, serv, + choose_pool(serv, pool, &state)); + if (error < 0) { + module_put(serv->sv_module); + break; + } + } + /* destroy old threads */ + while (nrservs < 0 && + (victim = choose_victim(serv, pool, &state)) != NULL) { + send_sig(serv->sv_kill_signal, victim, 1); + nrservs++; + } + + return error; +} + +/* * Called from a server thread as it's exiting. Caller must hold BKL. */ void @@ -238,6 +368,7 @@ svc_exit_thread(struct svc_rqst *rqstp) spin_lock_bh(&pool->sp_lock); pool->sp_nrthreads--; + list_del(&rqstp->rq_all); spin_unlock_bh(&pool->sp_lock); kfree(rqstp); -- Greg Banks, R&D Software Engineer, SGI Australian Software Group. I don't speak for SGI. ------------------------------------------------------------------------- Take Surveys. Earn Cash. Influence the Future of IT Join SourceForge.net's Techsay panel and you'll get the chance to share your opinions on IT & business topics through brief surveys -- and earn cash http://www.techsay.com/default.php?page=join.php&p=sourceforge&CID=DEVDEV _______________________________________________ NFS maillist - NFS@lists.sourceforge.net https://lists.sourceforge.net/lists/listinfo/nfs