by Trond Myklebust

[permalink] [raw]

Subject: Re: [PATCH v2 10/13] sunrpc: add add sysfs directory per xprt under each xprt_switch

On Thu, 2021-04-15 at 23:52 -0400, Olga Kornievskaia wrote:
> From: Olga Kornievskaia <[email protected]>
>
> Add individual transport directories under each transport switch
> group. For instance, for each nconnect=X connections there will be
> a transport directory. Naming conventions also identifies transport
> type -- xprt-<id>-<type> where type is udp, tcp, rdma, local, bc.
>
> Signed-off-by: Olga Kornievskaia <[email protected]>
> ---
> include/linux/sunrpc/xprt.h | 1 +
> net/sunrpc/sysfs.c          | 83
> +++++++++++++++++++++++++++++++++++++
> net/sunrpc/sysfs.h          | 9 ++++
> net/sunrpc/xprtmultipath.c | 2 +
> 4 files changed, 95 insertions(+)
>
> diff --git a/include/linux/sunrpc/xprt.h
> b/include/linux/sunrpc/xprt.h
> index a2edcc42e6c4..1e4906759a6a 100644
> --- a/include/linux/sunrpc/xprt.h
> +++ b/include/linux/sunrpc/xprt.h
> @@ -291,6 +291,7 @@ struct rpc_xprt {
> #endif
>         struct rcu_head         rcu;
>         const struct xprt_class *xprt_class;
> +       void                    *xprt_sysfs;
> };
>
> #if defined(CONFIG_SUNRPC_BACKCHANNEL)
> diff --git a/net/sunrpc/sysfs.c b/net/sunrpc/sysfs.c
> index ce2cad1b6aa6..5410d8fe1181 100644
> --- a/net/sunrpc/sysfs.c
> +++ b/net/sunrpc/sysfs.c
> @@ -81,6 +81,14 @@ static void rpc_sysfs_xprt_switch_release(struct
> kobject *kobj)
>         kfree(xprt_switch);
> }
>
> +static void rpc_sysfs_xprt_switch_xprt_release(struct kobject *kobj)
> +{
> +       struct rpc_sysfs_xprt_switch_xprt *xprt;
> +
> +       xprt = container_of(kobj, struct rpc_sysfs_xprt_switch_xprt,
> kobject);
> +       kfree(xprt);
> +}
> +
> static const void *rpc_sysfs_client_namespace(struct kobject *kobj)
> {
>         return container_of(kobj, struct rpc_sysfs_client, kobject)-
> >net;
> @@ -91,6 +99,12 @@ static const void
> *rpc_sysfs_xprt_switch_namespace(struct kobject *kobj)
>         return container_of(kobj, struct rpc_sysfs_xprt_switch,
> kobject)->net;
> }
>
> +static const void *rpc_sysfs_xprt_switch_xprt_namespace(struct
> kobject *kobj)
> +{
> +       return container_of(kobj, struct rpc_sysfs_xprt_switch_xprt,
> +                           kobject)->net;
> +}
> +
> static struct kobj_type rpc_sysfs_client_type = {
>         .release = rpc_sysfs_client_release,
>         .sysfs_ops = &kobj_sysfs_ops,
> @@ -103,6 +117,12 @@ static struct kobj_type
> rpc_sysfs_xprt_switch_type = {
>         .namespace = rpc_sysfs_xprt_switch_namespace,
> };
>
> +static struct kobj_type rpc_sysfs_xprt_switch_xprt_type = {
> +       .release = rpc_sysfs_xprt_switch_xprt_release,
> +       .sysfs_ops = &kobj_sysfs_ops,
> +       .namespace = rpc_sysfs_xprt_switch_xprt_namespace,
> +};
> +
> void rpc_sysfs_exit(void)
> {
>         kobject_put(rpc_sunrpc_client_kobj);
> @@ -149,6 +169,39 @@ rpc_sysfs_xprt_switch_alloc(struct kobject
> *parent,
>         return NULL;
> }
>
> +static struct rpc_sysfs_xprt_switch_xprt *
> +rpc_sysfs_xprt_switch_xprt_alloc(struct kobject *parent,
> +                                struct rpc_xprt *xprt,
> +                                struct net *net)
> +{
> +       struct rpc_sysfs_xprt_switch_xprt *p;
> +
> +       p = kzalloc(sizeof(*p), GFP_KERNEL);

Ditto. This cannot be called from a locked environment.

> +       if (p) {
> +               char type[6];
> +
> +               p->net = net;
> +               p->kobject.kset = rpc_sunrpc_kset;
> +               if (xprt->xprt_class->ident == XPRT_TRANSPORT_RDMA)
> +                       snprintf(type, sizeof(type), "rdma");
> +               else if (xprt->xprt_class->ident ==
> XPRT_TRANSPORT_TCP)
> +                       snprintf(type, sizeof(type), "tcp");
> +               else if (xprt->xprt_class->ident ==
> XPRT_TRANSPORT_UDP)
> +                       snprintf(type, sizeof(type), "udp");
> +               else if (xprt->xprt_class->ident ==
> XPRT_TRANSPORT_LOCAL)
> +                       snprintf(type, sizeof(type), "local");
> +               else if (xprt->xprt_class->ident ==
> XPRT_TRANSPORT_BC_TCP)
> +                       snprintf(type, sizeof(type), "bc");
> +               if (kobject_init_and_add(&p->kobject,
> +
> &rpc_sysfs_xprt_switch_xprt_type,
> +                                        parent, "xprt-%d-%s", xprt-
> >id,
> +                                        type) == 0)
> +                       return p;
> +               kobject_put(&p->kobject);
> +       }
> +       return NULL;
> +}
> +
> void rpc_sysfs_client_setup(struct rpc_clnt *clnt,
>                             struct rpc_xprt_switch *xprt_switch,
>                             struct net *net)
> @@ -197,6 +250,23 @@ void rpc_sysfs_xprt_switch_setup(struct
> rpc_xprt_switch *xprt_switch,
>         }
> }
>
> +void rpc_sysfs_xprt_switch_xprt_setup(struct rpc_xprt_switch
> *xprt_switch,
> +                                     struct rpc_xprt *xprt)
> +{
> +       struct rpc_sysfs_xprt_switch_xprt *rpc_xprt_switch_xprt;
> +       struct rpc_sysfs_xprt_switch *switch_obj =
> +               (struct rpc_sysfs_xprt_switch *)xprt_switch-
> >xps_sysfs;
> +
> +       rpc_xprt_switch_xprt =
> +               rpc_sysfs_xprt_switch_xprt_alloc(&switch_obj-
> >kobject,
> +                                                xprt, xprt-
> >xprt_net);
> +       if (rpc_xprt_switch_xprt) {
> +               xprt->xprt_sysfs = rpc_xprt_switch_xprt;
> +               rpc_xprt_switch_xprt->xprt = xprt;
> +               kobject_uevent(&rpc_xprt_switch_xprt->kobject,
> KOBJ_ADD);
> +       }
> +}
> +
> void rpc_sysfs_client_destroy(struct rpc_clnt *clnt)
> {
>         struct rpc_sysfs_client *rpc_client = clnt->cl_sysfs;
> @@ -225,3 +295,16 @@ void rpc_sysfs_xprt_switch_destroy(struct
> rpc_xprt_switch *xprt_switch)
>                 xprt_switch->xps_sysfs = NULL;
>         }
> }
> +
> +void rpc_sysfs_xprt_switch_xprt_destroy(struct rpc_xprt *xprt)
> +{
> +       struct rpc_sysfs_xprt_switch_xprt *rpc_xprt_switch_xprt =
> +                       xprt->xprt_sysfs;
> +
> +       if (rpc_xprt_switch_xprt) {
> +               kobject_uevent(&rpc_xprt_switch_xprt->kobject,
> KOBJ_REMOVE);
> +               kobject_del(&rpc_xprt_switch_xprt->kobject);
> +               kobject_put(&rpc_xprt_switch_xprt->kobject);
> +               xprt->xprt_sysfs = NULL;
> +       }
> +}
> diff --git a/net/sunrpc/sysfs.h b/net/sunrpc/sysfs.h
> index 9a0625b1cd65..52abe443ee8d 100644
> --- a/net/sunrpc/sysfs.h
> +++ b/net/sunrpc/sysfs.h
> @@ -19,6 +19,12 @@ struct rpc_sysfs_xprt_switch {
>         struct rpc_xprt *xprt;
> };
>
> +struct rpc_sysfs_xprt_switch_xprt {
> +       struct kobject kobject;
> +       struct net *net;
> +       struct rpc_xprt *xprt;
> +};
> +
> int rpc_sysfs_init(void);
> void rpc_sysfs_exit(void);
>
> @@ -29,5 +35,8 @@ void rpc_sysfs_client_destroy(struct rpc_clnt
> *clnt);
> void rpc_sysfs_xprt_switch_setup(struct rpc_xprt_switch
> *xprt_switch,
>                                  struct rpc_xprt *xprt);
> void rpc_sysfs_xprt_switch_destroy(struct rpc_xprt_switch *xprt);
> +void rpc_sysfs_xprt_switch_xprt_setup(struct rpc_xprt_switch
> *xprt_switch,
> +                                     struct rpc_xprt *xprt);
> +void rpc_sysfs_xprt_switch_xprt_destroy(struct rpc_xprt *xprt);
>
> #endif
> diff --git a/net/sunrpc/xprtmultipath.c b/net/sunrpc/xprtmultipath.c
> index 1ed16e4cc465..eba45cbf8448 100644
> --- a/net/sunrpc/xprtmultipath.c
> +++ b/net/sunrpc/xprtmultipath.c
> @@ -33,6 +33,7 @@ static void xprt_switch_add_xprt_locked(struct
> rpc_xprt_switch *xps,
> {
>         if (unlikely(xprt_get(xprt) == NULL))
>                 return;
> +       rpc_sysfs_xprt_switch_xprt_setup(xps, xprt);
>         list_add_tail_rcu(&xprt->xprt_switch, &xps->xps_xprt_list);
>         smp_wmb();
>         if (xps->xps_nxprts == 0)
> @@ -66,6 +67,7 @@ static void xprt_switch_remove_xprt_locked(struct
> rpc_xprt_switch *xps,
>                 return;
>         xps->xps_nactive--;
>         xps->xps_nxprts--;
> +       rpc_sysfs_xprt_switch_xprt_destroy(xprt);
>         if (xps->xps_nxprts == 0)
>                 xps->xps_net = NULL;
>         smp_wmb();

--
Trond Myklebust
Linux NFS client maintainer, Hammerspace
[email protected]

2021-04-26 15:30:59

by Olga Kornievskaia

[permalink] [raw]

Subject: Re: [PATCH v2 08/13] sunrpc: add xprt_switch direcotry to sunrpc's sysfs

On Fri, Apr 23, 2021 at 5:20 PM Trond Myklebust <[email protected]> wrote:
>
> On Thu, 2021-04-15 at 23:52 -0400, Olga Kornievskaia wrote:
> > From: Olga Kornievskaia <[email protected]>
> >
> > Add xprt_switch directory to the sysfs and create individual
> > xprt_swith subdirectories for multipath transport group.
> >
> > Signed-off-by: Olga Kornievskaia <[email protected]>
> > ---
> > include/linux/sunrpc/xprtmultipath.h | 1 +
> > net/sunrpc/sysfs.c | 97
> > ++++++++++++++++++++++++++--
> > net/sunrpc/sysfs.h | 10 +++
> > net/sunrpc/xprtmultipath.c | 4 ++
> > 4 files changed, 105 insertions(+), 7 deletions(-)
> >
> > diff --git a/include/linux/sunrpc/xprtmultipath.h
> > b/include/linux/sunrpc/xprtmultipath.h
> > index ef95a6f18ccf..47b0a85cdcfa 100644
> > --- a/include/linux/sunrpc/xprtmultipath.h
> > +++ b/include/linux/sunrpc/xprtmultipath.h
> > @@ -24,6 +24,7 @@ struct rpc_xprt_switch {
> >
> > const struct rpc_xprt_iter_ops *xps_iter_ops;
> >
> > + void *xps_sysfs;
> > struct rcu_head xps_rcu;
> > };
> >
> > diff --git a/net/sunrpc/sysfs.c b/net/sunrpc/sysfs.c
> > index d14d54f33c65..0c34330714ab 100644
> > --- a/net/sunrpc/sysfs.c
> > +++ b/net/sunrpc/sysfs.c
> > @@ -7,7 +7,7 @@
> > #include "sysfs.h"
> >
> > static struct kset *rpc_sunrpc_kset;
> > -static struct kobject *rpc_sunrpc_client_kobj;
> > +static struct kobject *rpc_sunrpc_client_kobj,
> > *rpc_sunrpc_xprt_switch_kobj;
> >
> > static void rpc_sysfs_object_release(struct kobject *kobj)
> > {
> > @@ -47,13 +47,22 @@ int rpc_sysfs_init(void)
> > rpc_sunrpc_kset = kset_create_and_add("sunrpc", NULL,
> > kernel_kobj);
> > if (!rpc_sunrpc_kset)
> > return -ENOMEM;
> > - rpc_sunrpc_client_kobj = rpc_sysfs_object_alloc("client",
> > rpc_sunrpc_kset, NULL);
> > - if (!rpc_sunrpc_client_kobj) {
> > - kset_unregister(rpc_sunrpc_kset);
> > - rpc_sunrpc_client_kobj = NULL;
> > - return -ENOMEM;
> > - }
> > + rpc_sunrpc_client_kobj =
> > + rpc_sysfs_object_alloc("rpc-clients",
> > rpc_sunrpc_kset, NULL);
> > + if (!rpc_sunrpc_client_kobj)
> > + goto err_client;
> > + rpc_sunrpc_xprt_switch_kobj =
> > + rpc_sysfs_object_alloc("xprt-switches",
> > rpc_sunrpc_kset, NULL);
> > + if (!rpc_sunrpc_xprt_switch_kobj)
> > + goto err_switch;
> > return 0;
> > +err_switch:
> > + kobject_put(rpc_sunrpc_client_kobj);
> > + rpc_sunrpc_client_kobj = NULL;
> > +err_client:
> > + kset_unregister(rpc_sunrpc_kset);
> > + rpc_sunrpc_kset = NULL;
> > + return -ENOMEM;
> > }
> >
> > static void rpc_sysfs_client_release(struct kobject *kobj)
> > @@ -64,20 +73,40 @@ static void rpc_sysfs_client_release(struct
> > kobject *kobj)
> > kfree(c);
> > }
> >
> > +static void rpc_sysfs_xprt_switch_release(struct kobject *kobj)
> > +{
> > + struct rpc_sysfs_xprt_switch *xprt_switch;
> > +
> > + xprt_switch = container_of(kobj, struct
> > rpc_sysfs_xprt_switch, kobject);
> > + kfree(xprt_switch);
> > +}
> > +
> > static const void *rpc_sysfs_client_namespace(struct kobject *kobj)
> > {
> > return container_of(kobj, struct rpc_sysfs_client, kobject)-
> > >net;
> > }
> >
> > +static const void *rpc_sysfs_xprt_switch_namespace(struct kobject
> > *kobj)
> > +{
> > + return container_of(kobj, struct rpc_sysfs_xprt_switch,
> > kobject)->net;
> > +}
> > +
> > static struct kobj_type rpc_sysfs_client_type = {
> > .release = rpc_sysfs_client_release,
> > .sysfs_ops = &kobj_sysfs_ops,
> > .namespace = rpc_sysfs_client_namespace,
> > };
> >
> > +static struct kobj_type rpc_sysfs_xprt_switch_type = {
> > + .release = rpc_sysfs_xprt_switch_release,
> > + .sysfs_ops = &kobj_sysfs_ops,
> > + .namespace = rpc_sysfs_xprt_switch_namespace,
> > +};
> > +
> > void rpc_sysfs_exit(void)
> > {
> > kobject_put(rpc_sunrpc_client_kobj);
> > + kobject_put(rpc_sunrpc_xprt_switch_kobj);
> > kset_unregister(rpc_sunrpc_kset);
> > }
> >
> > @@ -99,6 +128,27 @@ static struct rpc_sysfs_client
> > *rpc_sysfs_client_alloc(struct kobject *parent,
> > return NULL;
> > }
> >
> > +static struct rpc_sysfs_xprt_switch *
> > +rpc_sysfs_xprt_switch_alloc(struct kobject *parent,
> > + struct rpc_xprt_switch *xprt_switch,
> > + struct net *net)
> > +{
> > + struct rpc_sysfs_xprt_switch *p;
> > +
> > + p = kzalloc(sizeof(*p), GFP_KERNEL);
>
> Again, this needs to use the allocation mode of xprt_switch_alloc().
>
> > + if (p) {
> > + p->net = net;
> > + p->kobject.kset = rpc_sunrpc_kset;
> > + if (kobject_init_and_add(&p->kobject,
> > + &rpc_sysfs_xprt_switch_type,
> > + parent, "switch-%d",
> > + xprt_switch->xps_id) == 0)
> > + return p;
> > + kobject_put(&p->kobject);
> > + }
> > + return NULL;
> > +}
> > +
> > void rpc_sysfs_client_setup(struct rpc_clnt *clnt, struct net *net)
> > {
> > struct rpc_sysfs_client *rpc_client;
> > @@ -110,6 +160,27 @@ void rpc_sysfs_client_setup(struct rpc_clnt
> > *clnt, struct net *net)
> > }
> > }
> >
> > +void rpc_sysfs_xprt_switch_setup(struct rpc_xprt_switch
> > *xprt_switch,
> > + struct rpc_xprt *xprt)
> > +{
> > + struct rpc_sysfs_xprt_switch *rpc_xprt_switch;
> > + struct net *net;
> > +
> > + if (xprt_switch->xps_net)
> > + net = xprt_switch->xps_net;
> > + else
> > + net = xprt->xprt_net;
> > + rpc_xprt_switch =
> > + rpc_sysfs_xprt_switch_alloc(rpc_sunrpc_xprt_switch_ko
> > bj,
> > + xprt_switch, net);
> > + if (rpc_xprt_switch) {
> > + xprt_switch->xps_sysfs = rpc_xprt_switch;
> > + rpc_xprt_switch->xprt_switch = xprt_switch;
> > + rpc_xprt_switch->xprt = xprt;
> > + kobject_uevent(&rpc_xprt_switch->kobject, KOBJ_ADD);
>
> This probably cannot be called from a locked environment.

rpc_sysfs_xprt_switch_setup() isn't called from a locked environment.
Going backwards:
rpc_sysfs_xprt_switch_setup() from xprt_switch_alloc() from
rpc_create_xprt()/rpc_switch_client_transport() none of those are
called with a lock.

>
> > + }
> > +}
> > +
> > void rpc_sysfs_client_destroy(struct rpc_clnt *clnt)
> > {
> > struct rpc_sysfs_client *rpc_client = clnt->cl_sysfs;
> > @@ -121,3 +192,15 @@ void rpc_sysfs_client_destroy(struct rpc_clnt
> > *clnt)
> > clnt->cl_sysfs = NULL;
> > }
> > }
> > +
> > +void rpc_sysfs_xprt_switch_destroy(struct rpc_xprt_switch
> > *xprt_switch)
> > +{
> > + struct rpc_sysfs_xprt_switch *rpc_xprt_switch = xprt_switch-
> > >xps_sysfs;
> > +
> > + if (rpc_xprt_switch) {
> > + kobject_uevent(&rpc_xprt_switch->kobject,
> > KOBJ_REMOVE);
> > + kobject_del(&rpc_xprt_switch->kobject);
> > + kobject_put(&rpc_xprt_switch->kobject);
> > + xprt_switch->xps_sysfs = NULL;
> > + }
> > +}
> > diff --git a/net/sunrpc/sysfs.h b/net/sunrpc/sysfs.h
> > index c46afc848993..9b6acd3fd3dc 100644
> > --- a/net/sunrpc/sysfs.h
> > +++ b/net/sunrpc/sysfs.h
> > @@ -10,10 +10,20 @@ struct rpc_sysfs_client {
> > struct net *net;
> > };
> >
> > +struct rpc_sysfs_xprt_switch {
> > + struct kobject kobject;
> > + struct net *net;
> > + struct rpc_xprt_switch *xprt_switch;
> > + struct rpc_xprt *xprt;
> > +};
> > +
> > int rpc_sysfs_init(void);
> > void rpc_sysfs_exit(void);
> >
> > void rpc_sysfs_client_setup(struct rpc_clnt *clnt, struct net *net);
> > void rpc_sysfs_client_destroy(struct rpc_clnt *clnt);
> > +void rpc_sysfs_xprt_switch_setup(struct rpc_xprt_switch
> > *xprt_switch,
> > + struct rpc_xprt *xprt);
> > +void rpc_sysfs_xprt_switch_destroy(struct rpc_xprt_switch *xprt);
> >
> > #endif
> > diff --git a/net/sunrpc/xprtmultipath.c b/net/sunrpc/xprtmultipath.c
> > index b71dd95ad7de..1ed16e4cc465 100644
> > --- a/net/sunrpc/xprtmultipath.c
> > +++ b/net/sunrpc/xprtmultipath.c
> > @@ -19,6 +19,8 @@
> > #include <linux/sunrpc/addr.h>
> > #include <linux/sunrpc/xprtmultipath.h>
> >
> > +#include "sysfs.h"
> > +
> > typedef struct rpc_xprt *(*xprt_switch_find_xprt_t)(struct
> > rpc_xprt_switch *xps,
> > const struct rpc_xprt *cur);
> >
> > @@ -133,6 +135,7 @@ struct rpc_xprt_switch *xprt_switch_alloc(struct
> > rpc_xprt *xprt,
> > xps->xps_net = NULL;
> > INIT_LIST_HEAD(&xps->xps_xprt_list);
> > xps->xps_iter_ops = &rpc_xprt_iter_singular;
> > + rpc_sysfs_xprt_switch_setup(xps, xprt);
> > xprt_switch_add_xprt_locked(xps, xprt);
> > }
> >
> > @@ -161,6 +164,7 @@ static void xprt_switch_free(struct kref *kref)
> > struct rpc_xprt_switch, xps_kref);
> >
> > xprt_switch_free_entries(xps);
> > + rpc_sysfs_xprt_switch_destroy(xps);
> > xprt_switch_free_id(xps);
> > kfree_rcu(xps, xps_rcu);
> > }
>
> --
> Trond Myklebust
> Linux NFS client maintainer, Hammerspace
> [email protected]
>
>