Implement a new provider method for dma-buf base memory registration.
Signed-off-by: Shunsuke Mie <[email protected]>
---
providers/rxe/rxe.c | 21 +++++++++++++++++++++
1 file changed, 21 insertions(+)
diff --git a/providers/rxe/rxe.c b/providers/rxe/rxe.c
index 3c3ea8bb..84e00e60 100644
--- a/providers/rxe/rxe.c
+++ b/providers/rxe/rxe.c
@@ -239,6 +239,26 @@ static struct ibv_mr *rxe_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
return &vmr->ibv_mr;
}
+static struct ibv_mr *rxe_reg_dmabuf_mr(struct ibv_pd *pd, uint64_t offset,
+ size_t length, uint64_t iova, int fd,
+ int access)
+{
+ struct verbs_mr *vmr;
+ int ret;
+
+ vmr = malloc(sizeof(*vmr));
+ if (!vmr)
+ return NULL;
+
+ ret = ibv_cmd_reg_dmabuf_mr(pd, offset, length, iova, fd, access, vmr);
+ if (ret) {
+ free(vmr);
+ return NULL;
+ }
+
+ return &vmr->ibv_mr;
+}
+
static int rxe_dereg_mr(struct verbs_mr *vmr)
{
int ret;
@@ -1706,6 +1726,7 @@ static const struct verbs_context_ops rxe_ctx_ops = {
.alloc_pd = rxe_alloc_pd,
.dealloc_pd = rxe_dealloc_pd,
.reg_mr = rxe_reg_mr,
+ .reg_dmabuf_mr = rxe_reg_dmabuf_mr,
.dereg_mr = rxe_dereg_mr,
.alloc_mw = rxe_alloc_mw,
.dealloc_mw = rxe_dealloc_mw,
--
2.17.1
On Thu, Sep 30, 2021 at 2:20 PM Shunsuke Mie <[email protected]> wrote:
>
> Implement a new provider method for dma-buf base memory registration.
>
> Signed-off-by: Shunsuke Mie <[email protected]>
> ---
> providers/rxe/rxe.c | 21 +++++++++++++++++++++
> 1 file changed, 21 insertions(+)
>
> diff --git a/providers/rxe/rxe.c b/providers/rxe/rxe.c
> index 3c3ea8bb..84e00e60 100644
> --- a/providers/rxe/rxe.c
> +++ b/providers/rxe/rxe.c
> @@ -239,6 +239,26 @@ static struct ibv_mr *rxe_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
> return &vmr->ibv_mr;
> }
>
> +static struct ibv_mr *rxe_reg_dmabuf_mr(struct ibv_pd *pd, uint64_t offset,
> + size_t length, uint64_t iova, int fd,
> + int access)
> +{
> + struct verbs_mr *vmr;
> + int ret;
> +
> + vmr = malloc(sizeof(*vmr));
> + if (!vmr)
> + return NULL;
> +
Do we need to set vmr to zero like the following?
memset(vmr, 0, sizeof(*vmr));
Zhu Yanjun
> + ret = ibv_cmd_reg_dmabuf_mr(pd, offset, length, iova, fd, access, vmr);
> + if (ret) {
> + free(vmr);
> + return NULL;
> + }
> +
> + return &vmr->ibv_mr;
> +}
> +
> static int rxe_dereg_mr(struct verbs_mr *vmr)
> {
> int ret;
> @@ -1706,6 +1726,7 @@ static const struct verbs_context_ops rxe_ctx_ops = {
> .alloc_pd = rxe_alloc_pd,
> .dealloc_pd = rxe_dealloc_pd,
> .reg_mr = rxe_reg_mr,
> + .reg_dmabuf_mr = rxe_reg_dmabuf_mr,
> .dereg_mr = rxe_dereg_mr,
> .alloc_mw = rxe_alloc_mw,
> .dealloc_mw = rxe_dealloc_mw,
> --
> 2.17.1
>
2021年9月30日(木) 15:37 Zhu Yanjun <[email protected]>:
>
> On Thu, Sep 30, 2021 at 2:20 PM Shunsuke Mie <[email protected]> wrote:
> >
> > Implement a new provider method for dma-buf base memory registration.
> >
> > Signed-off-by: Shunsuke Mie <[email protected]>
> > ---
> > providers/rxe/rxe.c | 21 +++++++++++++++++++++
> > 1 file changed, 21 insertions(+)
> >
> > diff --git a/providers/rxe/rxe.c b/providers/rxe/rxe.c
> > index 3c3ea8bb..84e00e60 100644
> > --- a/providers/rxe/rxe.c
> > +++ b/providers/rxe/rxe.c
> > @@ -239,6 +239,26 @@ static struct ibv_mr *rxe_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
> > return &vmr->ibv_mr;
> > }
> >
> > +static struct ibv_mr *rxe_reg_dmabuf_mr(struct ibv_pd *pd, uint64_t offset,
> > + size_t length, uint64_t iova, int fd,
> > + int access)
> > +{
> > + struct verbs_mr *vmr;
> > + int ret;
> > +
> > + vmr = malloc(sizeof(*vmr));
> > + if (!vmr)
> > + return NULL;
> > +
>
> Do we need to set vmr to zero like the following?
>
> memset(vmr, 0, sizeof(*vmr));
>
> Zhu Yanjun
Thank you for your quick response.
I think it is better to clear the vmr. Actually the mlx5 driver allocates
the vmr using calloc().
In addition, rxe_reg_mr() (not rxe_reg_dmabuf_mr()) is used the malloc
and not clear the vmr. I think It has to be fixed too. Should I make
another patch to fix this problem?
Thanks a lot.
Shunsuke
~
On Thu, Sep 30, 2021 at 2:58 PM Shunsuke Mie <[email protected]> wrote:
>
> 2021年9月30日(木) 15:37 Zhu Yanjun <[email protected]>:
> >
> > On Thu, Sep 30, 2021 at 2:20 PM Shunsuke Mie <[email protected]> wrote:
> > >
> > > Implement a new provider method for dma-buf base memory registration.
> > >
> > > Signed-off-by: Shunsuke Mie <[email protected]>
> > > ---
> > > providers/rxe/rxe.c | 21 +++++++++++++++++++++
> > > 1 file changed, 21 insertions(+)
> > >
> > > diff --git a/providers/rxe/rxe.c b/providers/rxe/rxe.c
> > > index 3c3ea8bb..84e00e60 100644
> > > --- a/providers/rxe/rxe.c
> > > +++ b/providers/rxe/rxe.c
> > > @@ -239,6 +239,26 @@ static struct ibv_mr *rxe_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
> > > return &vmr->ibv_mr;
> > > }
> > >
> > > +static struct ibv_mr *rxe_reg_dmabuf_mr(struct ibv_pd *pd, uint64_t offset,
> > > + size_t length, uint64_t iova, int fd,
> > > + int access)
> > > +{
> > > + struct verbs_mr *vmr;
> > > + int ret;
> > > +
> > > + vmr = malloc(sizeof(*vmr));
> > > + if (!vmr)
> > > + return NULL;
> > > +
> >
> > Do we need to set vmr to zero like the following?
> >
> > memset(vmr, 0, sizeof(*vmr));
> >
> > Zhu Yanjun
> Thank you for your quick response.
>
> I think it is better to clear the vmr. Actually the mlx5 driver allocates
> the vmr using calloc().
>
> In addition, rxe_reg_mr() (not rxe_reg_dmabuf_mr()) is used the malloc
> and not clear the vmr. I think It has to be fixed too. Should I make
> another patch to fix this problem?
Yes. Please.
Zhu Yanjun
>
> Thanks a lot.
> Shunsuke
>
> ~
2021年9月30日(木) 16:23 Zhu Yanjun <[email protected]>:
>
> On Thu, Sep 30, 2021 at 2:58 PM Shunsuke Mie <[email protected]> wrote:
> >
> > 2021年9月30日(木) 15:37 Zhu Yanjun <[email protected]>:
> > >
> > > On Thu, Sep 30, 2021 at 2:20 PM Shunsuke Mie <[email protected]> wrote:
> > > >
> > > > Implement a new provider method for dma-buf base memory registration.
> > > >
> > > > Signed-off-by: Shunsuke Mie <[email protected]>
> > > > ---
> > > > providers/rxe/rxe.c | 21 +++++++++++++++++++++
> > > > 1 file changed, 21 insertions(+)
> > > >
> > > > diff --git a/providers/rxe/rxe.c b/providers/rxe/rxe.c
> > > > index 3c3ea8bb..84e00e60 100644
> > > > --- a/providers/rxe/rxe.c
> > > > +++ b/providers/rxe/rxe.c
> > > > @@ -239,6 +239,26 @@ static struct ibv_mr *rxe_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
> > > > return &vmr->ibv_mr;
> > > > }
> > > >
> > > > +static struct ibv_mr *rxe_reg_dmabuf_mr(struct ibv_pd *pd, uint64_t offset,
> > > > + size_t length, uint64_t iova, int fd,
> > > > + int access)
> > > > +{
> > > > + struct verbs_mr *vmr;
> > > > + int ret;
> > > > +
> > > > + vmr = malloc(sizeof(*vmr));
> > > > + if (!vmr)
> > > > + return NULL;
> > > > +
> > >
> > > Do we need to set vmr to zero like the following?
> > >
> > > memset(vmr, 0, sizeof(*vmr));
> > >
> > > Zhu Yanjun
> > Thank you for your quick response.
> >
> > I think it is better to clear the vmr. Actually the mlx5 driver allocates
> > the vmr using calloc().
> >
> > In addition, rxe_reg_mr() (not rxe_reg_dmabuf_mr()) is used the malloc
> > and not clear the vmr. I think It has to be fixed too. Should I make
> > another patch to fix this problem?
>
> Yes. Please.
>
> Zhu Yanjun
>
> >
> > Thanks a lot.
> > Shunsuke
> >
> > ~
I looked into the vmr more, but there was no need to clear it. Moreover,
some implementations also use malloc without memory clear.
Thanks,
Shunsuke
On Thu, Sep 30, 2021 at 7:06 PM Shunsuke Mie <[email protected]> wrote:
>
> 2021年9月30日(木) 16:23 Zhu Yanjun <[email protected]>:
> >
> > On Thu, Sep 30, 2021 at 2:58 PM Shunsuke Mie <[email protected]> wrote:
> > >
> > > 2021年9月30日(木) 15:37 Zhu Yanjun <[email protected]>:
> > > >
> > > > On Thu, Sep 30, 2021 at 2:20 PM Shunsuke Mie <[email protected]> wrote:
> > > > >
> > > > > Implement a new provider method for dma-buf base memory registration.
> > > > >
> > > > > Signed-off-by: Shunsuke Mie <[email protected]>
> > > > > ---
> > > > > providers/rxe/rxe.c | 21 +++++++++++++++++++++
> > > > > 1 file changed, 21 insertions(+)
> > > > >
> > > > > diff --git a/providers/rxe/rxe.c b/providers/rxe/rxe.c
> > > > > index 3c3ea8bb..84e00e60 100644
> > > > > --- a/providers/rxe/rxe.c
> > > > > +++ b/providers/rxe/rxe.c
> > > > > @@ -239,6 +239,26 @@ static struct ibv_mr *rxe_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
> > > > > return &vmr->ibv_mr;
> > > > > }
> > > > >
> > > > > +static struct ibv_mr *rxe_reg_dmabuf_mr(struct ibv_pd *pd, uint64_t offset,
> > > > > + size_t length, uint64_t iova, int fd,
> > > > > + int access)
> > > > > +{
> > > > > + struct verbs_mr *vmr;
> > > > > + int ret;
> > > > > +
> > > > > + vmr = malloc(sizeof(*vmr));
> > > > > + if (!vmr)
> > > > > + return NULL;
> > > > > +
> > > >
> > > > Do we need to set vmr to zero like the following?
> > > >
> > > > memset(vmr, 0, sizeof(*vmr));
> > > >
> > > > Zhu Yanjun
> > > Thank you for your quick response.
> > >
> > > I think it is better to clear the vmr. Actually the mlx5 driver allocates
> > > the vmr using calloc().
> > >
> > > In addition, rxe_reg_mr() (not rxe_reg_dmabuf_mr()) is used the malloc
> > > and not clear the vmr. I think It has to be fixed too. Should I make
> > > another patch to fix this problem?
> >
> > Yes. Please.
> >
> > Zhu Yanjun
> >
> > >
> > > Thanks a lot.
> > > Shunsuke
> > >
> > > ~
>
> I looked into the vmr more, but there was no need to clear it. Moreover,
> some implementations also use malloc without memory clear.
>
I confronted a lot of problems with memory not initialization.
And a latest bug is https://www.spinics.net/lists/linux-rdma/msg105001.html
So it is a good habit to clear a newly allocated memory.
Zhu Yanjun
> Thanks,
> Shunsuke
2021年9月30日(木) 20:18 Zhu Yanjun <[email protected]>:
>
> On Thu, Sep 30, 2021 at 7:06 PM Shunsuke Mie <[email protected]> wrote:
> >
> > 2021年9月30日(木) 16:23 Zhu Yanjun <[email protected]>:
> > >
> > > On Thu, Sep 30, 2021 at 2:58 PM Shunsuke Mie <[email protected]> wrote:
> > > >
> > > > 2021年9月30日(木) 15:37 Zhu Yanjun <[email protected]>:
> > > > >
> > > > > On Thu, Sep 30, 2021 at 2:20 PM Shunsuke Mie <[email protected]> wrote:
> > > > > >
> > > > > > Implement a new provider method for dma-buf base memory registration.
> > > > > >
> > > > > > Signed-off-by: Shunsuke Mie <[email protected]>
> > > > > > ---
> > > > > > providers/rxe/rxe.c | 21 +++++++++++++++++++++
> > > > > > 1 file changed, 21 insertions(+)
> > > > > >
> > > > > > diff --git a/providers/rxe/rxe.c b/providers/rxe/rxe.c
> > > > > > index 3c3ea8bb..84e00e60 100644
> > > > > > --- a/providers/rxe/rxe.c
> > > > > > +++ b/providers/rxe/rxe.c
> > > > > > @@ -239,6 +239,26 @@ static struct ibv_mr *rxe_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
> > > > > > return &vmr->ibv_mr;
> > > > > > }
> > > > > >
> > > > > > +static struct ibv_mr *rxe_reg_dmabuf_mr(struct ibv_pd *pd, uint64_t offset,
> > > > > > + size_t length, uint64_t iova, int fd,
> > > > > > + int access)
> > > > > > +{
> > > > > > + struct verbs_mr *vmr;
> > > > > > + int ret;
> > > > > > +
> > > > > > + vmr = malloc(sizeof(*vmr));
> > > > > > + if (!vmr)
> > > > > > + return NULL;
> > > > > > +
> > > > >
> > > > > Do we need to set vmr to zero like the following?
> > > > >
> > > > > memset(vmr, 0, sizeof(*vmr));
> > > > >
> > > > > Zhu Yanjun
> > > > Thank you for your quick response.
> > > >
> > > > I think it is better to clear the vmr. Actually the mlx5 driver allocates
> > > > the vmr using calloc().
> > > >
> > > > In addition, rxe_reg_mr() (not rxe_reg_dmabuf_mr()) is used the malloc
> > > > and not clear the vmr. I think It has to be fixed too. Should I make
> > > > another patch to fix this problem?
> > >
> > > Yes. Please.
> > >
> > > Zhu Yanjun
> > >
> > > >
> > > > Thanks a lot.
> > > > Shunsuke
> > > >
> > > > ~
> >
> > I looked into the vmr more, but there was no need to clear it. Moreover,
> > some implementations also use malloc without memory clear.
> >
>
> I confronted a lot of problems with memory not initialization.
> And a latest bug is https://www.spinics.net/lists/linux-rdma/msg105001.html
>
> So it is a good habit to clear a newly allocated memory.
>
> Zhu Yanjun
>
> > Thanks,
> > Shunsuke
Ok, I understand that thought. I'll send a patch.
Shunsuke.