MIME-Version: 1.0
References: <20230203100418.2981144-1-mie@igel.co.jp> <20230203100418.2981144-4-mie@igel.co.jp>
 <20230203051844-mutt-send-email-mst@kernel.org>
In-Reply-To: <20230203051844-mutt-send-email-mst@kernel.org>
From:   Shunsuke Mie <mie@igel.co.jp>
Date:   Tue, 7 Feb 2023 20:05:08 +0900
Message-ID: <CANXvt5r10bec+X=j35ertph9i+d08bkxDG7ffwUjc5z_1NgKtw@mail.gmail.com>
Subject: Re: [RFC PATCH 3/4] PCI: endpoint: Introduce virtio library for EP functions
To:     "Michael S. Tsirkin" <mst@redhat.com>
Cc:     Lorenzo Pieralisi <lpieralisi@kernel.org>,
        =?UTF-8?Q?Krzysztof_Wilczy=C5=84ski?= <kw@linux.com>,
        Manivannan Sadhasivam <mani@kernel.org>,
        Kishon Vijay Abraham I <kishon@kernel.org>,
        Bjorn Helgaas <bhelgaas@google.com>,
        Jason Wang <jasowang@redhat.com>, Frank Li <Frank.Li@nxp.com>,
        Jon Mason <jdmason@kudzu.us>,
        Ren Zhijie <renzhijie2@huawei.com>,
        Takanari Hayama <taki@igel.co.jp>,
        linux-kernel@vger.kernel.org, linux-pci@vger.kernel.org,
        virtualization@lists.linux-foundation.org
Content-Type: text/plain; charset="UTF-8"
Content-Transfer-Encoding: quoted-printable
Precedence: bulk

2023=E5=B9=B42=E6=9C=883=E6=97=A5(=E9=87=91) 19:20 Michael S. Tsirkin <mst@=
redhat.com>:
>
> On Fri, Feb 03, 2023 at 07:04:17PM +0900, Shunsuke Mie wrote:
> > Add a new library to access a virtio ring located on PCIe host memory. =
The
> > library generates struct pci_epf_vringh that is introduced in this patc=
h.
> > The struct has a vringh member, so vringh APIs can be used to access th=
e
> > virtio ring.
> >
> > Signed-off-by: Shunsuke Mie <mie@igel.co.jp>
> > Signed-off-by: Takanari Hayama <taki@igel.co.jp>
> > ---
> >  drivers/pci/endpoint/Kconfig          |   7 ++
> >  drivers/pci/endpoint/Makefile         |   1 +
> >  drivers/pci/endpoint/pci-epf-virtio.c | 113 ++++++++++++++++++++++++++
> >  include/linux/pci-epf-virtio.h        |  25 ++++++
> >  4 files changed, 146 insertions(+)
> >  create mode 100644 drivers/pci/endpoint/pci-epf-virtio.c
> >  create mode 100644 include/linux/pci-epf-virtio.h
> >
> > diff --git a/drivers/pci/endpoint/Kconfig b/drivers/pci/endpoint/Kconfi=
g
> > index 17bbdc9bbde0..07276dcc43c8 100644
> > --- a/drivers/pci/endpoint/Kconfig
> > +++ b/drivers/pci/endpoint/Kconfig
> > @@ -28,6 +28,13 @@ config PCI_ENDPOINT_CONFIGFS
> >          configure the endpoint function and used to bind the
> >          function with a endpoint controller.
> >
> > +config PCI_ENDPOINT_VIRTIO
> > +     tristate
> > +     depends on PCI_ENDPOINT
> > +     select VHOST_IOMEM
> > +     help
> > +       TODO update this comment
> > +
> >  source "drivers/pci/endpoint/functions/Kconfig"
> >
> >  endmenu
> > diff --git a/drivers/pci/endpoint/Makefile b/drivers/pci/endpoint/Makef=
ile
> > index 95b2fe47e3b0..95712f0a13d1 100644
> > --- a/drivers/pci/endpoint/Makefile
> > +++ b/drivers/pci/endpoint/Makefile
> > @@ -4,5 +4,6 @@
> >  #
> >
> >  obj-$(CONFIG_PCI_ENDPOINT_CONFIGFS)  +=3D pci-ep-cfs.o
> > +obj-$(CONFIG_PCI_ENDPOINT_VIRTIO)    +=3D pci-epf-virtio.o
> >  obj-$(CONFIG_PCI_ENDPOINT)           +=3D pci-epc-core.o pci-epf-core.=
o\
> >                                          pci-epc-mem.o functions/
> > diff --git a/drivers/pci/endpoint/pci-epf-virtio.c b/drivers/pci/endpoi=
nt/pci-epf-virtio.c
> > new file mode 100644
> > index 000000000000..7134ca407a03
> > --- /dev/null
> > +++ b/drivers/pci/endpoint/pci-epf-virtio.c
> > @@ -0,0 +1,113 @@
> > +// SPDX-License-Identifier: GPL-2.0
> > +/*
> > + * Virtio library for PCI Endpoint function
> > + */
> > +#include <linux/kernel.h>
> > +#include <linux/pci-epf-virtio.h>
> > +#include <linux/pci-epc.h>
> > +#include <linux/virtio_pci.h>
> > +
> > +static void __iomem *epf_virtio_map_vq(struct pci_epf *epf, u32 pfn,
> > +                                    size_t size, phys_addr_t *vq_phys)
> > +{
> > +     int err;
> > +     phys_addr_t vq_addr;
> > +     size_t vq_size;
> > +     void __iomem *vq_virt;
> > +
> > +     vq_addr =3D (phys_addr_t)pfn << VIRTIO_PCI_QUEUE_ADDR_SHIFT;
> > +
> > +     vq_size =3D vring_size(size, VIRTIO_PCI_VRING_ALIGN) + 100;
>
> 100?
It is a mistake and will be removed.
> Also ugh, this uses the legacy vring_size.
> Did not look closely but is all this limited to legacy virtio then?
> Pls make sure you code builds with #define VIRTIO_RING_NO_LEGACY.
Thanks for your suggestion, but this device works as a legacy device.
In this case, the NO_LEGACY macro can not applicable I think. Is that right=
?
> > +
> > +     vq_virt =3D pci_epc_mem_alloc_addr(epf->epc, vq_phys, vq_size);
> > +     if (!vq_virt) {
> > +             pr_err("Failed to allocate epc memory\n");
> > +             return ERR_PTR(-ENOMEM);
> > +     }
> > +
> > +     err =3D pci_epc_map_addr(epf->epc, epf->func_no, epf->vfunc_no, *=
vq_phys,
> > +                            vq_addr, vq_size);
> > +     if (err) {
> > +             pr_err("Failed to map virtuqueue to local");
> > +             goto err_free;
> > +     }
> > +
> > +     return vq_virt;
> > +
> > +err_free:
> > +     pci_epc_mem_free_addr(epf->epc, *vq_phys, vq_virt, vq_size);
> > +
> > +     return ERR_PTR(err);
> > +}
> > +
> > +static void epf_virtio_unmap_vq(struct pci_epf *epf, void __iomem *vq_=
virt,
> > +                             phys_addr_t vq_phys, size_t size)
> > +{
> > +     pci_epc_unmap_addr(epf->epc, epf->func_no, epf->vfunc_no, vq_phys=
);
> > +     pci_epc_mem_free_addr(epf->epc, vq_phys, vq_virt,
> > +                           vring_size(size, VIRTIO_PCI_VRING_ALIGN));
> > +}
> > +
> > +/**
> > + * pci_epf_virtio_alloc_vringh() - allocate epf vringh from @pfn
> > + * @epf: the EPF device that communicates to host virtio dirver
> > + * @features: the virtio features of device
> > + * @pfn: page frame number of virtqueue located on host memory. It is
> > + *           passed during virtqueue negotiation.
> > + * @size: a length of virtqueue
> > + */
> > +struct pci_epf_vringh *pci_epf_virtio_alloc_vringh(struct pci_epf *epf=
,
> > +                                                u64 features, u32 pfn,
> > +                                                size_t size)
> > +{
> > +     int err;
> > +     struct vring vring;
> > +     struct pci_epf_vringh *evrh;
> > +
> > +     evrh =3D kmalloc(sizeof(*evrh), GFP_KERNEL);
> > +     if (!evrh) {
> > +             err =3D -ENOMEM;
> > +             goto err_unmap_vq;
> > +     }
> > +
> > +     evrh->size =3D size;
> > +
> > +     evrh->virt =3D epf_virtio_map_vq(epf, pfn, size, &evrh->phys);
> > +     if (IS_ERR(evrh->virt))
> > +             return evrh->virt;
> > +
> > +     vring_init(&vring, size, evrh->virt, VIRTIO_PCI_VRING_ALIGN);
> > +
> > +     err =3D vringh_init_iomem(&evrh->vrh, features, size, false, GFP_=
KERNEL,
> > +                             vring.desc, vring.avail, vring.used);
> > +     if (err)
> > +             goto err_free_epf_vq;
> > +
> > +     return evrh;
> > +
> > +err_free_epf_vq:
> > +     kfree(evrh);
> > +
> > +err_unmap_vq:
> > +     epf_virtio_unmap_vq(epf, evrh->virt, evrh->phys, evrh->size);
> > +
> > +     return ERR_PTR(err);
> > +}
> > +EXPORT_SYMBOL_GPL(pci_epf_virtio_alloc_vringh);
> > +
> > +/**
> > + * pci_epf_virtio_free_vringh() - release allocated epf vring
> > + * @epf: the EPF device that communicates to host virtio dirver
> > + * @evrh: epf vringh to free
> > + */
> > +void pci_epf_virtio_free_vringh(struct pci_epf *epf,
> > +                             struct pci_epf_vringh *evrh)
> > +{
> > +     epf_virtio_unmap_vq(epf, evrh->virt, evrh->phys, evrh->size);
> > +     kfree(evrh);
> > +}
> > +EXPORT_SYMBOL_GPL(pci_epf_virtio_free_vringh);
> > +
> > +MODULE_DESCRIPTION("PCI EP Virtio Library");
> > +MODULE_AUTHOR("Shunsuke Mie <mie@igel.co.jp>");
> > +MODULE_LICENSE("GPL");
> > diff --git a/include/linux/pci-epf-virtio.h b/include/linux/pci-epf-vir=
tio.h
> > new file mode 100644
> > index 000000000000..ae09087919a9
> > --- /dev/null
> > +++ b/include/linux/pci-epf-virtio.h
> > @@ -0,0 +1,25 @@
> > +/* SPDX-License-Identifier: GPL-2.0 */
> > +/*
> > + * PCI Endpoint Function (EPF) for virtio definitions
> > + */
> > +#ifndef __LINUX_PCI_EPF_VIRTIO_H
> > +#define __LINUX_PCI_EPF_VIRTIO_H
> > +
> > +#include <linux/types.h>
> > +#include <linux/vringh.h>
> > +#include <linux/pci-epf.h>
> > +
> > +struct pci_epf_vringh {
> > +     struct vringh vrh;
> > +     void __iomem *virt;
> > +     phys_addr_t phys;
> > +     size_t size;
> > +};
> > +
> > +struct pci_epf_vringh *pci_epf_virtio_alloc_vringh(struct pci_epf *epf=
,
> > +                                                u64 features, u32 pfn,
> > +                                                size_t size);
> > +void pci_epf_virtio_free_vringh(struct pci_epf *epf,
> > +                             struct pci_epf_vringh *evrh);
> > +
> > +#endif // __LINUX_PCI_EPF_VIRTIO_H
> > --
> > 2.25.1
>
Best,
Shunsuke