Hi all
Here is the reconnect support in vduse, The steps
is map the pages from kernel to userland, usespace
app will wirte the vq_info in the pages
And then sync this information in kernel/user-space
I just the split mode, So only use the last_avail_idx
here, Will send the patch for DPDK later
Tested in vduse + dpdk test-pmd
Cindy Lu (4):
vduse: Add the struct to save the vq reconnect info
vduse: Add file operation for mmap
vduse: Add the function for get/free the mapp pages
vduse: update the vq_info in ioctl
drivers/vdpa/vdpa_user/vduse_dev.c | 107 +++++++++++++++++++++++++++++
1 file changed, 107 insertions(+)
--
2.34.3
From: Your Name <[email protected]>
Add the function for get/free pages, ad this info
will saved in dev->reconnect_info
Signed-off-by: Cindy Lu <[email protected]>
---
drivers/vdpa/vdpa_user/vduse_dev.c | 35 ++++++++++++++++++++++++++++++
1 file changed, 35 insertions(+)
diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c
index 1b833bf0ae37..3df1256eccb4 100644
--- a/drivers/vdpa/vdpa_user/vduse_dev.c
+++ b/drivers/vdpa/vdpa_user/vduse_dev.c
@@ -1313,6 +1313,35 @@ static struct vduse_dev *vduse_dev_get_from_minor(int minor)
return dev;
}
+int vduse_get_vq_reconnnect(struct vduse_dev *dev, u16 idx)
+{
+ struct vdpa_reconnect_info *area;
+ void *addr = (void *)get_zeroed_page(GFP_KERNEL);
+
+ area = &dev->reconnect_info[idx];
+
+ area->addr = virt_to_phys(addr);
+ area->vaddr = (unsigned long)addr;
+ area->size = PAGE_SIZE;
+ area->index = idx;
+
+ return 0;
+}
+
+int vduse_free_vq_reconnnect(struct vduse_dev *dev, u16 idx)
+{
+ struct vdpa_reconnect_info *area;
+
+ area = &dev->reconnect_info[idx];
+ if ((area->size == PAGE_SIZE) && (area->addr != NULL)) {
+ free_page(area->vaddr);
+ area->size = 0;
+ area->addr = 0;
+ area->vaddr = 0;
+ }
+
+ return 0;
+}
static vm_fault_t vduse_vm_fault(struct vm_fault *vmf)
{
@@ -1446,6 +1475,10 @@ static int vduse_destroy_dev(char *name)
mutex_unlock(&dev->lock);
return -EBUSY;
}
+ for (int i = 0; i < dev->vq_num; i++) {
+
+ vduse_free_vq_reconnnect(dev, i);
+ }
dev->connected = true;
mutex_unlock(&dev->lock);
@@ -1583,6 +1616,8 @@ static int vduse_create_dev(struct vduse_dev_config *config,
INIT_WORK(&dev->vqs[i].kick, vduse_vq_kick_work);
spin_lock_init(&dev->vqs[i].kick_lock);
spin_lock_init(&dev->vqs[i].irq_lock);
+
+ vduse_get_vq_reconnnect(dev, i);
}
ret = idr_alloc(&vduse_idr, dev, 1, VDUSE_DEV_MAX, GFP_KERNEL);
--
2.34.3
From: Your Name <[email protected]>
Add the operation for mmap, The user space APP will
use this function to map the pages to userspace
Signed-off-by: Cindy Lu <[email protected]>
---
drivers/vdpa/vdpa_user/vduse_dev.c | 49 ++++++++++++++++++++++++++++++
1 file changed, 49 insertions(+)
diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c
index f845dc46b1db..1b833bf0ae37 100644
--- a/drivers/vdpa/vdpa_user/vduse_dev.c
+++ b/drivers/vdpa/vdpa_user/vduse_dev.c
@@ -1313,6 +1313,54 @@ static struct vduse_dev *vduse_dev_get_from_minor(int minor)
return dev;
}
+
+static vm_fault_t vduse_vm_fault(struct vm_fault *vmf)
+{
+ struct vduse_dev *dev = vmf->vma->vm_file->private_data;
+ struct vm_area_struct *vma = vmf->vma;
+ u16 index = vma->vm_pgoff;
+
+ struct vdpa_reconnect_info *info;
+ info = &dev->reconnect_info[index];
+
+ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+ if (remap_pfn_range(vma, vmf->address & PAGE_MASK, PFN_DOWN(info->addr),
+ PAGE_SIZE, vma->vm_page_prot))
+ return VM_FAULT_SIGBUS;
+ return VM_FAULT_NOPAGE;
+}
+
+static const struct vm_operations_struct vduse_vm_ops = {
+ .fault = vduse_vm_fault,
+};
+
+static int vduse_mmap(struct file *file, struct vm_area_struct *vma)
+{
+ struct vduse_dev *dev = file->private_data;
+ struct vdpa_reconnect_info *info;
+ unsigned long index = vma->vm_pgoff;
+
+ if (vma->vm_end - vma->vm_start != PAGE_SIZE)
+ return -EINVAL;
+ if ((vma->vm_flags & VM_SHARED) == 0)
+ return -EINVAL;
+
+ if (index > 65535)
+ return -EINVAL;
+
+ info = &dev->reconnect_info[index];
+ if (info->addr & (PAGE_SIZE - 1))
+ return -EINVAL;
+ if (vma->vm_end - vma->vm_start != info->size) {
+ return -ENOTSUPP;
+ }
+
+ vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
+ vma->vm_ops = &vduse_vm_ops;
+
+ return 0;
+}
+
static int vduse_dev_open(struct inode *inode, struct file *file)
{
int ret;
@@ -1345,6 +1393,7 @@ static const struct file_operations vduse_dev_fops = {
.unlocked_ioctl = vduse_dev_ioctl,
.compat_ioctl = compat_ptr_ioctl,
.llseek = noop_llseek,
+ .mmap = vduse_mmap,
};
static struct vduse_dev *vduse_dev_create(void)
--
2.34.3
On Wed, Jun 28, 2023 at 2:59 PM Cindy Lu <[email protected]> wrote:
>
> From: Your Name <[email protected]>
>
> Add the operation for mmap, The user space APP will
> use this function to map the pages to userspace
Please be specific in the log. E.g why and what the main goal for this mmap.
>
> Signed-off-by: Cindy Lu <[email protected]>
> ---
> drivers/vdpa/vdpa_user/vduse_dev.c | 49 ++++++++++++++++++++++++++++++
> 1 file changed, 49 insertions(+)
>
> diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c
> index f845dc46b1db..1b833bf0ae37 100644
> --- a/drivers/vdpa/vdpa_user/vduse_dev.c
> +++ b/drivers/vdpa/vdpa_user/vduse_dev.c
> @@ -1313,6 +1313,54 @@ static struct vduse_dev *vduse_dev_get_from_minor(int minor)
> return dev;
> }
>
> +
> +static vm_fault_t vduse_vm_fault(struct vm_fault *vmf)
> +{
> + struct vduse_dev *dev = vmf->vma->vm_file->private_data;
> + struct vm_area_struct *vma = vmf->vma;
> + u16 index = vma->vm_pgoff;
> +
> + struct vdpa_reconnect_info *info;
> + info = &dev->reconnect_info[index];
> +
> + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
> + if (remap_pfn_range(vma, vmf->address & PAGE_MASK, PFN_DOWN(info->addr),
> + PAGE_SIZE, vma->vm_page_prot))
I'm not sure if this can work e.g do we want to use separate pages for
each virtqueue (I think the answer is yes).
> + return VM_FAULT_SIGBUS;
> + return VM_FAULT_NOPAGE;
> +}
> +
> +static const struct vm_operations_struct vduse_vm_ops = {
> + .fault = vduse_vm_fault,
> +};
> +
> +static int vduse_mmap(struct file *file, struct vm_area_struct *vma)
> +{
> + struct vduse_dev *dev = file->private_data;
> + struct vdpa_reconnect_info *info;
> + unsigned long index = vma->vm_pgoff;
> +
> + if (vma->vm_end - vma->vm_start != PAGE_SIZE)
> + return -EINVAL;
> + if ((vma->vm_flags & VM_SHARED) == 0)
> + return -EINVAL;
> +
> + if (index > 65535)
> + return -EINVAL;
> +
> + info = &dev->reconnect_info[index];
> + if (info->addr & (PAGE_SIZE - 1))
> + return -EINVAL;
> + if (vma->vm_end - vma->vm_start != info->size) {
> + return -ENOTSUPP;
> + }
How can userspace know the correct size (info->size) here?
> +
> + vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
Why do you need VM_IO, VM_PFNMAP and VM_DONTDUMP here?
Thanks
> + vma->vm_ops = &vduse_vm_ops;
> +
> + return 0;
> +}
> +
> static int vduse_dev_open(struct inode *inode, struct file *file)
> {
> int ret;
> @@ -1345,6 +1393,7 @@ static const struct file_operations vduse_dev_fops = {
> .unlocked_ioctl = vduse_dev_ioctl,
> .compat_ioctl = compat_ptr_ioctl,
> .llseek = noop_llseek,
> + .mmap = vduse_mmap,
> };
>
> static struct vduse_dev *vduse_dev_create(void)
> --
> 2.34.3
>
On Wed, Jun 28, 2023 at 4:08 PM Jason Wang <[email protected]> wrote:
>
> On Wed, Jun 28, 2023 at 2:59 PM Cindy Lu <[email protected]> wrote:
> >
> > From: Your Name <[email protected]>
> >
> > Add the operation for mmap, The user space APP will
> > use this function to map the pages to userspace
>
> Please be specific in the log. E.g why and what the main goal for this mmap.
>
> >
> > Signed-off-by: Cindy Lu <[email protected]>
> > ---
> > drivers/vdpa/vdpa_user/vduse_dev.c | 49 ++++++++++++++++++++++++++++++
> > 1 file changed, 49 insertions(+)
> >
> > diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c
> > index f845dc46b1db..1b833bf0ae37 100644
> > --- a/drivers/vdpa/vdpa_user/vduse_dev.c
> > +++ b/drivers/vdpa/vdpa_user/vduse_dev.c
> > @@ -1313,6 +1313,54 @@ static struct vduse_dev *vduse_dev_get_from_minor(int minor)
> > return dev;
> > }
> >
> > +
> > +static vm_fault_t vduse_vm_fault(struct vm_fault *vmf)
> > +{
> > + struct vduse_dev *dev = vmf->vma->vm_file->private_data;
> > + struct vm_area_struct *vma = vmf->vma;
> > + u16 index = vma->vm_pgoff;
> > +
> > + struct vdpa_reconnect_info *info;
> > + info = &dev->reconnect_info[index];
> > +
> > + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
> > + if (remap_pfn_range(vma, vmf->address & PAGE_MASK, PFN_DOWN(info->addr),
> > + PAGE_SIZE, vma->vm_page_prot))
>
> I'm not sure if this can work e.g do we want to use separate pages for
> each virtqueue (I think the answer is yes).
>
yes, this map the separate pages per vq, beads on my test this works
> > + return VM_FAULT_SIGBUS;
> > + return VM_FAULT_NOPAGE;
> > +}
> > +
> > +static const struct vm_operations_struct vduse_vm_ops = {
> > + .fault = vduse_vm_fault,
> > +};
> > +
> > +static int vduse_mmap(struct file *file, struct vm_area_struct *vma)
> > +{
> > + struct vduse_dev *dev = file->private_data;
> > + struct vdpa_reconnect_info *info;
> > + unsigned long index = vma->vm_pgoff;
> > +
> > + if (vma->vm_end - vma->vm_start != PAGE_SIZE)
> > + return -EINVAL;
> > + if ((vma->vm_flags & VM_SHARED) == 0)
> > + return -EINVAL;
> > +
> > + if (index > 65535)
> > + return -EINVAL;
> > +
> > + info = &dev->reconnect_info[index];
> > + if (info->addr & (PAGE_SIZE - 1))
> > + return -EINVAL;
> > + if (vma->vm_end - vma->vm_start != info->size) {
> > + return -ENOTSUPP;
> > + }
>
> How can userspace know the correct size (info->size) here?
>
I had hard code the size in userpace , I will add the new ioctl of get
the map size
Thanks
cindy
> > +
> > + vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
>
> Why do you need VM_IO, VM_PFNMAP and VM_DONTDUMP here?
>
> Thanks
>
> > + vma->vm_ops = &vduse_vm_ops;
> > +
> > + return 0;
> > +}
> > +
> > static int vduse_dev_open(struct inode *inode, struct file *file)
> > {
> > int ret;
> > @@ -1345,6 +1393,7 @@ static const struct file_operations vduse_dev_fops = {
> > .unlocked_ioctl = vduse_dev_ioctl,
> > .compat_ioctl = compat_ptr_ioctl,
> > .llseek = noop_llseek,
> > + .mmap = vduse_mmap,
> > };
> >
> > static struct vduse_dev *vduse_dev_create(void)
> > --
> > 2.34.3
> >
>
On 6/28/23 08:59, Cindy Lu wrote:
> From: Your Name <[email protected]>
>
> Add the operation for mmap, The user space APP will
> use this function to map the pages to userspace
>
> Signed-off-by: Cindy Lu <[email protected]>
> ---
> drivers/vdpa/vdpa_user/vduse_dev.c | 49 ++++++++++++++++++++++++++++++
> 1 file changed, 49 insertions(+)
>
> diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c
> index f845dc46b1db..1b833bf0ae37 100644
> --- a/drivers/vdpa/vdpa_user/vduse_dev.c
> +++ b/drivers/vdpa/vdpa_user/vduse_dev.c
> @@ -1313,6 +1313,54 @@ static struct vduse_dev *vduse_dev_get_from_minor(int minor)
> return dev;
> }
>
> +
> +static vm_fault_t vduse_vm_fault(struct vm_fault *vmf)
> +{
> + struct vduse_dev *dev = vmf->vma->vm_file->private_data;
> + struct vm_area_struct *vma = vmf->vma;
> + u16 index = vma->vm_pgoff;
> +
> + struct vdpa_reconnect_info *info;
> + info = &dev->reconnect_info[index];
> +
> + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
> + if (remap_pfn_range(vma, vmf->address & PAGE_MASK, PFN_DOWN(info->addr),
> + PAGE_SIZE, vma->vm_page_prot))
> + return VM_FAULT_SIGBUS;
> + return VM_FAULT_NOPAGE;
> +}
> +
> +static const struct vm_operations_struct vduse_vm_ops = {
> + .fault = vduse_vm_fault,
> +};
> +
> +static int vduse_mmap(struct file *file, struct vm_area_struct *vma)
> +{
> + struct vduse_dev *dev = file->private_data;
> + struct vdpa_reconnect_info *info;
> + unsigned long index = vma->vm_pgoff;
> +
> + if (vma->vm_end - vma->vm_start != PAGE_SIZE)
> + return -EINVAL;
> + if ((vma->vm_flags & VM_SHARED) == 0)
> + return -EINVAL;
> +
> + if (index > 65535)
> + return -EINVAL;
You declare an array of 64 entries in patch 1, so it can overflow.
> +
> + info = &dev->reconnect_info[index];
> + if (info->addr & (PAGE_SIZE - 1))
> + return -EINVAL;
> + if (vma->vm_end - vma->vm_start != info->size) {
> + return -ENOTSUPP;
> + }
> +
> + vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
> + vma->vm_ops = &vduse_vm_ops;
> +
> + return 0;
> +}
> +
> static int vduse_dev_open(struct inode *inode, struct file *file)
> {
> int ret;
> @@ -1345,6 +1393,7 @@ static const struct file_operations vduse_dev_fops = {
> .unlocked_ioctl = vduse_dev_ioctl,
> .compat_ioctl = compat_ptr_ioctl,
> .llseek = noop_llseek,
> + .mmap = vduse_mmap,
> };
>
> static struct vduse_dev *vduse_dev_create(void)