Here is the reconnect support in vduse,
Kernel will allocate pages for reconnection.
Userspace needs to use mmap to map the memory to userspace and use these pages to
save the reconnect information.
test passd in vduse+dpdk-testpmd
change in V2
1. Address the comments from v1
2. Add the document for reconnect process
change in V3
1. Move the vdpa_vq_state to the uAPI. vduse will use this to synchronize the vq info between the kernel and userspace app.
2. Add a new ioctl VDUSE_DEV_GET_CONFIG. userspace app use this to get config space
3. Rewrite the commit message.
4. Only save the address for the page address and remove the index.
5. remove the ioctl VDUSE_GET_RECONNECT_INFO, userspace app will use uAPI VDUSE_RECONNCT_MMAP_SIZE to mmap
6. Rewrite the document for the reconnect process to make it clearer.
change in v4
1. Change the number of map pages to VQ numbers. UserSpace APP can define and maintain the structure for saving reconnection information in userspace. The kernel will not maintain this information.
2. Rewrite the document for the reconnect process to make it clearer.
3. add the new ioctl for VDUSE_DEV_GET_CONFIG/VDUSE_DEV_GET_STATUS
Cindy Lu (5):
vduse: Add new ioctl VDUSE_DEV_GET_CONFIG
vduse: Add new ioctl VDUSE_DEV_GET_STATUS
vduse: Add function to get/free the pages for reconnection
vduse: Add file operation for mmap
Documentation: Add reconnect process for VDUSE
Documentation/userspace-api/vduse.rst | 32 +++++++
drivers/vdpa/vdpa_user/vduse_dev.c | 125 ++++++++++++++++++++++++++
include/uapi/linux/vduse.h | 5 ++
3 files changed, 162 insertions(+)
--
2.43.0
The ioctl VDUSE_DEV_GET_CONFIG is used by the Userspace App
to get the device configuration space.
Signed-off-by: Cindy Lu <[email protected]>
---
drivers/vdpa/vdpa_user/vduse_dev.c | 21 +++++++++++++++++++++
include/uapi/linux/vduse.h | 3 +++
2 files changed, 24 insertions(+)
diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c
index 1c1d71d69026..ab246da27616 100644
--- a/drivers/vdpa/vdpa_user/vduse_dev.c
+++ b/drivers/vdpa/vdpa_user/vduse_dev.c
@@ -1368,6 +1368,27 @@ static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
ret = 0;
break;
}
+ case VDUSE_DEV_GET_CONFIG: {
+ struct vduse_config_data config;
+ unsigned long size = offsetof(struct vduse_config_data, buffer);
+
+ ret = -EFAULT;
+ if (copy_from_user(&config, argp, size))
+ break;
+
+ ret = -EINVAL;
+ if (config.offset > dev->config_size || config.length == 0 ||
+ config.length > dev->config_size - config.offset)
+ break;
+
+ if (copy_to_user(argp + size, dev->config + config.offset,
+ config.length)) {
+ ret = -EFAULT;
+ break;
+ }
+ ret = 0;
+ break;
+ }
default:
ret = -ENOIOCTLCMD;
break;
diff --git a/include/uapi/linux/vduse.h b/include/uapi/linux/vduse.h
index 11bd48c72c6c..125d7529d91b 100644
--- a/include/uapi/linux/vduse.h
+++ b/include/uapi/linux/vduse.h
@@ -350,4 +350,7 @@ struct vduse_dev_response {
};
};
+/* get device configuration space */
+#define VDUSE_DEV_GET_CONFIG _IOR(VDUSE_BASE, 0x1b, struct vduse_config_data)
+
#endif /* _UAPI_VDUSE_H_ */
--
2.43.0
The ioctl VDUSE_DEV_GET_STATUS is used by the Userspace App
to get the device status
Signed-off-by: Cindy Lu <[email protected]>
---
drivers/vdpa/vdpa_user/vduse_dev.c | 7 +++++++
include/uapi/linux/vduse.h | 2 ++
2 files changed, 9 insertions(+)
diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c
index ab246da27616..ef3c9681941e 100644
--- a/drivers/vdpa/vdpa_user/vduse_dev.c
+++ b/drivers/vdpa/vdpa_user/vduse_dev.c
@@ -1389,6 +1389,13 @@ static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
ret = 0;
break;
}
+
+ case VDUSE_DEV_GET_STATUS:
+ /*
+ * Returns the status read from device
+ */
+ ret = put_user(dev->status, (u8 __user *)argp);
+ break;
default:
ret = -ENOIOCTLCMD;
break;
diff --git a/include/uapi/linux/vduse.h b/include/uapi/linux/vduse.h
index 125d7529d91b..f501173a9d69 100644
--- a/include/uapi/linux/vduse.h
+++ b/include/uapi/linux/vduse.h
@@ -353,4 +353,6 @@ struct vduse_dev_response {
/* get device configuration space */
#define VDUSE_DEV_GET_CONFIG _IOR(VDUSE_BASE, 0x1b, struct vduse_config_data)
+#define VDUSE_DEV_GET_STATUS _IOR(VDUSE_BASE, 0x1c, __u8)
+
#endif /* _UAPI_VDUSE_H_ */
--
2.43.0
Add the function vduse_alloc_reconnnect_info_mem
and vduse_alloc_reconnnect_info_mem
These functions allow vduse to allocate and free memory for reconnection
information. The amount of memory allocated is vq_num pages.
Each VQS will map its own page where the reconnection information will be saved
Signed-off-by: Cindy Lu <[email protected]>
---
drivers/vdpa/vdpa_user/vduse_dev.c | 40 ++++++++++++++++++++++++++++++
1 file changed, 40 insertions(+)
diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c
index ef3c9681941e..2da659d5f4a8 100644
--- a/drivers/vdpa/vdpa_user/vduse_dev.c
+++ b/drivers/vdpa/vdpa_user/vduse_dev.c
@@ -65,6 +65,7 @@ struct vduse_virtqueue {
int irq_effective_cpu;
struct cpumask irq_affinity;
struct kobject kobj;
+ unsigned long vdpa_reconnect_vaddr;
};
struct vduse_dev;
@@ -1105,6 +1106,38 @@ static void vduse_vq_update_effective_cpu(struct vduse_virtqueue *vq)
vq->irq_effective_cpu = curr_cpu;
}
+static int vduse_alloc_reconnnect_info_mem(struct vduse_dev *dev)
+{
+ unsigned long vaddr = 0;
+ struct vduse_virtqueue *vq;
+
+ for (int i = 0; i < dev->vq_num; i++) {
+ /*page 0~ vq_num save the reconnect info for vq*/
+ vq = dev->vqs[i];
+ vaddr = get_zeroed_page(GFP_KERNEL);
+ if (vaddr == 0)
+ return -ENOMEM;
+
+ vq->vdpa_reconnect_vaddr = vaddr;
+ }
+
+ return 0;
+}
+
+static int vduse_free_reconnnect_info_mem(struct vduse_dev *dev)
+{
+ struct vduse_virtqueue *vq;
+
+ for (int i = 0; i < dev->vq_num; i++) {
+ vq = dev->vqs[i];
+
+ if (vq->vdpa_reconnect_vaddr)
+ free_page(vq->vdpa_reconnect_vaddr);
+ vq->vdpa_reconnect_vaddr = 0;
+ }
+
+ return 0;
+}
static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
unsigned long arg)
@@ -1672,6 +1705,8 @@ static int vduse_destroy_dev(char *name)
mutex_unlock(&dev->lock);
return -EBUSY;
}
+ vduse_free_reconnnect_info_mem(dev);
+
dev->connected = true;
mutex_unlock(&dev->lock);
@@ -1855,12 +1890,17 @@ static int vduse_create_dev(struct vduse_dev_config *config,
ret = vduse_dev_init_vqs(dev, config->vq_align, config->vq_num);
if (ret)
goto err_vqs;
+ ret = vduse_alloc_reconnnect_info_mem(dev);
+ if (ret < 0)
+ goto err_mem;
__module_get(THIS_MODULE);
return 0;
err_vqs:
device_destroy(&vduse_class, MKDEV(MAJOR(vduse_major), dev->minor));
+err_mem:
+ vduse_free_reconnnect_info_mem(dev);
err_dev:
idr_remove(&vduse_idr, dev->minor);
err_idr:
--
2.43.0
Add the operation for mmap, This function will be used by the user space
application to map the pages to the user space.
Signed-off-by: Cindy Lu <[email protected]>
---
drivers/vdpa/vdpa_user/vduse_dev.c | 57 ++++++++++++++++++++++++++++++
1 file changed, 57 insertions(+)
diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c
index 2da659d5f4a8..083fe0047677 100644
--- a/drivers/vdpa/vdpa_user/vduse_dev.c
+++ b/drivers/vdpa/vdpa_user/vduse_dev.c
@@ -1465,6 +1465,61 @@ static struct vduse_dev *vduse_dev_get_from_minor(int minor)
return dev;
}
+static vm_fault_t vduse_vm_fault(struct vm_fault *vmf)
+{
+ struct vduse_dev *dev = vmf->vma->vm_file->private_data;
+ struct vm_area_struct *vma = vmf->vma;
+ u16 index = vma->vm_pgoff;
+ struct vduse_virtqueue *vq;
+ unsigned long vaddr;
+
+ /* index 1+vq_number page reserved for virtqueue state*/
+ vq = dev->vqs[index];
+ vaddr = vq->vdpa_reconnect_vaddr;
+ if (remap_pfn_range(vma, vmf->address & PAGE_MASK,
+ PFN_DOWN(virt_to_phys((void *)vaddr)), PAGE_SIZE,
+ vma->vm_page_prot))
+ return VM_FAULT_SIGBUS;
+ return VM_FAULT_NOPAGE;
+}
+
+static const struct vm_operations_struct vduse_vm_ops = {
+ .fault = vduse_vm_fault,
+};
+
+static int vduse_dev_mmap(struct file *file, struct vm_area_struct *vma)
+{
+ struct vduse_dev *dev = file->private_data;
+ unsigned long vaddr = 0;
+ unsigned long index = vma->vm_pgoff;
+ struct vduse_virtqueue *vq;
+
+ if (vma->vm_end - vma->vm_start != PAGE_SIZE)
+ return -EINVAL;
+ if ((vma->vm_flags & VM_SHARED) == 0)
+ return -EINVAL;
+
+ if (index > dev->vq_num)
+ return -EINVAL;
+
+ vq = dev->vqs[index];
+ vaddr = vq->vdpa_reconnect_vaddr;
+ if (vaddr == 0)
+ return -EOPNOTSUPP;
+
+ if (virt_to_phys((void *)vaddr) & (PAGE_SIZE - 1))
+ return -EINVAL;
+
+ /* Check if the Userspace App mapped the correct size */
+ if (vma->vm_end - vma->vm_start != PAGE_SIZE)
+ return -EOPNOTSUPP;
+
+ vm_flags_set(vma, VM_DONTEXPAND);
+ vma->vm_ops = &vduse_vm_ops;
+
+ return 0;
+}
+
static int vduse_dev_open(struct inode *inode, struct file *file)
{
int ret;
@@ -1497,6 +1552,8 @@ static const struct file_operations vduse_dev_fops = {
.unlocked_ioctl = vduse_dev_ioctl,
.compat_ioctl = compat_ptr_ioctl,
.llseek = noop_llseek,
+ .mmap = vduse_dev_mmap,
+
};
static ssize_t irq_cb_affinity_show(struct vduse_virtqueue *vq, char *buf)
--
2.43.0
Add a document explaining the reconnect process, including what the
Userspace App needs to do and how it works with the kernel.
Signed-off-by: Cindy Lu <[email protected]>
---
Documentation/userspace-api/vduse.rst | 32 +++++++++++++++++++++++++++
1 file changed, 32 insertions(+)
diff --git a/Documentation/userspace-api/vduse.rst b/Documentation/userspace-api/vduse.rst
index bdb880e01132..a2be85e0e516 100644
--- a/Documentation/userspace-api/vduse.rst
+++ b/Documentation/userspace-api/vduse.rst
@@ -231,3 +231,35 @@ able to start the dataplane processing as follows:
after the used ring is filled.
For more details on the uAPI, please see include/uapi/linux/vduse.h.
+
+HOW VDUSE devices reconnectoin works
+----------------
+0. Userspace APP checks if the device /dev/vduse/vduse_name exists.
+ If it does not exist, need to create the instance.goto step 1
+ If it does exist, it means this is a reconnect and goto step 3.
+
+1. Create a new VDUSE instance with ioctl(VDUSE_CREATE_DEV) on
+ /dev/vduse/control.
+
+2. When the ioctl(VDUSE_CREATE_DEV) function is called, the kernel allocates memory
+ to save the reconnect information.
+
+3. Userspace App need to mmap the pages to userspace
+ Userspace App need to map Pages 0 to vq_number for vq status,
+ Users can define the structure for saving the reconnect information themselves
+ in the userspace.
+
+4. Check if the infomatin sutiable for reconnect
+ If this is reconnect:
+ Before attempting to reconnect, The userspace application need to the
+ ioctl VDUSE_DEV_GET_CONFIG,VDUSE_DEV_GET_STATUS,VDUSE_DEV_GET_FEATURES...
+ to get the and confirm if these information are suitable for reconnecting.
+
+5. Start the userspace App.
+ While running, the application should store the relevant information about
+ reconnections in mapped pages.
+ When calling ioctl VDUSE_VQ_GET_INFO from the userspace APP to get vq information, it is necessary
+ to check if this is a reconnection. If a reconnection has occurred, the vq-related information
+ must be get from the mapped pages.
+
+6. When the Userspace App exits, it is necessary to unmap all the reconnect pages.
--
2.43.0
On Wed, Feb 07, 2024 at 01:43:27PM +0800, Cindy Lu wrote:
> Here is the reconnect support in vduse,
>
> Kernel will allocate pages for reconnection.
> Userspace needs to use mmap to map the memory to userspace and use these pages to
> save the reconnect information.
What is "reconnect"? Not really clear from documentation - it seems to
be assumed that reader has an idea but most don't.
Also what's with all the typos? reconnect with 3 nnn s, sutiable and so
on. Can you pls run a speller?
> test passd in vduse+dpdk-testpmd
>
> change in V2
> 1. Address the comments from v1
> 2. Add the document for reconnect process
>
> change in V3
> 1. Move the vdpa_vq_state to the uAPI. vduse will use this to synchronize the vq info between the kernel and userspace app.
> 2. Add a new ioctl VDUSE_DEV_GET_CONFIG. userspace app use this to get config space
> 3. Rewrite the commit message.
> 4. Only save the address for the page address and remove the index.
> 5. remove the ioctl VDUSE_GET_RECONNECT_INFO, userspace app will use uAPI VDUSE_RECONNCT_MMAP_SIZE to mmap
> 6. Rewrite the document for the reconnect process to make it clearer.
>
> change in v4
> 1. Change the number of map pages to VQ numbers. UserSpace APP can define and maintain the structure for saving reconnection information in userspace. The kernel will not maintain this information.
> 2. Rewrite the document for the reconnect process to make it clearer.
> 3. add the new ioctl for VDUSE_DEV_GET_CONFIG/VDUSE_DEV_GET_STATUS
>
> Cindy Lu (5):
> vduse: Add new ioctl VDUSE_DEV_GET_CONFIG
> vduse: Add new ioctl VDUSE_DEV_GET_STATUS
> vduse: Add function to get/free the pages for reconnection
> vduse: Add file operation for mmap
> Documentation: Add reconnect process for VDUSE
>
> Documentation/userspace-api/vduse.rst | 32 +++++++
> drivers/vdpa/vdpa_user/vduse_dev.c | 125 ++++++++++++++++++++++++++
> include/uapi/linux/vduse.h | 5 ++
> 3 files changed, 162 insertions(+)
>
> --
> 2.43.0
On Wed, Feb 7, 2024 at 1:47 PM Cindy Lu <[email protected]> wrote:
>
> Add a document explaining the reconnect process, including what the
> Userspace App needs to do and how it works with the kernel.
>
> Signed-off-by: Cindy Lu <[email protected]>
> ---
> Documentation/userspace-api/vduse.rst | 32 +++++++++++++++++++++++++++
> 1 file changed, 32 insertions(+)
>
> diff --git a/Documentation/userspace-api/vduse.rst b/Documentation/userspace-api/vduse.rst
> index bdb880e01132..a2be85e0e516 100644
> --- a/Documentation/userspace-api/vduse.rst
> +++ b/Documentation/userspace-api/vduse.rst
> @@ -231,3 +231,35 @@ able to start the dataplane processing as follows:
> after the used ring is filled.
>
> For more details on the uAPI, please see include/uapi/linux/vduse.h.
> +
> +HOW VDUSE devices reconnectoin works
Typos, let's use a spell checker.
> +----------------
> +0. Userspace APP checks if the device /dev/vduse/vduse_name exists.
> + If it does not exist, need to create the instance.goto step 1
> + If it does exist, it means this is a reconnect and goto step 3.
> +
> +1. Create a new VDUSE instance with ioctl(VDUSE_CREATE_DEV) on
> + /dev/vduse/control.
> +
> +2. When the ioctl(VDUSE_CREATE_DEV) function is called, the kernel allocates memory
> + to save the reconnect information.
> +
> +3. Userspace App need to mmap the pages to userspace
Need to describe what kind of pages need to be mapped. And what's more:
Does this require cooperation from the disconnected application? If
yes, how to distinguish from the one that is not cooperative (doesn't
support reconnection)?
> + Userspace App need to map Pages 0 to vq_number for vq status,
> + Users can define the structure for saving the reconnect information themselves
> + in the userspace.
> +
> +4. Check if the infomatin sutiable for reconnect
Typos again.
> + If this is reconnect:
> + Before attempting to reconnect, The userspace application need to the
> + ioctl VDUSE_DEV_GET_CONFIG,VDUSE_DEV_GET_STATUS,VDUSE_DEV_GET_FEATURES...
"..." is not good, let's describe the steps in detail.
> + to get the and confirm if these information are suitable for reconnecting.
> +
> +5. Start the userspace App.
I had a hard time understanding this, does it mean the application
needs to restart itself?
> + While running, the application should store the relevant information about
> + reconnections in mapped pages.
> + When calling ioctl VDUSE_VQ_GET_INFO from the userspace APP
Abbrev is not good here. Let's use "application" for consistency.
> to get vq information, it is necessary
> + to check if this is a reconnection.
How to check whether or not it is reconnected?
> If a reconnection has occurred, the vq-related information
> + must be get from the mapped pages.
> +
> +6. When the Userspace App exits, it is necessary to unmap all the reconnect pages.
> --
> 2.43.0
>
Thanks
On Mon, Mar 4, 2024 at 1:11 PM Jason Wang <[email protected]> wrote:
>
> On Wed, Feb 7, 2024 at 1:47 PM Cindy Lu <[email protected]> wrote:
> >
> > Add a document explaining the reconnect process, including what the
> > Userspace App needs to do and how it works with the kernel.
> >
> > Signed-off-by: Cindy Lu <[email protected]>
> > ---
> > Documentation/userspace-api/vduse.rst | 32 +++++++++++++++++++++++++++
> > 1 file changed, 32 insertions(+)
> >
> > diff --git a/Documentation/userspace-api/vduse.rst b/Documentation/userspace-api/vduse.rst
> > index bdb880e01132..a2be85e0e516 100644
> > --- a/Documentation/userspace-api/vduse.rst
> > +++ b/Documentation/userspace-api/vduse.rst
> > @@ -231,3 +231,35 @@ able to start the dataplane processing as follows:
> > after the used ring is filled.
> >
> > For more details on the uAPI, please see include/uapi/linux/vduse.h.
> > +
> > +HOW VDUSE devices reconnectoin works
>
> Typos, let's use a spell checker.
>
> > +----------------
> > +0. Userspace APP checks if the device /dev/vduse/vduse_name exists.
> > + If it does not exist, need to create the instance.goto step 1
> > + If it does exist, it means this is a reconnect and goto step 3.
> > +
> > +1. Create a new VDUSE instance with ioctl(VDUSE_CREATE_DEV) on
> > + /dev/vduse/control.
> > +
> > +2. When the ioctl(VDUSE_CREATE_DEV) function is called, the kernel allocates memory
> > + to save the reconnect information.
> > +
> > +3. Userspace App need to mmap the pages to userspace
>
> Need to describe what kind of pages need to be mapped. And what's more:
>
> Does this require cooperation from the disconnected application? If
> yes, how to distinguish from the one that is not cooperative (doesn't
> support reconnection)?
>
> > + Userspace App need to map Pages 0 to vq_number for vq status,
> > + Users can define the structure for saving the reconnect information themselves
> > + in the userspace.
> > +
> > +4. Check if the infomatin sutiable for reconnect
>
> Typos again.
>
> > + If this is reconnect:
> > + Before attempting to reconnect, The userspace application need to the
> > + ioctl VDUSE_DEV_GET_CONFIG,VDUSE_DEV_GET_STATUS,VDUSE_DEV_GET_FEATURES...
>
> "..." is not good, let's describe the steps in detail.
>
> > + to get the and confirm if these information are suitable for reconnecting.
> > +
> > +5. Start the userspace App.
>
> I had a hard time understanding this, does it mean the application
> needs to restart itself?
>
> > + While running, the application should store the relevant information about
> > + reconnections in mapped pages.
> > + When calling ioctl VDUSE_VQ_GET_INFO from the userspace APP
>
> Abbrev is not good here. Let's use "application" for consistency.
>
> > to get vq information, it is necessary
> > + to check if this is a reconnection.
>
> How to check whether or not it is reconnected?
>
> > If a reconnection has occurred, the vq-related information
> > + must be get from the mapped pages.
> > +
> > +6. When the Userspace App exits, it is necessary to unmap all the reconnect pages.
> > --
> > 2.43.0
> >
>
> Thanks
Thanks Jason, I will rewrite this part and send the new version soon
Thanks
cindy
>
On Mon, Mar 04, 2024 at 01:11:02PM +0800, Jason Wang wrote:
> On Wed, Feb 7, 2024 at 1:47 PM Cindy Lu <[email protected]> wrote:
> >
> > Add a document explaining the reconnect process, including what the
> > Userspace App needs to do and how it works with the kernel.
> >
> > Signed-off-by: Cindy Lu <[email protected]>
> > ---
> > Documentation/userspace-api/vduse.rst | 32 +++++++++++++++++++++++++++
> > 1 file changed, 32 insertions(+)
> >
> > diff --git a/Documentation/userspace-api/vduse.rst b/Documentation/userspace-api/vduse.rst
> > index bdb880e01132..a2be85e0e516 100644
> > --- a/Documentation/userspace-api/vduse.rst
> > +++ b/Documentation/userspace-api/vduse.rst
> > @@ -231,3 +231,35 @@ able to start the dataplane processing as follows:
> > after the used ring is filled.
> >
> > For more details on the uAPI, please see include/uapi/linux/vduse.h.
> > +
> > +HOW VDUSE devices reconnectoin works
>
> Typos, let's use a spell checker.
>
> > +----------------
> > +0. Userspace APP checks if the device /dev/vduse/vduse_name exists.
> > + If it does not exist, need to create the instance.goto step 1
> > + If it does exist, it means this is a reconnect and goto step 3.
> > +
> > +1. Create a new VDUSE instance with ioctl(VDUSE_CREATE_DEV) on
> > + /dev/vduse/control.
> > +
> > +2. When the ioctl(VDUSE_CREATE_DEV) function is called, the kernel allocates memory
> > + to save the reconnect information.
> > +
> > +3. Userspace App need to mmap the pages to userspace
>
> Need to describe what kind of pages need to be mapped. And what's more:
>
> Does this require cooperation from the disconnected application? If
> yes, how to distinguish from the one that is not cooperative (doesn't
> support reconnection)?
>
> > + Userspace App need to map Pages 0 to vq_number for vq status,
> > + Users can define the structure for saving the reconnect information themselves
> > + in the userspace.
> > +
> > +4. Check if the infomatin sutiable for reconnect
>
> Typos again.
>
> > + If this is reconnect:
> > + Before attempting to reconnect, The userspace application need to the
> > + ioctl VDUSE_DEV_GET_CONFIG,VDUSE_DEV_GET_STATUS,VDUSE_DEV_GET_FEATURES...
>
> "..." is not good, let's describe the steps in detail.
>
> > + to get the and confirm if these information are suitable for reconnecting.
> > +
> > +5. Start the userspace App.
>
> I had a hard time understanding this, does it mean the application
> needs to restart itself?
>
> > + While running, the application should store the relevant information about
> > + reconnections in mapped pages.
> > + When calling ioctl VDUSE_VQ_GET_INFO from the userspace APP
>
> Abbrev is not good here. Let's use "application" for consistency.
>
> > to get vq information, it is necessary
> > + to check if this is a reconnection.
>
> How to check whether or not it is reconnected?
>
> > If a reconnection has occurred, the vq-related information
> > + must be get from the mapped pages.
> > +
> > +6. When the Userspace App exits, it is necessary to unmap all the reconnect pages.
> > --
> > 2.43.0
> >
>
> Thanks
Are comments only on patch 5? Ack the rest?
--
MST
On Wed, Feb 7, 2024 at 1:47 PM Cindy Lu <[email protected]> wrote:
>
> Here is the reconnect support in vduse,
>
> Kernel will allocate pages for reconnection.
> Userspace needs to use mmap to map the memory to userspace and use these pages to
> save the reconnect information.
>
> test passd in vduse+dpdk-testpmd
>
> change in V2
> 1. Address the comments from v1
> 2. Add the document for reconnect process
>
> change in V3
> 1. Move the vdpa_vq_state to the uAPI. vduse will use this to synchronize the vq info between the kernel and userspace app.
> 2. Add a new ioctl VDUSE_DEV_GET_CONFIG. userspace app use this to get config space
> 3. Rewrite the commit message.
> 4. Only save the address for the page address and remove the index.
> 5. remove the ioctl VDUSE_GET_RECONNECT_INFO, userspace app will use uAPI VDUSE_RECONNCT_MMAP_SIZE to mmap
> 6. Rewrite the document for the reconnect process to make it clearer.
>
> change in v4
> 1. Change the number of map pages to VQ numbers. UserSpace APP can define and maintain the structure for saving reconnection information in userspace. The kernel will not maintain this information.
So this means the structure (e.g inflight descriptors) are application
specific, we can't do cross application reconnection?
Thanks
> 2. Rewrite the document for the reconnect process to make it clearer.
> 3. add the new ioctl for VDUSE_DEV_GET_CONFIG/VDUSE_DEV_GET_STATUS
>
> Cindy Lu (5):
> vduse: Add new ioctl VDUSE_DEV_GET_CONFIG
> vduse: Add new ioctl VDUSE_DEV_GET_STATUS
> vduse: Add function to get/free the pages for reconnection
> vduse: Add file operation for mmap
> Documentation: Add reconnect process for VDUSE
>
> Documentation/userspace-api/vduse.rst | 32 +++++++
> drivers/vdpa/vdpa_user/vduse_dev.c | 125 ++++++++++++++++++++++++++
> include/uapi/linux/vduse.h | 5 ++
> 3 files changed, 162 insertions(+)
>
> --
> 2.43.0
>
On Wed, Feb 7, 2024 at 1:47 PM Cindy Lu <[email protected]> wrote:
>
> Add the operation for mmap, This function will be used by the user space
> application to map the pages to the user space.
>
> Signed-off-by: Cindy Lu <[email protected]>
> ---
> drivers/vdpa/vdpa_user/vduse_dev.c | 57 ++++++++++++++++++++++++++++++
> 1 file changed, 57 insertions(+)
>
> diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c
> index 2da659d5f4a8..083fe0047677 100644
> --- a/drivers/vdpa/vdpa_user/vduse_dev.c
> +++ b/drivers/vdpa/vdpa_user/vduse_dev.c
> @@ -1465,6 +1465,61 @@ static struct vduse_dev *vduse_dev_get_from_minor(int minor)
> return dev;
> }
>
> +static vm_fault_t vduse_vm_fault(struct vm_fault *vmf)
> +{
> + struct vduse_dev *dev = vmf->vma->vm_file->private_data;
> + struct vm_area_struct *vma = vmf->vma;
> + u16 index = vma->vm_pgoff;
> + struct vduse_virtqueue *vq;
> + unsigned long vaddr;
> +
> + /* index 1+vq_number page reserved for virtqueue state*/
This comment seem wrong?
THanks
> + vq = dev->vqs[index];
> + vaddr = vq->vdpa_reconnect_vaddr;
> + if (remap_pfn_range(vma, vmf->address & PAGE_MASK,
> + PFN_DOWN(virt_to_phys((void *)vaddr)), PAGE_SIZE,
> + vma->vm_page_prot))
> + return VM_FAULT_SIGBUS;
> + return VM_FAULT_NOPAGE;
> +}
> +
> +static const struct vm_operations_struct vduse_vm_ops = {
> + .fault = vduse_vm_fault,
> +};
> +
> +static int vduse_dev_mmap(struct file *file, struct vm_area_struct *vma)
> +{
> + struct vduse_dev *dev = file->private_data;
> + unsigned long vaddr = 0;
> + unsigned long index = vma->vm_pgoff;
> + struct vduse_virtqueue *vq;
> +
> + if (vma->vm_end - vma->vm_start != PAGE_SIZE)
> + return -EINVAL;
> + if ((vma->vm_flags & VM_SHARED) == 0)
> + return -EINVAL;
> +
> + if (index > dev->vq_num)
> + return -EINVAL;
> +
> + vq = dev->vqs[index];
> + vaddr = vq->vdpa_reconnect_vaddr;
> + if (vaddr == 0)
> + return -EOPNOTSUPP;
> +
> + if (virt_to_phys((void *)vaddr) & (PAGE_SIZE - 1))
> + return -EINVAL;
> +
> + /* Check if the Userspace App mapped the correct size */
> + if (vma->vm_end - vma->vm_start != PAGE_SIZE)
> + return -EOPNOTSUPP;
> +
> + vm_flags_set(vma, VM_DONTEXPAND);
> + vma->vm_ops = &vduse_vm_ops;
> +
> + return 0;
> +}
> +
> static int vduse_dev_open(struct inode *inode, struct file *file)
> {
> int ret;
> @@ -1497,6 +1552,8 @@ static const struct file_operations vduse_dev_fops = {
> .unlocked_ioctl = vduse_dev_ioctl,
> .compat_ioctl = compat_ptr_ioctl,
> .llseek = noop_llseek,
> + .mmap = vduse_dev_mmap,
> +
> };
>
> static ssize_t irq_cb_affinity_show(struct vduse_virtqueue *vq, char *buf)
> --
> 2.43.0
>
On Fri, Mar 8, 2024 at 2:08 PM Jason Wang <[email protected]> wrote:
>
> On Wed, Feb 7, 2024 at 1:47 PM Cindy Lu <[email protected]> wrote:
> >
> > Here is the reconnect support in vduse,
> >
> > Kernel will allocate pages for reconnection.
> > Userspace needs to use mmap to map the memory to userspace and use these pages to
> > save the reconnect information.
> >
> > test passd in vduse+dpdk-testpmd
> >
> > change in V2
> > 1. Address the comments from v1
> > 2. Add the document for reconnect process
> >
> > change in V3
> > 1. Move the vdpa_vq_state to the uAPI. vduse will use this to synchronize the vq info between the kernel and userspace app.
> > 2. Add a new ioctl VDUSE_DEV_GET_CONFIG. userspace app use this to get config space
> > 3. Rewrite the commit message.
> > 4. Only save the address for the page address and remove the index.
> > 5. remove the ioctl VDUSE_GET_RECONNECT_INFO, userspace app will use uAPI VDUSE_RECONNCT_MMAP_SIZE to mmap
> > 6. Rewrite the document for the reconnect process to make it clearer.
> >
> > change in v4
> > 1. Change the number of map pages to VQ numbers. UserSpace APP can define and maintain the structure for saving reconnection information in userspace. The kernel will not maintain this information.
>
> So this means the structure (e.g inflight descriptors) are application
> specific, we can't do cross application reconnection?
>
> Thanks
>
yes, this is also defined by the application itself. so maybe we can't
support the
cross-application reconnection
thanks
cindy
> > 2. Rewrite the document for the reconnect process to make it clearer.
> > 3. add the new ioctl for VDUSE_DEV_GET_CONFIG/VDUSE_DEV_GET_STATUS
> >
> > Cindy Lu (5):
> > vduse: Add new ioctl VDUSE_DEV_GET_CONFIG
> > vduse: Add new ioctl VDUSE_DEV_GET_STATUS
> > vduse: Add function to get/free the pages for reconnection
> > vduse: Add file operation for mmap
> > Documentation: Add reconnect process for VDUSE
> >
> > Documentation/userspace-api/vduse.rst | 32 +++++++
> > drivers/vdpa/vdpa_user/vduse_dev.c | 125 ++++++++++++++++++++++++++
> > include/uapi/linux/vduse.h | 5 ++
> > 3 files changed, 162 insertions(+)
> >
> > --
> > 2.43.0
> >
>
On Fri, Mar 8, 2024 at 2:08 PM Jason Wang <[email protected]> wrote:
>
> On Wed, Feb 7, 2024 at 1:47 PM Cindy Lu <[email protected]> wrote:
> >
> > Add the operation for mmap, This function will be used by the user space
> > application to map the pages to the user space.
> >
> > Signed-off-by: Cindy Lu <[email protected]>
> > ---
> > drivers/vdpa/vdpa_user/vduse_dev.c | 57 ++++++++++++++++++++++++++++++
> > 1 file changed, 57 insertions(+)
> >
> > diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c
> > index 2da659d5f4a8..083fe0047677 100644
> > --- a/drivers/vdpa/vdpa_user/vduse_dev.c
> > +++ b/drivers/vdpa/vdpa_user/vduse_dev.c
> > @@ -1465,6 +1465,61 @@ static struct vduse_dev *vduse_dev_get_from_minor(int minor)
> > return dev;
> > }
> >
> > +static vm_fault_t vduse_vm_fault(struct vm_fault *vmf)
> > +{
> > + struct vduse_dev *dev = vmf->vma->vm_file->private_data;
> > + struct vm_area_struct *vma = vmf->vma;
> > + u16 index = vma->vm_pgoff;
> > + struct vduse_virtqueue *vq;
> > + unsigned long vaddr;
> > +
> > + /* index 1+vq_number page reserved for virtqueue state*/
>
> This comment seem wrong?
>
> THanks
>
sure,will fix this
thanks
cindy
>
> > + vq = dev->vqs[index];
> > + vaddr = vq->vdpa_reconnect_vaddr;
> > + if (remap_pfn_range(vma, vmf->address & PAGE_MASK,
> > + PFN_DOWN(virt_to_phys((void *)vaddr)), PAGE_SIZE,
> > + vma->vm_page_prot))
> > + return VM_FAULT_SIGBUS;
> > + return VM_FAULT_NOPAGE;
> > +}
> > +
> > +static const struct vm_operations_struct vduse_vm_ops = {
> > + .fault = vduse_vm_fault,
> > +};
> > +
> > +static int vduse_dev_mmap(struct file *file, struct vm_area_struct *vma)
> > +{
> > + struct vduse_dev *dev = file->private_data;
> > + unsigned long vaddr = 0;
> > + unsigned long index = vma->vm_pgoff;
> > + struct vduse_virtqueue *vq;
> > +
> > + if (vma->vm_end - vma->vm_start != PAGE_SIZE)
> > + return -EINVAL;
> > + if ((vma->vm_flags & VM_SHARED) == 0)
> > + return -EINVAL;
> > +
> > + if (index > dev->vq_num)
> > + return -EINVAL;
> > +
> > + vq = dev->vqs[index];
> > + vaddr = vq->vdpa_reconnect_vaddr;
> > + if (vaddr == 0)
> > + return -EOPNOTSUPP;
> > +
> > + if (virt_to_phys((void *)vaddr) & (PAGE_SIZE - 1))
> > + return -EINVAL;
> > +
> > + /* Check if the Userspace App mapped the correct size */
> > + if (vma->vm_end - vma->vm_start != PAGE_SIZE)
> > + return -EOPNOTSUPP;
> > +
> > + vm_flags_set(vma, VM_DONTEXPAND);
> > + vma->vm_ops = &vduse_vm_ops;
> > +
> > + return 0;
> > +}
> > +
> > static int vduse_dev_open(struct inode *inode, struct file *file)
> > {
> > int ret;
> > @@ -1497,6 +1552,8 @@ static const struct file_operations vduse_dev_fops = {
> > .unlocked_ioctl = vduse_dev_ioctl,
> > .compat_ioctl = compat_ptr_ioctl,
> > .llseek = noop_llseek,
> > + .mmap = vduse_dev_mmap,
> > +
> > };
> >
> > static ssize_t irq_cb_affinity_show(struct vduse_virtqueue *vq, char *buf)
> > --
> > 2.43.0
> >
>
On Fri, Feb 23, 2024 at 3:18 AM Michael S. Tsirkin <[email protected]> wrote:
>
> On Wed, Feb 07, 2024 at 01:43:27PM +0800, Cindy Lu wrote:
> > Here is the reconnect support in vduse,
> >
> > Kernel will allocate pages for reconnection.
> > Userspace needs to use mmap to map the memory to userspace and use these pages to
> > save the reconnect information.
>
> What is "reconnect"? Not really clear from documentation - it seems to
> be assumed that reader has an idea but most don't.
>
> Also what's with all the typos? reconnect with 3 nnn s, sutiable and so
> on. Can you pls run a speller?
>
Thanks a lot, Micheal. I will fix these and also update the speller
thanks
Cindy
> > test passd in vduse+dpdk-testpmd
> >
> > change in V2
> > 1. Address the comments from v1
> > 2. Add the document for reconnect process
> >
> > change in V3
> > 1. Move the vdpa_vq_state to the uAPI. vduse will use this to synchronize the vq info between the kernel and userspace app.
> > 2. Add a new ioctl VDUSE_DEV_GET_CONFIG. userspace app use this to get config space
> > 3. Rewrite the commit message.
> > 4. Only save the address for the page address and remove the index.
> > 5. remove the ioctl VDUSE_GET_RECONNECT_INFO, userspace app will use uAPI VDUSE_RECONNCT_MMAP_SIZE to mmap
> > 6. Rewrite the document for the reconnect process to make it clearer.
> >
> > change in v4
> > 1. Change the number of map pages to VQ numbers. UserSpace APP can define and maintain the structure for saving reconnection information in userspace. The kernel will not maintain this information.
> > 2. Rewrite the document for the reconnect process to make it clearer.
> > 3. add the new ioctl for VDUSE_DEV_GET_CONFIG/VDUSE_DEV_GET_STATUS
> >
> > Cindy Lu (5):
> > vduse: Add new ioctl VDUSE_DEV_GET_CONFIG
> > vduse: Add new ioctl VDUSE_DEV_GET_STATUS
> > vduse: Add function to get/free the pages for reconnection
> > vduse: Add file operation for mmap
> > Documentation: Add reconnect process for VDUSE
> >
> > Documentation/userspace-api/vduse.rst | 32 +++++++
> > drivers/vdpa/vdpa_user/vduse_dev.c | 125 ++++++++++++++++++++++++++
> > include/uapi/linux/vduse.h | 5 ++
> > 3 files changed, 162 insertions(+)
> >
> > --
> > 2.43.0
>
On Fri, Mar 8, 2024 at 3:27 PM Cindy Lu <[email protected]> wrote:
>
> On Fri, Mar 8, 2024 at 2:08 PM Jason Wang <[email protected]> wrote:
> >
> > On Wed, Feb 7, 2024 at 1:47 PM Cindy Lu <[email protected]> wrote:
> > >
> > > Here is the reconnect support in vduse,
> > >
> > > Kernel will allocate pages for reconnection.
> > > Userspace needs to use mmap to map the memory to userspace and use these pages to
> > > save the reconnect information.
> > >
> > > test passd in vduse+dpdk-testpmd
> > >
> > > change in V2
> > > 1. Address the comments from v1
> > > 2. Add the document for reconnect process
> > >
> > > change in V3
> > > 1. Move the vdpa_vq_state to the uAPI. vduse will use this to synchronize the vq info between the kernel and userspace app.
> > > 2. Add a new ioctl VDUSE_DEV_GET_CONFIG. userspace app use this to get config space
> > > 3. Rewrite the commit message.
> > > 4. Only save the address for the page address and remove the index.
> > > 5. remove the ioctl VDUSE_GET_RECONNECT_INFO, userspace app will use uAPI VDUSE_RECONNCT_MMAP_SIZE to mmap
> > > 6. Rewrite the document for the reconnect process to make it clearer.
> > >
> > > change in v4
> > > 1. Change the number of map pages to VQ numbers. UserSpace APP can define and maintain the structure for saving reconnection information in userspace. The kernel will not maintain this information.
> >
> > So this means the structure (e.g inflight descriptors) are application
> > specific, we can't do cross application reconnection?
> >
> > Thanks
> >
> yes, this is also defined by the application itself. so maybe we can't
> support the
> cross-application reconnection
> thanks
> cindy
Well, ok. We can probably start from this but let's document this
explicitly in the doc patch.
Thanks
> > > 2. Rewrite the document for the reconnect process to make it clearer.
> > > 3. add the new ioctl for VDUSE_DEV_GET_CONFIG/VDUSE_DEV_GET_STATUS
> > >
> > > Cindy Lu (5):
> > > vduse: Add new ioctl VDUSE_DEV_GET_CONFIG
> > > vduse: Add new ioctl VDUSE_DEV_GET_STATUS
> > > vduse: Add function to get/free the pages for reconnection
> > > vduse: Add file operation for mmap
> > > Documentation: Add reconnect process for VDUSE
> > >
> > > Documentation/userspace-api/vduse.rst | 32 +++++++
> > > drivers/vdpa/vdpa_user/vduse_dev.c | 125 ++++++++++++++++++++++++++
> > > include/uapi/linux/vduse.h | 5 ++
> > > 3 files changed, 162 insertions(+)
> > >
> > > --
> > > 2.43.0
> > >
> >
>
On Fri, Mar 08, 2024 at 03:29:50PM +0800, Cindy Lu wrote:
> On Fri, Feb 23, 2024 at 3:18 AM Michael S. Tsirkin <[email protected]> wrote:
> >
> > On Wed, Feb 07, 2024 at 01:43:27PM +0800, Cindy Lu wrote:
> > > Here is the reconnect support in vduse,
> > >
> > > Kernel will allocate pages for reconnection.
> > > Userspace needs to use mmap to map the memory to userspace and use these pages to
> > > save the reconnect information.
> >
> > What is "reconnect"? Not really clear from documentation - it seems to
> > be assumed that reader has an idea but most don't.
> >
> > Also what's with all the typos? reconnect with 3 nnn s, sutiable and so
> > on. Can you pls run a speller?
> >
> Thanks a lot, Micheal. I will fix these and also update the speller
> thanks
> Cindy
Didn't get an updated version, dropped the patch from the pull for this
merge window.
> > > test passd in vduse+dpdk-testpmd
> > >
> > > change in V2
> > > 1. Address the comments from v1
> > > 2. Add the document for reconnect process
> > >
> > > change in V3
> > > 1. Move the vdpa_vq_state to the uAPI. vduse will use this to synchronize the vq info between the kernel and userspace app.
> > > 2. Add a new ioctl VDUSE_DEV_GET_CONFIG. userspace app use this to get config space
> > > 3. Rewrite the commit message.
> > > 4. Only save the address for the page address and remove the index.
> > > 5. remove the ioctl VDUSE_GET_RECONNECT_INFO, userspace app will use uAPI VDUSE_RECONNCT_MMAP_SIZE to mmap
> > > 6. Rewrite the document for the reconnect process to make it clearer.
> > >
> > > change in v4
> > > 1. Change the number of map pages to VQ numbers. UserSpace APP can define and maintain the structure for saving reconnection information in userspace. The kernel will not maintain this information.
> > > 2. Rewrite the document for the reconnect process to make it clearer.
> > > 3. add the new ioctl for VDUSE_DEV_GET_CONFIG/VDUSE_DEV_GET_STATUS
> > >
> > > Cindy Lu (5):
> > > vduse: Add new ioctl VDUSE_DEV_GET_CONFIG
> > > vduse: Add new ioctl VDUSE_DEV_GET_STATUS
> > > vduse: Add function to get/free the pages for reconnection
> > > vduse: Add file operation for mmap
> > > Documentation: Add reconnect process for VDUSE
> > >
> > > Documentation/userspace-api/vduse.rst | 32 +++++++
> > > drivers/vdpa/vdpa_user/vduse_dev.c | 125 ++++++++++++++++++++++++++
> > > include/uapi/linux/vduse.h | 5 ++
> > > 3 files changed, 162 insertions(+)
> > >
> > > --
> > > 2.43.0
> >
On Tue, Mar 19, 2024 at 2:37 PM Michael S. Tsirkin <[email protected]> wrote:
>
> On Fri, Mar 08, 2024 at 03:29:50PM +0800, Cindy Lu wrote:
> > On Fri, Feb 23, 2024 at 3:18 AM Michael S. Tsirkin <[email protected]> wrote:
> > >
> > > On Wed, Feb 07, 2024 at 01:43:27PM +0800, Cindy Lu wrote:
> > > > Here is the reconnect support in vduse,
> > > >
> > > > Kernel will allocate pages for reconnection.
> > > > Userspace needs to use mmap to map the memory to userspace and use these pages to
> > > > save the reconnect information.
> > >
> > > What is "reconnect"? Not really clear from documentation - it seems to
> > > be assumed that reader has an idea but most don't.
> > >
> > > Also what's with all the typos? reconnect with 3 nnn s, sutiable and so
> > > on. Can you pls run a speller?
> > >
> > Thanks a lot, Micheal. I will fix these and also update the speller
> > thanks
> > Cindy
>
> Didn't get an updated version, dropped the patch from the pull for this
> merge window.
>
Hi Micheal
Really apologize for the delay, I was working in an emergency bug, I
will provide the updated version soon
apologize again for this mistake
Thanks
Cindy
> > > > test passd in vduse+dpdk-testpmd
> > > >
> > > > change in V2
> > > > 1. Address the comments from v1
> > > > 2. Add the document for reconnect process
> > > >
> > > > change in V3
> > > > 1. Move the vdpa_vq_state to the uAPI. vduse will use this to synchronize the vq info between the kernel and userspace app.
> > > > 2. Add a new ioctl VDUSE_DEV_GET_CONFIG. userspace app use this to get config space
> > > > 3. Rewrite the commit message.
> > > > 4. Only save the address for the page address and remove the index.
> > > > 5. remove the ioctl VDUSE_GET_RECONNECT_INFO, userspace app will use uAPI VDUSE_RECONNCT_MMAP_SIZE to mmap
> > > > 6. Rewrite the document for the reconnect process to make it clearer.
> > > >
> > > > change in v4
> > > > 1. Change the number of map pages to VQ numbers. UserSpace APP can define and maintain the structure for saving reconnection information in userspace. The kernel will not maintain this information.
> > > > 2. Rewrite the document for the reconnect process to make it clearer.
> > > > 3. add the new ioctl for VDUSE_DEV_GET_CONFIG/VDUSE_DEV_GET_STATUS
> > > >
> > > > Cindy Lu (5):
> > > > vduse: Add new ioctl VDUSE_DEV_GET_CONFIG
> > > > vduse: Add new ioctl VDUSE_DEV_GET_STATUS
> > > > vduse: Add function to get/free the pages for reconnection
> > > > vduse: Add file operation for mmap
> > > > Documentation: Add reconnect process for VDUSE
> > > >
> > > > Documentation/userspace-api/vduse.rst | 32 +++++++
> > > > drivers/vdpa/vdpa_user/vduse_dev.c | 125 ++++++++++++++++++++++++++
> > > > include/uapi/linux/vduse.h | 5 ++
> > > > 3 files changed, 162 insertions(+)
> > > >
> > > > --
> > > > 2.43.0
> > >
>