Hi All
This code provides the iommufd support for vdpa device
This code fixes the bugs from the last version and also add the asid support. rebase on kernel
v6,6-rc3
Test passed in the physical device (vp_vdpa), but there are still some problems in the emulated device (vdpa_sim_net),
I will continue working on it
The kernel code is
https://gitlab.com/lulu6/vhost/-/tree/iommufdRFC_v1
Signed-off-by: Cindy Lu <[email protected]>
Cindy Lu (8):
vhost/iommufd: Add the functions support iommufd
Kconfig: Add the new file vhost/iommufd
vhost: Add 3 new uapi to support iommufd
vdpa: Add new vdpa_config_ops to support iommufd
vdpa_sim :Add support for iommufd
vdpa: change the map/unmap process to support iommufd
vp_vdpa::Add support for iommufd
iommu: expose the function iommu_device_use_default_domain
drivers/iommu/iommu.c | 2 +
drivers/vdpa/vdpa_sim/vdpa_sim.c | 8 ++
drivers/vdpa/virtio_pci/vp_vdpa.c | 4 +
drivers/vhost/Kconfig | 1 +
drivers/vhost/Makefile | 1 +
drivers/vhost/iommufd.c | 178 +++++++++++++++++++++++++
drivers/vhost/vdpa.c | 210 +++++++++++++++++++++++++++++-
drivers/vhost/vhost.h | 21 +++
include/linux/vdpa.h | 38 +++++-
include/uapi/linux/vhost.h | 66 ++++++++++
10 files changed, 525 insertions(+), 4 deletions(-)
create mode 100644 drivers/vhost/iommufd.c
--
2.34.3
Add new vdpa_config_ops function to support iommufd
Signed-off-by: Cindy Lu <[email protected]>
---
drivers/vdpa/vdpa_sim/vdpa_sim.c | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
index 76d41058add9..9400ec32ec41 100644
--- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
+++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
@@ -762,6 +762,10 @@ static const struct vdpa_config_ops vdpasim_config_ops = {
.bind_mm = vdpasim_bind_mm,
.unbind_mm = vdpasim_unbind_mm,
.free = vdpasim_free,
+ .bind_iommufd = vdpa_iommufd_emulated_bind,
+ .unbind_iommufd = vdpa_iommufd_emulated_unbind,
+ .attach_ioas = vdpa_iommufd_emulated_attach_ioas,
+ .detach_ioas = vdpa_iommufd_emulated_detach_ioas,
};
static const struct vdpa_config_ops vdpasim_batch_config_ops = {
@@ -799,6 +803,10 @@ static const struct vdpa_config_ops vdpasim_batch_config_ops = {
.bind_mm = vdpasim_bind_mm,
.unbind_mm = vdpasim_unbind_mm,
.free = vdpasim_free,
+ .bind_iommufd = vdpa_iommufd_emulated_bind,
+ .unbind_iommufd = vdpa_iommufd_emulated_unbind,
+ .attach_ioas = vdpa_iommufd_emulated_attach_ioas,
+ .detach_ioas = vdpa_iommufd_emulated_detach_ioas,
};
MODULE_VERSION(DRV_VERSION);
--
2.34.3
Add 4 new vdpa_config_ops function to support iommufd
Signed-off-by: Cindy Lu <[email protected]>
---
include/linux/vdpa.h | 38 +++++++++++++++++++++++++++++++++++++-
1 file changed, 37 insertions(+), 1 deletion(-)
diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h
index 0e652026b776..233d80f9d910 100644
--- a/include/linux/vdpa.h
+++ b/include/linux/vdpa.h
@@ -5,6 +5,7 @@
#include <linux/kernel.h>
#include <linux/device.h>
#include <linux/interrupt.h>
+#include <linux/iommufd.h>
#include <linux/vhost_iotlb.h>
#include <linux/virtio_net.h>
#include <linux/if_ether.h>
@@ -97,6 +98,12 @@ struct vdpa_device {
struct vdpa_mgmt_dev *mdev;
unsigned int ngroups;
unsigned int nas;
+ struct iommufd_access *iommufd_access;
+ struct iommufd_device *iommufd_device;
+ struct iommufd_ctx *iommufd_ictx;
+ unsigned long *vq_bitmap;
+ atomic_t iommufd_users;
+ bool iommufd_attached;
};
/**
@@ -332,6 +339,17 @@ struct vdpa_map_file {
* @vdev: vdpa device
* @free: Free resources that belongs to vDPA (optional)
* @vdev: vdpa device
+ * @bind_iommufd: use vdpa_iommufd_physical_bind for an IOMMU
+ * backed device.
+ * otherwise use vdpa_iommufd_emulated_bind
+ * @unbind_iommufd: use vdpa_iommufd_physical_unbind for an IOMMU
+ * backed device.
+ * otherwise, use vdpa_iommufd_emulated_unbind
+ * @attach_ioas: use vdpa_iommufd_physical_attach_ioas for an
+ * IOMMU backed device.
+ * @detach_ioas: Opposite of attach_ioas
+ * @free: Free resources that belongs to vDPA (optional)
+ * @vdev: vdpa device
*/
struct vdpa_config_ops {
/* Virtqueue ops */
@@ -402,6 +420,13 @@ struct vdpa_config_ops {
/* Free device resources */
void (*free)(struct vdpa_device *vdev);
+ /* IOMMUFD ops */
+ int (*bind_iommufd)(struct vdpa_device *vdev, struct iommufd_ctx *ictx,
+ u32 *out_device_id);
+ void (*unbind_iommufd)(struct vdpa_device *vdev);
+ int (*attach_ioas)(struct vdpa_device *vdev, u32 *pt_id);
+ int (*detach_ioas)(struct vdpa_device *vdev);
+
};
struct vdpa_device *__vdpa_alloc_device(struct device *parent,
@@ -570,4 +595,15 @@ struct vdpa_mgmt_dev {
int vdpa_mgmtdev_register(struct vdpa_mgmt_dev *mdev);
void vdpa_mgmtdev_unregister(struct vdpa_mgmt_dev *mdev);
-#endif /* _LINUX_VDPA_H */
+int vdpa_iommufd_physical_bind(struct vdpa_device *vdpa,
+ struct iommufd_ctx *ictx, u32 *out_device_id);
+void vdpa_iommufd_physical_unbind(struct vdpa_device *vdpa);
+int vdpa_iommufd_physical_attach_ioas(struct vdpa_device *vdpa, u32 *pt_id);
+int vdpa_iommufd_physical_detach_ioas(struct vdpa_device *vdpa);
+int vdpa_iommufd_emulated_bind(struct vdpa_device *vdpa,
+ struct iommufd_ctx *ictx, u32 *out_device_id);
+void vdpa_iommufd_emulated_unbind(struct vdpa_device *vdpa);
+int vdpa_iommufd_emulated_attach_ioas(struct vdpa_device *vdpa, u32 *pt_id);
+int vdpa_iommufd_emulated_detach_ioas(struct vdpa_device *vdpa);
+
+#endif
--
2.34.3
VHOST_VDPA_SET_IOMMU_FD: bind the device to iommufd device
VDPA_DEVICE_ATTACH_IOMMUFD_AS: Attach a vdpa device to an iommufd
address space specified by IOAS id.
VDPA_DEVICE_DETACH_IOMMUFD_AS: Detach a vdpa device
from the iommufd address space
Signed-off-by: Cindy Lu <[email protected]>
---
drivers/vhost/vdpa.c | 171 +++++++++++++++++++++++++++++++++++++
include/uapi/linux/vhost.h | 66 ++++++++++++++
2 files changed, 237 insertions(+)
diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
index 78379ffd2336..dfaddd833364 100644
--- a/drivers/vhost/vdpa.c
+++ b/drivers/vhost/vdpa.c
@@ -18,6 +18,7 @@
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/iommu.h>
+#include <linux/iommufd.h>
#include <linux/uuid.h>
#include <linux/vdpa.h>
#include <linux/nospec.h>
@@ -25,6 +26,8 @@
#include "vhost.h"
+MODULE_IMPORT_NS(IOMMUFD);
+
enum {
VHOST_VDPA_BACKEND_FEATURES =
(1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2) |
@@ -69,6 +72,15 @@ static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v,
struct vhost_iotlb *iotlb, u64 start,
u64 last, u32 asid);
+void vhost_vdpa_lockdep_assert_held(struct vdpa_device *vdpa)
+{
+ struct vhost_vdpa *v = vdpa_get_drvdata(vdpa);
+
+ if (WARN_ON(!v))
+ return;
+ lockdep_assert_held(&v->vdev.mutex);
+}
+
static inline u32 iotlb_to_asid(struct vhost_iotlb *iotlb)
{
struct vhost_vdpa_as *as = container_of(iotlb, struct
@@ -551,6 +563,149 @@ static long vhost_vdpa_suspend(struct vhost_vdpa *v)
return ops->suspend(vdpa);
}
+static long vhost_vdpa_iommufd_set_device(struct vhost_vdpa *v,
+ void __user *argp)
+{
+ struct device *dma_dev = vdpa_get_dma_dev(v->vdpa);
+ struct vhost_vdpa_set_iommufd set_iommufd;
+ struct vdpa_device *vdpa = v->vdpa;
+ struct iommufd_ctx *ictx;
+ unsigned long minsz;
+ u32 ioas_id, dev_id;
+ struct fd f;
+ long r = 0;
+
+ minsz = offsetofend(struct vhost_vdpa_set_iommufd, iommufd_ioasid);
+ if (copy_from_user(&set_iommufd, argp, minsz))
+ return -EFAULT;
+
+ /* Unset IOMMUFD */
+ if (set_iommufd.iommufd < 0) {
+ if (!vdpa->iommufd_ictx || !vdpa->iommufd_device)
+ return -EINVAL;
+ if (atomic_read(&vdpa->iommufd_users)) {
+ atomic_dec(&vdpa->iommufd_users);
+ return 0;
+ }
+ vdpa_iommufd_unbind(v->vdpa);
+ vdpa->iommufd_device = NULL;
+ vdpa->iommufd_ictx = NULL;
+ return iommu_attach_device(v->domain, dma_dev);
+ }
+
+ /* For same device but different groups, ++refcount only */
+ if (vdpa->iommufd_device)
+ goto out_inc;
+
+ r = -EBADF;
+ f = fdget(set_iommufd.iommufd);
+ if (!f.file)
+ goto out;
+
+ r = -EINVAL;
+ ictx = iommufd_ctx_from_file(f.file);
+ if (IS_ERR(ictx))
+ goto out_fdput;
+
+ if (v->domain) {
+ iommu_device_unuse_default_domain(dma_dev);
+ iommu_detach_device(v->domain, dma_dev);
+ }
+
+ ioas_id = set_iommufd.iommufd_ioasid;
+ r = vdpa_iommufd_bind(vdpa, ictx, &ioas_id, &dev_id);
+ if (r)
+ goto out_reattach;
+
+ set_iommufd.out_dev_id = dev_id;
+ r = copy_to_user(argp + minsz, &set_iommufd.out_dev_id,
+ sizeof(set_iommufd.out_dev_id)) ?
+ -EFAULT :
+ 0;
+ if (r)
+ goto out_device_unbind;
+
+ vdpa->iommufd_ictx = ictx;
+
+out_inc:
+ atomic_inc(&vdpa->iommufd_users);
+
+ goto out_fdput;
+
+out_device_unbind:
+
+ vdpa_iommufd_unbind(vdpa);
+out_reattach:
+ iommu_device_use_default_domain(dma_dev);
+ iommu_attach_device(v->domain, dma_dev);
+ iommufd_ctx_put(ictx);
+out_fdput:
+ fdput(f);
+out:
+ return r;
+}
+int vhost_vdpa_iommufd_ioas_attach(struct vhost_vdpa *v, void __user *arg)
+{
+ struct vdpa_device_attach_iommufd_as attach;
+ unsigned long minsz;
+ int ret;
+
+ minsz = offsetofend(struct vdpa_device_attach_iommufd_as, ioas_id);
+
+ if (copy_from_user(&attach, (void __user *)arg, minsz))
+ return -EFAULT;
+
+ if (attach.argsz < minsz || attach.flags)
+ return -EINVAL;
+
+ if (!v->vdpa->config->bind_iommufd)
+ return -ENODEV;
+
+ if (!v->vdpa->iommufd_ictx) {
+ ret = -EINVAL;
+ return ret;
+ }
+
+ ret = v->vdpa->config->attach_ioas(v->vdpa, &attach.ioas_id);
+
+ if (ret)
+ return ret;
+
+ ret = copy_to_user(
+ (void __user *)arg +
+ offsetofend(struct vdpa_device_attach_iommufd_as,
+ flags),
+ &attach.ioas_id, sizeof(attach.ioas_id)) ?
+ -EFAULT :
+ 0;
+
+ if (ret)
+ return ret;
+
+ return 0;
+}
+
+int vhost_vdpa_iommufd_ioas_detach(struct vhost_vdpa *v, void __user *arg)
+{
+ struct vdpa_device_detach_iommufd_as detach;
+ unsigned long minsz;
+
+ minsz = offsetofend(struct vdpa_device_detach_iommufd_as, flags);
+
+ if (copy_from_user(&detach, (void __user *)arg, minsz))
+ return -EFAULT;
+
+ if (detach.argsz < minsz || detach.flags)
+ return -EINVAL;
+
+ if (!v->vdpa->config->bind_iommufd)
+ return -ENODEV;
+
+ if (v->vdpa->iommufd_ictx) {
+ return -EINVAL;
+ }
+ return v->vdpa->config->detach_ioas(v->vdpa);
+}
/* After a successful return of this ioctl the device resumes processing
* virtqueue descriptors. The device becomes fully operational the same way it
@@ -744,6 +899,18 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep,
case VHOST_SET_LOG_FD:
r = -ENOIOCTLCMD;
break;
+ case VHOST_VDPA_SET_IOMMU_FD:
+
+ r = vhost_vdpa_iommufd_set_device(v, argp);
+ break;
+ case VDPA_DEVICE_ATTACH_IOMMUFD_AS:
+ r = vhost_vdpa_iommufd_ioas_attach(v, (void __user *)arg);
+ break;
+
+ case VDPA_DEVICE_DETACH_IOMMUFD_AS:
+ r = vhost_vdpa_iommufd_ioas_detach(v, (void __user *)arg);
+ break;
+
case VHOST_VDPA_SET_CONFIG_CALL:
r = vhost_vdpa_set_config_call(v, argp);
break;
@@ -896,6 +1063,10 @@ static int vhost_vdpa_map(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
} else if (ops->set_map) {
if (!v->in_batch)
r = ops->set_map(vdpa, asid, iotlb);
+ } else if (!vdpa->iommufd_ictx) {
+ /* Legacy iommu domain pathway without IOMMUFD */
+ r = iommu_map(v->domain, iova, pa, size,
+ perm_to_iommu_flags(perm), GFP_KERNEL);
} else {
r = iommu_map(v->domain, iova, pa, size,
perm_to_iommu_flags(perm), GFP_KERNEL);
diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h
index f5c48b61ab62..07e1b2c443ca 100644
--- a/include/uapi/linux/vhost.h
+++ b/include/uapi/linux/vhost.h
@@ -219,4 +219,70 @@
*/
#define VHOST_VDPA_RESUME _IO(VHOST_VIRTIO, 0x7E)
+/* vhost_vdpa_set_iommufd
+ * Input parameters:
+ * @iommufd: file descriptor from /dev/iommu; pass -1 to unset
+ * @iommufd_ioasid: IOAS identifier returned from ioctl(IOMMU_IOAS_ALLOC)
+ * Output parameters:
+ * @out_dev_id: device identifier
+ */
+struct vhost_vdpa_set_iommufd {
+ __s32 iommufd;
+ __u32 iommufd_ioasid;
+ __u32 out_dev_id;
+};
+
+#define VHOST_VDPA_SET_IOMMU_FD \
+ _IOW(VHOST_VIRTIO, 0x7F, struct vhost_vdpa_set_iommufd)
+
+/*
+ * VDPA_DEVICE_ATTACH_IOMMUFD_AS -
+ * _IOW(VHOST_VIRTIO, 0x7f, struct vdpa_device_attach_iommufd_as)
+ *
+ * Attach a vdpa device to an iommufd address space specified by IOAS
+ * id.
+ *
+ * Available only after a device has been bound to iommufd via
+ * VHOST_VDPA_SET_IOMMU_FD
+ *
+ * Undo by VDPA_DEVICE_DETACH_IOMMUFD_AS or device fd close.
+ *
+ * @argsz: user filled size of this data.
+ * @flags: must be 0.
+ * @ioas_id: Input the target id which can represent an ioas
+ * allocated via iommufd subsystem.
+ *
+ * Return: 0 on success, -errno on failure.
+ */
+struct vdpa_device_attach_iommufd_as {
+ __u32 argsz;
+ __u32 flags;
+ __u32 ioas_id;
+};
+
+#define VDPA_DEVICE_ATTACH_IOMMUFD_AS \
+ _IOW(VHOST_VIRTIO, 0x82, struct vdpa_device_attach_iommufd_as)
+
+/*
+ * VDPA_DEVICE_DETACH_IOMMUFD_AS
+ *
+ * Detach a vdpa device from the iommufd address space it has been
+ * attached to. After it, device should be in a blocking DMA state.
+ *
+ * Available only after a device has been bound to iommufd via
+ * VHOST_VDPA_SET_IOMMU_FD
+ *
+ * @argsz: user filled size of this data.
+ * @flags: must be 0.
+ *
+ * Return: 0 on success, -errno on failure.
+ */
+struct vdpa_device_detach_iommufd_as {
+ __u32 argsz;
+ __u32 flags;
+};
+
+#define VDPA_DEVICE_DETACH_IOMMUFD_AS \
+ _IOW(VHOST_VIRTIO, 0x83, struct vdpa_device_detach_iommufd_as)
+
#endif
--
2.34.3
Add the check for iommufd_ictx,If vdpa don't have the iommufd_ictx
then will use the Legacy iommu domain pathway
Signed-off-by: Cindy Lu <[email protected]>
---
drivers/vhost/vdpa.c | 43 ++++++++++++++++++++++++++++++++++++++-----
1 file changed, 38 insertions(+), 5 deletions(-)
diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
index dfaddd833364..0e2dba59e1ce 100644
--- a/drivers/vhost/vdpa.c
+++ b/drivers/vhost/vdpa.c
@@ -1067,9 +1067,6 @@ static int vhost_vdpa_map(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
/* Legacy iommu domain pathway without IOMMUFD */
r = iommu_map(v->domain, iova, pa, size,
perm_to_iommu_flags(perm), GFP_KERNEL);
- } else {
- r = iommu_map(v->domain, iova, pa, size,
- perm_to_iommu_flags(perm), GFP_KERNEL);
}
if (r) {
vhost_iotlb_del_range(iotlb, iova, iova + size - 1);
@@ -1095,8 +1092,10 @@ static void vhost_vdpa_unmap(struct vhost_vdpa *v,
if (ops->set_map) {
if (!v->in_batch)
ops->set_map(vdpa, asid, iotlb);
+ } else if (!vdpa->iommufd_ictx) {
+ /* Legacy iommu domain pathway without IOMMUFD */
+ iommu_unmap(v->domain, iova, size);
}
-
}
static int vhost_vdpa_va_map(struct vhost_vdpa *v,
@@ -1149,7 +1148,36 @@ static int vhost_vdpa_va_map(struct vhost_vdpa *v,
return ret;
}
+#if 0
+int vhost_pin_pages(struct vdpa_device *device, dma_addr_t iova, int npage,
+ int prot, struct page **pages)
+{
+ if (!pages || !npage)
+ return -EINVAL;
+ //if (!device->config->dma_unmap)
+ //return -EINVAL;
+
+ if (0) { //device->iommufd_access) {
+ int ret;
+
+ if (iova > ULONG_MAX)
+ return -EINVAL;
+ ret = iommufd_access_pin_pages(
+ device->iommufd_access, iova, npage * PAGE_SIZE, pages,
+ (prot & IOMMU_WRITE) ? IOMMUFD_ACCESS_RW_WRITE : 0);
+ if (ret) {
+
+ return ret;
+ }
+
+ return npage;
+ } else {
+ return pin_user_pages(iova, npage, prot, pages);
+ }
+ return -EINVAL;
+}
+#endif
static int vhost_vdpa_pa_map(struct vhost_vdpa *v,
struct vhost_iotlb *iotlb,
u64 iova, u64 size, u64 uaddr, u32 perm)
@@ -1418,9 +1446,13 @@ static void vhost_vdpa_free_domain(struct vhost_vdpa *v)
struct device *dma_dev = vdpa_get_dma_dev(vdpa);
if (v->domain) {
- iommu_detach_device(v->domain, dma_dev);
+ if (!vdpa->iommufd_ictx) {
+ iommu_detach_device(v->domain, dma_dev);
+ }
iommu_domain_free(v->domain);
}
+ if (vdpa->iommufd_ictx)
+ vdpa_iommufd_unbind(vdpa);
v->domain = NULL;
}
@@ -1645,6 +1677,7 @@ static int vhost_vdpa_probe(struct vdpa_device *vdpa)
}
atomic_set(&v->opened, 0);
+ atomic_set(&vdpa->iommufd_users, 0);
v->minor = minor;
v->vdpa = vdpa;
v->nvqs = vdpa->nvqs;
--
2.34.3
Add new vdpa_config_ops function to support iommufd
Signed-off-by: Cindy Lu <[email protected]>
---
drivers/vdpa/virtio_pci/vp_vdpa.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/drivers/vdpa/virtio_pci/vp_vdpa.c b/drivers/vdpa/virtio_pci/vp_vdpa.c
index 281287fae89f..dd2c372d36a6 100644
--- a/drivers/vdpa/virtio_pci/vp_vdpa.c
+++ b/drivers/vdpa/virtio_pci/vp_vdpa.c
@@ -460,6 +460,10 @@ static const struct vdpa_config_ops vp_vdpa_ops = {
.set_config = vp_vdpa_set_config,
.set_config_cb = vp_vdpa_set_config_cb,
.get_vq_irq = vp_vdpa_get_vq_irq,
+ .bind_iommufd = vdpa_iommufd_physical_bind,
+ .unbind_iommufd = vdpa_iommufd_physical_unbind,
+ .attach_ioas = vdpa_iommufd_physical_attach_ioas,
+ .detach_ioas = vdpa_iommufd_physical_detach_ioas,
};
static void vp_vdpa_free_irq_vectors(void *data)
--
2.34.3
Expose the function iommu_device_use_default_domain() and
iommu_device_unuse_default_domain()，
While vdpa bind the iommufd device and detach the iommu device,
vdpa need to call the function
iommu_device_unuse_default_domain() to release the owner
Signed-off-by: Cindy Lu <[email protected]>
---
drivers/iommu/iommu.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index 3bfc56df4f78..987cbf8c9a87 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -3164,6 +3164,7 @@ int iommu_device_use_default_domain(struct device *dev)
return ret;
}
+EXPORT_SYMBOL_GPL(iommu_device_use_default_domain);
/**
* iommu_device_unuse_default_domain() - Device driver stops handling device
@@ -3187,6 +3188,7 @@ void iommu_device_unuse_default_domain(struct device *dev)
mutex_unlock(&group->mutex);
iommu_group_put(group);
}
+EXPORT_SYMBOL_GPL(iommu_device_unuse_default_domain);
static int __iommu_group_alloc_blocking_domain(struct iommu_group *group)
{
--
2.34.3
On Sat, Nov 04, 2023 at 01:16:41AM +0800, Cindy Lu wrote:
> Expose the function iommu_device_use_default_domain() and
> iommu_device_unuse_default_domain()，
> While vdpa bind the iommufd device and detach the iommu device,
> vdpa need to call the function
> iommu_device_unuse_default_domain() to release the owner
Definately not. You need to set the driver_managed_dma flag.
Jason
On Sat, Nov 4, 2023 at 1:16 AM Cindy Lu <[email protected]> wrote:
>
>
> Hi All
> This code provides the iommufd support for vdpa device
> This code fixes the bugs from the last version and also add the asid support. rebase on kernel
> v6,6-rc3
> Test passed in the physical device (vp_vdpa), but there are still some problems in the emulated device (vdpa_sim_net),
> I will continue working on it
>
> The kernel code is
> https://gitlab.com/lulu6/vhost/-/tree/iommufdRFC_v1
>
> Signed-off-by: Cindy Lu <[email protected]>
It would be better to have a change summary here.
Thanks
>
>
> Cindy Lu (8):
> vhost/iommufd: Add the functions support iommufd
> Kconfig: Add the new file vhost/iommufd
> vhost: Add 3 new uapi to support iommufd
> vdpa: Add new vdpa_config_ops to support iommufd
> vdpa_sim :Add support for iommufd
> vdpa: change the map/unmap process to support iommufd
> vp_vdpa::Add support for iommufd
> iommu: expose the function iommu_device_use_default_domain
>
> drivers/iommu/iommu.c | 2 +
> drivers/vdpa/vdpa_sim/vdpa_sim.c | 8 ++
> drivers/vdpa/virtio_pci/vp_vdpa.c | 4 +
> drivers/vhost/Kconfig | 1 +
> drivers/vhost/Makefile | 1 +
> drivers/vhost/iommufd.c | 178 +++++++++++++++++++++++++
> drivers/vhost/vdpa.c | 210 +++++++++++++++++++++++++++++-
> drivers/vhost/vhost.h | 21 +++
> include/linux/vdpa.h | 38 +++++-
> include/uapi/linux/vhost.h | 66 ++++++++++
> 10 files changed, 525 insertions(+), 4 deletions(-)
> create mode 100644 drivers/vhost/iommufd.c
>
> --
> 2.34.3
>
On Sat, Nov 4, 2023 at 1:17 AM Cindy Lu <[email protected]> wrote:
>
> Add new vdpa_config_ops function to support iommufd
>
> Signed-off-by: Cindy Lu <[email protected]>
> ---
> drivers/vdpa/virtio_pci/vp_vdpa.c | 4 ++++
> 1 file changed, 4 insertions(+)
>
> diff --git a/drivers/vdpa/virtio_pci/vp_vdpa.c b/drivers/vdpa/virtio_pci/vp_vdpa.c
> index 281287fae89f..dd2c372d36a6 100644
> --- a/drivers/vdpa/virtio_pci/vp_vdpa.c
> +++ b/drivers/vdpa/virtio_pci/vp_vdpa.c
> @@ -460,6 +460,10 @@ static const struct vdpa_config_ops vp_vdpa_ops = {
> .set_config = vp_vdpa_set_config,
> .set_config_cb = vp_vdpa_set_config_cb,
> .get_vq_irq = vp_vdpa_get_vq_irq,
> + .bind_iommufd = vdpa_iommufd_physical_bind,
> + .unbind_iommufd = vdpa_iommufd_physical_unbind,
> + .attach_ioas = vdpa_iommufd_physical_attach_ioas,
> + .detach_ioas = vdpa_iommufd_physical_detach_ioas,
For the device that depends on the platform IOMMU, any reason we still
bother a per device config ops here just as an indirection?
Thanks
> };
>
> static void vp_vdpa_free_irq_vectors(void *data)
> --
> 2.34.3
>
On Sat, Nov 4, 2023 at 1:18 AM Cindy Lu <[email protected]> wrote:
>
> Expose the function iommu_device_use_default_domain() and
> iommu_device_unuse_default_domain()，
> While vdpa bind the iommufd device and detach the iommu device,
> vdpa need to call the function
> iommu_device_unuse_default_domain() to release the owner
>
> Signed-off-by: Cindy Lu <[email protected]>
This is the end of the series, who is the user then?
Thanks
> ---
> drivers/iommu/iommu.c | 2 ++
> 1 file changed, 2 insertions(+)
>
> diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
> index 3bfc56df4f78..987cbf8c9a87 100644
> --- a/drivers/iommu/iommu.c
> +++ b/drivers/iommu/iommu.c
> @@ -3164,6 +3164,7 @@ int iommu_device_use_default_domain(struct device *dev)
>
> return ret;
> }
> +EXPORT_SYMBOL_GPL(iommu_device_use_default_domain);
>
> /**
> * iommu_device_unuse_default_domain() - Device driver stops handling device
> @@ -3187,6 +3188,7 @@ void iommu_device_unuse_default_domain(struct device *dev)
> mutex_unlock(&group->mutex);
> iommu_group_put(group);
> }
> +EXPORT_SYMBOL_GPL(iommu_device_unuse_default_domain);
>
> static int __iommu_group_alloc_blocking_domain(struct iommu_group *group)
> {
> --
> 2.34.3
>
On Sat, Nov 4, 2023 at 1:17 AM Cindy Lu <[email protected]> wrote:
>
> VHOST_VDPA_SET_IOMMU_FD: bind the device to iommufd device
>
> VDPA_DEVICE_ATTACH_IOMMUFD_AS: Attach a vdpa device to an iommufd
> address space specified by IOAS id.
>
> VDPA_DEVICE_DETACH_IOMMUFD_AS: Detach a vdpa device
> from the iommufd address space
>
> Signed-off-by: Cindy Lu <[email protected]>
As discussed in the previous version, any reason/advantages of this
compared to just having a single VDPA_DEVICE_ATTACH_IOMMUFD_AS?
Thanks
On Sat, Nov 4, 2023 at 1:17 AM Cindy Lu <[email protected]> wrote:
>
> VHOST_VDPA_SET_IOMMU_FD: bind the device to iommufd device
>
> VDPA_DEVICE_ATTACH_IOMMUFD_AS: Attach a vdpa device to an iommufd
> address space specified by IOAS id.
>
> VDPA_DEVICE_DETACH_IOMMUFD_AS: Detach a vdpa device
> from the iommufd address space
>
> Signed-off-by: Cindy Lu <[email protected]>
> ---
[...]
> diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h
> index f5c48b61ab62..07e1b2c443ca 100644
> --- a/include/uapi/linux/vhost.h
> +++ b/include/uapi/linux/vhost.h
> @@ -219,4 +219,70 @@
> */
> #define VHOST_VDPA_RESUME _IO(VHOST_VIRTIO, 0x7E)
>
> +/* vhost_vdpa_set_iommufd
> + * Input parameters:
> + * @iommufd: file descriptor from /dev/iommu; pass -1 to unset
> + * @iommufd_ioasid: IOAS identifier returned from ioctl(IOMMU_IOAS_ALLOC)
> + * Output parameters:
> + * @out_dev_id: device identifier
> + */
> +struct vhost_vdpa_set_iommufd {
> + __s32 iommufd;
> + __u32 iommufd_ioasid;
> + __u32 out_dev_id;
> +};
> +
> +#define VHOST_VDPA_SET_IOMMU_FD \
> + _IOW(VHOST_VIRTIO, 0x7F, struct vhost_vdpa_set_iommufd)
> +
> +/*
> + * VDPA_DEVICE_ATTACH_IOMMUFD_AS -
> + * _IOW(VHOST_VIRTIO, 0x7f, struct vdpa_device_attach_iommufd_as)
> + *
> + * Attach a vdpa device to an iommufd address space specified by IOAS
> + * id.
> + *
> + * Available only after a device has been bound to iommufd via
> + * VHOST_VDPA_SET_IOMMU_FD
> + *
> + * Undo by VDPA_DEVICE_DETACH_IOMMUFD_AS or device fd close.
> + *
> + * @argsz: user filled size of this data.
> + * @flags: must be 0.
> + * @ioas_id: Input the target id which can represent an ioas
> + * allocated via iommufd subsystem.
> + *
> + * Return: 0 on success, -errno on failure.
> + */
> +struct vdpa_device_attach_iommufd_as {
> + __u32 argsz;
> + __u32 flags;
> + __u32 ioas_id;
> +};
I think we need to map ioas to vDPA AS, so there should be an ASID
from the view of vDPA?
Thanks
> +
> +#define VDPA_DEVICE_ATTACH_IOMMUFD_AS \
> + _IOW(VHOST_VIRTIO, 0x82, struct vdpa_device_attach_iommufd_as)
> +
> +/*
> + * VDPA_DEVICE_DETACH_IOMMUFD_AS
> + *
> + * Detach a vdpa device from the iommufd address space it has been
> + * attached to. After it, device should be in a blocking DMA state.
> + *
> + * Available only after a device has been bound to iommufd via
> + * VHOST_VDPA_SET_IOMMU_FD
> + *
> + * @argsz: user filled size of this data.
> + * @flags: must be 0.
> + *
> + * Return: 0 on success, -errno on failure.
> + */
> +struct vdpa_device_detach_iommufd_as {
> + __u32 argsz;
> + __u32 flags;
> +};
> +
> +#define VDPA_DEVICE_DETACH_IOMMUFD_AS \
> + _IOW(VHOST_VIRTIO, 0x83, struct vdpa_device_detach_iommufd_as)
> +
> #endif
> --
> 2.34.3
>
On 2023/11/6 12:11, Jason Wang wrote:
> On Sat, Nov 4, 2023 at 1:16 AM Cindy Lu <[email protected]> wrote:
>>
>>
>> Hi All
>> This code provides the iommufd support for vdpa device
>> This code fixes the bugs from the last version and also add the asid support. rebase on kernel
>> v6,6-rc3
>> Test passed in the physical device (vp_vdpa), but there are still some problems in the emulated device (vdpa_sim_net),
>> I will continue working on it
>>
>> The kernel code is
>> https://gitlab.com/lulu6/vhost/-/tree/iommufdRFC_v1
>>
>> Signed-off-by: Cindy Lu <[email protected]>
>
> It would be better to have a change summary here.
yeah, the version number is also incorrect.
> Thanks
>
>>
>>
>> Cindy Lu (8):
>> vhost/iommufd: Add the functions support iommufd
>> Kconfig: Add the new file vhost/iommufd
>> vhost: Add 3 new uapi to support iommufd
>> vdpa: Add new vdpa_config_ops to support iommufd
>> vdpa_sim :Add support for iommufd
>> vdpa: change the map/unmap process to support iommufd
>> vp_vdpa::Add support for iommufd
>> iommu: expose the function iommu_device_use_default_domain
>>
>> drivers/iommu/iommu.c | 2 +
>> drivers/vdpa/vdpa_sim/vdpa_sim.c | 8 ++
>> drivers/vdpa/virtio_pci/vp_vdpa.c | 4 +
>> drivers/vhost/Kconfig | 1 +
>> drivers/vhost/Makefile | 1 +
>> drivers/vhost/iommufd.c | 178 +++++++++++++++++++++++++
>> drivers/vhost/vdpa.c | 210 +++++++++++++++++++++++++++++-
>> drivers/vhost/vhost.h | 21 +++
>> include/linux/vdpa.h | 38 +++++-
>> include/uapi/linux/vhost.h | 66 ++++++++++
>> 10 files changed, 525 insertions(+), 4 deletions(-)
>> create mode 100644 drivers/vhost/iommufd.c
>>
>> --
>> 2.34.3
>>
>
--
Regards,
Yi Liu
On 2023/11/4 01:16, Cindy Lu wrote:
> Add 4 new vdpa_config_ops function to support iommufd
>
> Signed-off-by: Cindy Lu <[email protected]>
> ---
> include/linux/vdpa.h | 38 +++++++++++++++++++++++++++++++++++++-
> 1 file changed, 37 insertions(+), 1 deletion(-)
>
> diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h
> index 0e652026b776..233d80f9d910 100644
> --- a/include/linux/vdpa.h
> +++ b/include/linux/vdpa.h
> @@ -5,6 +5,7 @@
> #include <linux/kernel.h>
> #include <linux/device.h>
> #include <linux/interrupt.h>
> +#include <linux/iommufd.h>
> #include <linux/vhost_iotlb.h>
> #include <linux/virtio_net.h>
> #include <linux/if_ether.h>
> @@ -97,6 +98,12 @@ struct vdpa_device {
> struct vdpa_mgmt_dev *mdev;
> unsigned int ngroups;
> unsigned int nas;
> + struct iommufd_access *iommufd_access;
> + struct iommufd_device *iommufd_device;
> + struct iommufd_ctx *iommufd_ictx;
> + unsigned long *vq_bitmap;
> + atomic_t iommufd_users;
> + bool iommufd_attached;
> };
>
> /**
> @@ -332,6 +339,17 @@ struct vdpa_map_file {
> * @vdev: vdpa device
> * @free: Free resources that belongs to vDPA (optional)
> * @vdev: vdpa device
> + * @bind_iommufd: use vdpa_iommufd_physical_bind for an IOMMU
> + * backed device.
> + * otherwise use vdpa_iommufd_emulated_bind
> + * @unbind_iommufd: use vdpa_iommufd_physical_unbind for an IOMMU
> + * backed device.
> + * otherwise, use vdpa_iommufd_emulated_unbind
> + * @attach_ioas: use vdpa_iommufd_physical_attach_ioas for an
> + * IOMMU backed device.
> + * @detach_ioas: Opposite of attach_ioas
> + * @free: Free resources that belongs to vDPA (optional)
> + * @vdev: vdpa device
duplicated kdoc.
> */
> struct vdpa_config_ops {
> /* Virtqueue ops */
> @@ -402,6 +420,13 @@ struct vdpa_config_ops {
>
> /* Free device resources */
> void (*free)(struct vdpa_device *vdev);
> + /* IOMMUFD ops */
> + int (*bind_iommufd)(struct vdpa_device *vdev, struct iommufd_ctx *ictx,
> + u32 *out_device_id);
> + void (*unbind_iommufd)(struct vdpa_device *vdev);
> + int (*attach_ioas)(struct vdpa_device *vdev, u32 *pt_id);
> + int (*detach_ioas)(struct vdpa_device *vdev);
> +
> };
>
> struct vdpa_device *__vdpa_alloc_device(struct device *parent,
> @@ -570,4 +595,15 @@ struct vdpa_mgmt_dev {
> int vdpa_mgmtdev_register(struct vdpa_mgmt_dev *mdev);
> void vdpa_mgmtdev_unregister(struct vdpa_mgmt_dev *mdev);
>
> -#endif /* _LINUX_VDPA_H */
> +int vdpa_iommufd_physical_bind(struct vdpa_device *vdpa,
> + struct iommufd_ctx *ictx, u32 *out_device_id);
> +void vdpa_iommufd_physical_unbind(struct vdpa_device *vdpa);
> +int vdpa_iommufd_physical_attach_ioas(struct vdpa_device *vdpa, u32 *pt_id);
> +int vdpa_iommufd_physical_detach_ioas(struct vdpa_device *vdpa);
> +int vdpa_iommufd_emulated_bind(struct vdpa_device *vdpa,
> + struct iommufd_ctx *ictx, u32 *out_device_id);
> +void vdpa_iommufd_emulated_unbind(struct vdpa_device *vdpa);
> +int vdpa_iommufd_emulated_attach_ioas(struct vdpa_device *vdpa, u32 *pt_id);
> +int vdpa_iommufd_emulated_detach_ioas(struct vdpa_device *vdpa);
> +
> +#endif
--
Regards,
Yi Liu
On 2023/11/4 01:16, Cindy Lu wrote:
> Add the check for iommufd_ictx,If vdpa don't have the iommufd_ictx
> then will use the Legacy iommu domain pathway
>
> Signed-off-by: Cindy Lu <[email protected]>
> ---
> drivers/vhost/vdpa.c | 43 ++++++++++++++++++++++++++++++++++++++-----
> 1 file changed, 38 insertions(+), 5 deletions(-)
>
> diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
> index dfaddd833364..0e2dba59e1ce 100644
> --- a/drivers/vhost/vdpa.c
> +++ b/drivers/vhost/vdpa.c
> @@ -1067,9 +1067,6 @@ static int vhost_vdpa_map(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
> /* Legacy iommu domain pathway without IOMMUFD */
> r = iommu_map(v->domain, iova, pa, size,
> perm_to_iommu_flags(perm), GFP_KERNEL);
> - } else {
> - r = iommu_map(v->domain, iova, pa, size,
> - perm_to_iommu_flags(perm), GFP_KERNEL);
> }
> if (r) {
> vhost_iotlb_del_range(iotlb, iova, iova + size - 1);
> @@ -1095,8 +1092,10 @@ static void vhost_vdpa_unmap(struct vhost_vdpa *v,
> if (ops->set_map) {
> if (!v->in_batch)
> ops->set_map(vdpa, asid, iotlb);
> + } else if (!vdpa->iommufd_ictx) {
> + /* Legacy iommu domain pathway without IOMMUFD */
> + iommu_unmap(v->domain, iova, size);
> }
> -
> }
>
> static int vhost_vdpa_va_map(struct vhost_vdpa *v,
> @@ -1149,7 +1148,36 @@ static int vhost_vdpa_va_map(struct vhost_vdpa *v,
>
> return ret;
> }
> +#if 0
> +int vhost_pin_pages(struct vdpa_device *device, dma_addr_t iova, int npage,
> + int prot, struct page **pages)
> +{
> + if (!pages || !npage)
> + return -EINVAL;
> + //if (!device->config->dma_unmap)
> + //return -EINVAL;
> +
> + if (0) { //device->iommufd_access) {
> + int ret;
> +
> + if (iova > ULONG_MAX)
> + return -EINVAL;
>
> + ret = iommufd_access_pin_pages(
> + device->iommufd_access, iova, npage * PAGE_SIZE, pages,
> + (prot & IOMMU_WRITE) ? IOMMUFD_ACCESS_RW_WRITE : 0);
> + if (ret) {
> +
> + return ret;
> + }
> +
> + return npage;
> + } else {
> + return pin_user_pages(iova, npage, prot, pages);
> + }
> + return -EINVAL;
> +}
> +#endif
Is above code needed or not?
> static int vhost_vdpa_pa_map(struct vhost_vdpa *v,
> struct vhost_iotlb *iotlb,
> u64 iova, u64 size, u64 uaddr, u32 perm)
> @@ -1418,9 +1446,13 @@ static void vhost_vdpa_free_domain(struct vhost_vdpa *v)
> struct device *dma_dev = vdpa_get_dma_dev(vdpa);
>
> if (v->domain) {
> - iommu_detach_device(v->domain, dma_dev);
> + if (!vdpa->iommufd_ictx) {
> + iommu_detach_device(v->domain, dma_dev);
> + }
> iommu_domain_free(v->domain);
> }
> + if (vdpa->iommufd_ictx)
> + vdpa_iommufd_unbind(vdpa);
>
> v->domain = NULL;
> }
> @@ -1645,6 +1677,7 @@ static int vhost_vdpa_probe(struct vdpa_device *vdpa)
> }
>
> atomic_set(&v->opened, 0);
> + atomic_set(&vdpa->iommufd_users, 0);
> v->minor = minor;
> v->vdpa = vdpa;
> v->nvqs = vdpa->nvqs;
--
Regards,
Yi Liu
On Mon, Nov 6, 2023 at 3:26 PM Jason Wang <[email protected]> wrote:
>
> On Sat, Nov 4, 2023 at 1:18 AM Cindy Lu <[email protected]> wrote:
> >
> > Expose the function iommu_device_use_default_domain() and
> > iommu_device_unuse_default_domain()，
> > While vdpa bind the iommufd device and detach the iommu device,
> > vdpa need to call the function
> > iommu_device_unuse_default_domain() to release the owner
> >
> > Signed-off-by: Cindy Lu <[email protected]>
>
> This is the end of the series, who is the user then?
>
> Thanks
>
hi Jason
These 2 functions was called in vhost_vdpa_iommufd_set_device(), Vdpa need to
release the dma owner, otherwise, the function will fail when
iommufd called iommu_device_claim_dma_owner() in iommufd_device_bind()
I will change this sequence, Or maybe will find some other way to fix
this problem
thanks
cindy
> > ---
> > drivers/iommu/iommu.c | 2 ++
> > 1 file changed, 2 insertions(+)
> >
> > diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
> > index 3bfc56df4f78..987cbf8c9a87 100644
> > --- a/drivers/iommu/iommu.c
> > +++ b/drivers/iommu/iommu.c
> > @@ -3164,6 +3164,7 @@ int iommu_device_use_default_domain(struct device *dev)
> >
> > return ret;
> > }
> > +EXPORT_SYMBOL_GPL(iommu_device_use_default_domain);
> >
> > /**
> > * iommu_device_unuse_default_domain() - Device driver stops handling device
> > @@ -3187,6 +3188,7 @@ void iommu_device_unuse_default_domain(struct device *dev)
> > mutex_unlock(&group->mutex);
> > iommu_group_put(group);
> > }
> > +EXPORT_SYMBOL_GPL(iommu_device_unuse_default_domain);
> >
> > static int __iommu_group_alloc_blocking_domain(struct iommu_group *group)
> > {
> > --
> > 2.34.3
> >
>
On Mon, Nov 6, 2023 at 4:52 PM Yi Liu <[email protected]> wrote:
>
> On 2023/11/4 01:16, Cindy Lu wrote:
> > Add the check for iommufd_ictx,If vdpa don't have the iommufd_ictx
> > then will use the Legacy iommu domain pathway
> >
> > Signed-off-by: Cindy Lu <[email protected]>
> > ---
> > drivers/vhost/vdpa.c | 43 ++++++++++++++++++++++++++++++++++++++-----
> > 1 file changed, 38 insertions(+), 5 deletions(-)
> >
> > diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
> > index dfaddd833364..0e2dba59e1ce 100644
> > --- a/drivers/vhost/vdpa.c
> > +++ b/drivers/vhost/vdpa.c
> > @@ -1067,9 +1067,6 @@ static int vhost_vdpa_map(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
> > /* Legacy iommu domain pathway without IOMMUFD */
> > r = iommu_map(v->domain, iova, pa, size,
> > perm_to_iommu_flags(perm), GFP_KERNEL);
> > - } else {
> > - r = iommu_map(v->domain, iova, pa, size,
> > - perm_to_iommu_flags(perm), GFP_KERNEL);
> > }
> > if (r) {
> > vhost_iotlb_del_range(iotlb, iova, iova + size - 1);
> > @@ -1095,8 +1092,10 @@ static void vhost_vdpa_unmap(struct vhost_vdpa *v,
> > if (ops->set_map) {
> > if (!v->in_batch)
> > ops->set_map(vdpa, asid, iotlb);
> > + } else if (!vdpa->iommufd_ictx) {
> > + /* Legacy iommu domain pathway without IOMMUFD */
> > + iommu_unmap(v->domain, iova, size);
> > }
> > -
> > }
> >
> > static int vhost_vdpa_va_map(struct vhost_vdpa *v,
> > @@ -1149,7 +1148,36 @@ static int vhost_vdpa_va_map(struct vhost_vdpa *v,
> >
> > return ret;
> > }
> > +#if 0
> > +int vhost_pin_pages(struct vdpa_device *device, dma_addr_t iova, int npage,
> > + int prot, struct page **pages)
> > +{
> > + if (!pages || !npage)
> > + return -EINVAL;
> > + //if (!device->config->dma_unmap)
> > + //return -EINVAL;
> > +
> > + if (0) { //device->iommufd_access) {
> > + int ret;
> > +
> > + if (iova > ULONG_MAX)
> > + return -EINVAL;
> >
> > + ret = iommufd_access_pin_pages(
> > + device->iommufd_access, iova, npage * PAGE_SIZE, pages,
> > + (prot & IOMMU_WRITE) ? IOMMUFD_ACCESS_RW_WRITE : 0);
> > + if (ret) {
> > +
> > + return ret;
> > + }
> > +
> > + return npage;
> > + } else {
> > + return pin_user_pages(iova, npage, prot, pages);
> > + }
> > + return -EINVAL;
> > +}
> > +#endif
>
> Is above code needed or not?
this code is for simulator， and this device still has some bugs I will
continue working in it,
Thanks
cindy
>
> > static int vhost_vdpa_pa_map(struct vhost_vdpa *v,
> > struct vhost_iotlb *iotlb,
> > u64 iova, u64 size, u64 uaddr, u32 perm)
> > @@ -1418,9 +1446,13 @@ static void vhost_vdpa_free_domain(struct vhost_vdpa *v)
> > struct device *dma_dev = vdpa_get_dma_dev(vdpa);
> >
> > if (v->domain) {
> > - iommu_detach_device(v->domain, dma_dev);
> > + if (!vdpa->iommufd_ictx) {
> > + iommu_detach_device(v->domain, dma_dev);
> > + }
> > iommu_domain_free(v->domain);
> > }
> > + if (vdpa->iommufd_ictx)
> > + vdpa_iommufd_unbind(vdpa);
> >
> > v->domain = NULL;
> > }
> > @@ -1645,6 +1677,7 @@ static int vhost_vdpa_probe(struct vdpa_device *vdpa)
> > }
> >
> > atomic_set(&v->opened, 0);
> > + atomic_set(&vdpa->iommufd_users, 0);
> > v->minor = minor;
> > v->vdpa = vdpa;
> > v->nvqs = vdpa->nvqs;
>
> --
> Regards,
> Yi Liu
>
On Mon, Nov 6, 2023 at 3:30 PM Jason Wang <[email protected]> wrote:
>
> On Sat, Nov 4, 2023 at 1:17 AM Cindy Lu <[email protected]> wrote:
> >
> > VHOST_VDPA_SET_IOMMU_FD: bind the device to iommufd device
> >
> > VDPA_DEVICE_ATTACH_IOMMUFD_AS: Attach a vdpa device to an iommufd
> > address space specified by IOAS id.
> >
> > VDPA_DEVICE_DETACH_IOMMUFD_AS: Detach a vdpa device
> > from the iommufd address space
> >
> > Signed-off-by: Cindy Lu <[email protected]>
> > ---
>
> [...]
>
> > diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h
> > index f5c48b61ab62..07e1b2c443ca 100644
> > --- a/include/uapi/linux/vhost.h
> > +++ b/include/uapi/linux/vhost.h
> > @@ -219,4 +219,70 @@
> > */
> > #define VHOST_VDPA_RESUME _IO(VHOST_VIRTIO, 0x7E)
> >
> > +/* vhost_vdpa_set_iommufd
> > + * Input parameters:
> > + * @iommufd: file descriptor from /dev/iommu; pass -1 to unset
> > + * @iommufd_ioasid: IOAS identifier returned from ioctl(IOMMU_IOAS_ALLOC)
> > + * Output parameters:
> > + * @out_dev_id: device identifier
> > + */
> > +struct vhost_vdpa_set_iommufd {
> > + __s32 iommufd;
> > + __u32 iommufd_ioasid;
> > + __u32 out_dev_id;
> > +};
> > +
> > +#define VHOST_VDPA_SET_IOMMU_FD \
> > + _IOW(VHOST_VIRTIO, 0x7F, struct vhost_vdpa_set_iommufd)
> > +
> > +/*
> > + * VDPA_DEVICE_ATTACH_IOMMUFD_AS -
> > + * _IOW(VHOST_VIRTIO, 0x7f, struct vdpa_device_attach_iommufd_as)
> > + *
> > + * Attach a vdpa device to an iommufd address space specified by IOAS
> > + * id.
> > + *
> > + * Available only after a device has been bound to iommufd via
> > + * VHOST_VDPA_SET_IOMMU_FD
> > + *
> > + * Undo by VDPA_DEVICE_DETACH_IOMMUFD_AS or device fd close.
> > + *
> > + * @argsz: user filled size of this data.
> > + * @flags: must be 0.
> > + * @ioas_id: Input the target id which can represent an ioas
> > + * allocated via iommufd subsystem.
> > + *
> > + * Return: 0 on success, -errno on failure.
> > + */
> > +struct vdpa_device_attach_iommufd_as {
> > + __u32 argsz;
> > + __u32 flags;
> > + __u32 ioas_id;
> > +};
>
> I think we need to map ioas to vDPA AS, so there should be an ASID
> from the view of vDPA?
>
> Thanks
>
The qemu will have a structure save and maintain this information,So
I didn't add this
in kernel，we can add this but maybe only for check?
this in
Thanks
Cindy
> > +
> > +#define VDPA_DEVICE_ATTACH_IOMMUFD_AS \
> > + _IOW(VHOST_VIRTIO, 0x82, struct vdpa_device_attach_iommufd_as)
> > +
> > +/*
> > + * VDPA_DEVICE_DETACH_IOMMUFD_AS
> > + *
> > + * Detach a vdpa device from the iommufd address space it has been
> > + * attached to. After it, device should be in a blocking DMA state.
> > + *
> > + * Available only after a device has been bound to iommufd via
> > + * VHOST_VDPA_SET_IOMMU_FD
> > + *
> > + * @argsz: user filled size of this data.
> > + * @flags: must be 0.
> > + *
> > + * Return: 0 on success, -errno on failure.
> > + */
> > +struct vdpa_device_detach_iommufd_as {
> > + __u32 argsz;
> > + __u32 flags;
> > +};
> > +
> > +#define VDPA_DEVICE_DETACH_IOMMUFD_AS \
> > + _IOW(VHOST_VIRTIO, 0x83, struct vdpa_device_detach_iommufd_as)
> > +
> > #endif
> > --
> > 2.34.3
> >
>
On Sat, Nov 04, 2023 at 01:16:33AM +0800, Cindy Lu wrote:
>
> Hi All
> This code provides the iommufd support for vdpa device
> This code fixes the bugs from the last version and also add the asid support. rebase on kernel
> v6,6-rc3
> Test passed in the physical device (vp_vdpa), but there are still some problems in the emulated device (vdpa_sim_net),
What kind of problems? Understanding that will make it easier
to figure out whether this is worth reviewing.
> I will continue working on it
>
> The kernel code is
> https://gitlab.com/lulu6/vhost/-/tree/iommufdRFC_v1
>
> Signed-off-by: Cindy Lu <[email protected]>
Please also Cc iommufd maintainers:
Jason Gunthorpe <[email protected]> (maintainer:IOMMUFD)
Kevin Tian <[email protected]> (maintainer:IOMMUFD)
Joerg Roedel <[email protected]> (maintainer:IOMMU SUBSYSTEM)
Will Deacon <[email protected]> (maintainer:IOMMU SUBSYSTEM)
Robin Murphy <[email protected]> (reviewer:IOMMU SUBSYSTEM)
[email protected] (open list:IOMMUFD)
[email protected] (open list)
--
MST
On Sat, Nov 04, 2023 at 01:16:33AM +0800, Cindy Lu wrote:
>
> Hi All
> This code provides the iommufd support for vdpa device
> This code fixes the bugs from the last version and also add the asid support. rebase on kernel
> v6,6-rc3
> Test passed in the physical device (vp_vdpa), but there are still some problems in the emulated device (vdpa_sim_net),
> I will continue working on it
>
> The kernel code is
> https://gitlab.com/lulu6/vhost/-/tree/iommufdRFC_v1
>
> Signed-off-by: Cindy Lu <[email protected]>
Was this abandoned?
>
> Cindy Lu (8):
> vhost/iommufd: Add the functions support iommufd
> Kconfig: Add the new file vhost/iommufd
> vhost: Add 3 new uapi to support iommufd
> vdpa: Add new vdpa_config_ops to support iommufd
> vdpa_sim :Add support for iommufd
> vdpa: change the map/unmap process to support iommufd
> vp_vdpa::Add support for iommufd
> iommu: expose the function iommu_device_use_default_domain
>
> drivers/iommu/iommu.c | 2 +
> drivers/vdpa/vdpa_sim/vdpa_sim.c | 8 ++
> drivers/vdpa/virtio_pci/vp_vdpa.c | 4 +
> drivers/vhost/Kconfig | 1 +
> drivers/vhost/Makefile | 1 +
> drivers/vhost/iommufd.c | 178 +++++++++++++++++++++++++
> drivers/vhost/vdpa.c | 210 +++++++++++++++++++++++++++++-
> drivers/vhost/vhost.h | 21 +++
> include/linux/vdpa.h | 38 +++++-
> include/uapi/linux/vhost.h | 66 ++++++++++
> 10 files changed, 525 insertions(+), 4 deletions(-)
> create mode 100644 drivers/vhost/iommufd.c
>
> --
> 2.34.3
On Thu, Jan 11, 2024 at 6:25 AM Michael S. Tsirkin <[email protected]> wrote:
>
> On Sat, Nov 04, 2023 at 01:16:33AM +0800, Cindy Lu wrote:
> >
> > Hi All
> > This code provides the iommufd support for vdpa device
> > This code fixes the bugs from the last version and also add the asid support. rebase on kernel
> > v6,6-rc3
> > Test passed in the physical device (vp_vdpa), but there are still some problems in the emulated device (vdpa_sim_net),
> > I will continue working on it
> >
> > The kernel code is
> > https://gitlab.com/lulu6/vhost/-/tree/iommufdRFC_v1
> >
> > Signed-off-by: Cindy Lu <[email protected]>
>
> Was this abandoned?
>
Thanks Micheal. I'm really sorry for the delay. I will continue working on this
Thanks
Cindy
> >
> > Cindy Lu (8):
> > vhost/iommufd: Add the functions support iommufd
> > Kconfig: Add the new file vhost/iommufd
> > vhost: Add 3 new uapi to support iommufd
> > vdpa: Add new vdpa_config_ops to support iommufd
> > vdpa_sim :Add support for iommufd
> > vdpa: change the map/unmap process to support iommufd
> > vp_vdpa::Add support for iommufd
> > iommu: expose the function iommu_device_use_default_domain
> >
> > drivers/iommu/iommu.c | 2 +
> > drivers/vdpa/vdpa_sim/vdpa_sim.c | 8 ++
> > drivers/vdpa/virtio_pci/vp_vdpa.c | 4 +
> > drivers/vhost/Kconfig | 1 +
> > drivers/vhost/Makefile | 1 +
> > drivers/vhost/iommufd.c | 178 +++++++++++++++++++++++++
> > drivers/vhost/vdpa.c | 210 +++++++++++++++++++++++++++++-
> > drivers/vhost/vhost.h | 21 +++
> > include/linux/vdpa.h | 38 +++++-
> > include/uapi/linux/vhost.h | 66 ++++++++++
> > 10 files changed, 525 insertions(+), 4 deletions(-)
> > create mode 100644 drivers/vhost/iommufd.c
> >
> > --
> > 2.34.3
>