2021-02-23 21:19:56

by Auger Eric

[permalink] [raw]
Subject: [PATCH v12 03/13] vfio: VFIO_IOMMU_SET_MSI_BINDING

This patch adds the VFIO_IOMMU_SET_MSI_BINDING ioctl which aim
to (un)register the guest MSI binding to the host. This latter
then can use those stage 1 bindings to build a nested stage
binding targeting the physical MSIs.

Signed-off-by: Eric Auger <[email protected]>

---

v11 -> v12:
- Share VFIO_BASE + 20 with VFIO_IOMMU_SPAPR_TCE_REMOVE
- rework returned values

v10 -> v11:
- renamed ustruct into msi_binding
- return 0 on unbind

v8 -> v9:
- merge VFIO_IOMMU_BIND_MSI/VFIO_IOMMU_UNBIND_MSI into a single
VFIO_IOMMU_SET_MSI_BINDING ioctl
- ioctl id changed

v6 -> v7:
- removed the dev arg

v3 -> v4:
- add UNBIND
- unwind on BIND error

v2 -> v3:
- adapt to new proto of bind_guest_msi
- directly use vfio_iommu_for_each_dev

v1 -> v2:
- s/vfio_iommu_type1_guest_msi_binding/vfio_iommu_type1_bind_guest_msi
---
drivers/vfio/vfio_iommu_type1.c | 62 +++++++++++++++++++++++++++++++++
include/uapi/linux/vfio.h | 20 +++++++++++
2 files changed, 82 insertions(+)

diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index b4057ce809b0..0e6af4fe8c3d 100644
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -2655,6 +2655,41 @@ static int vfio_cache_inv_fn(struct device *dev, void *data)
return iommu_uapi_cache_invalidate(dc->domain, dev, (void __user *)arg);
}

+static int
+vfio_bind_msi(struct vfio_iommu *iommu,
+ dma_addr_t giova, phys_addr_t gpa, size_t size)
+{
+ struct vfio_domain *d;
+ int ret = 0;
+
+ mutex_lock(&iommu->lock);
+
+ list_for_each_entry(d, &iommu->domain_list, next) {
+ ret = iommu_bind_guest_msi(d->domain, giova, gpa, size);
+ if (ret)
+ goto unwind;
+ }
+ goto unlock;
+unwind:
+ list_for_each_entry_continue_reverse(d, &iommu->domain_list, next) {
+ iommu_unbind_guest_msi(d->domain, giova);
+ }
+unlock:
+ mutex_unlock(&iommu->lock);
+ return ret;
+}
+
+static void
+vfio_unbind_msi(struct vfio_iommu *iommu, dma_addr_t giova)
+{
+ struct vfio_domain *d;
+
+ mutex_lock(&iommu->lock);
+ list_for_each_entry(d, &iommu->domain_list, next)
+ iommu_unbind_guest_msi(d->domain, giova);
+ mutex_unlock(&iommu->lock);
+}
+
static int vfio_iommu_migration_build_caps(struct vfio_iommu *iommu,
struct vfio_info_cap *caps)
{
@@ -2859,6 +2894,31 @@ static int vfio_iommu_type1_cache_invalidate(struct vfio_iommu *iommu,
return ret;
}

+static int vfio_iommu_type1_set_msi_binding(struct vfio_iommu *iommu,
+ unsigned long arg)
+{
+ struct vfio_iommu_type1_set_msi_binding msi_binding;
+ unsigned long minsz;
+
+ minsz = offsetofend(struct vfio_iommu_type1_set_msi_binding,
+ size);
+
+ if (copy_from_user(&msi_binding, (void __user *)arg, minsz))
+ return -EFAULT;
+
+ if (msi_binding.argsz < minsz)
+ return -EINVAL;
+
+ if (msi_binding.flags == VFIO_IOMMU_UNBIND_MSI) {
+ vfio_unbind_msi(iommu, msi_binding.iova);
+ return 0;
+ } else if (msi_binding.flags == VFIO_IOMMU_BIND_MSI) {
+ return vfio_bind_msi(iommu, msi_binding.iova,
+ msi_binding.gpa, msi_binding.size);
+ }
+ return -EINVAL;
+}
+
static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu,
unsigned long arg)
{
@@ -2983,6 +3043,8 @@ static long vfio_iommu_type1_ioctl(void *iommu_data,
return vfio_iommu_type1_set_pasid_table(iommu, arg);
case VFIO_IOMMU_CACHE_INVALIDATE:
return vfio_iommu_type1_cache_invalidate(iommu, arg);
+ case VFIO_IOMMU_SET_MSI_BINDING:
+ return vfio_iommu_type1_set_msi_binding(iommu, arg);
default:
return -ENOTTY;
}
diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
index ee6747ff8006..4c82e8f21618 100644
--- a/include/uapi/linux/vfio.h
+++ b/include/uapi/linux/vfio.h
@@ -1213,6 +1213,26 @@ struct vfio_iommu_type1_cache_invalidate {
};
#define VFIO_IOMMU_CACHE_INVALIDATE _IO(VFIO_TYPE, VFIO_BASE + 19)

+/**
+ * VFIO_IOMMU_SET_MSI_BINDING - _IOWR(VFIO_TYPE, VFIO_BASE + 20,
+ * struct vfio_iommu_type1_set_msi_binding)
+ *
+ * Pass a stage 1 MSI doorbell mapping to the host so that this
+ * latter can build a nested stage2 mapping. Or conversely tear
+ * down a previously bound stage 1 MSI binding.
+ */
+struct vfio_iommu_type1_set_msi_binding {
+ __u32 argsz;
+ __u32 flags;
+#define VFIO_IOMMU_BIND_MSI (1 << 0)
+#define VFIO_IOMMU_UNBIND_MSI (1 << 1)
+ __u64 iova; /* MSI guest IOVA */
+ /* Fields below are used on BIND */
+ __u64 gpa; /* MSI guest physical address */
+ __u64 size; /* size of stage1 mapping (bytes) */
+};
+#define VFIO_IOMMU_SET_MSI_BINDING _IO(VFIO_TYPE, VFIO_BASE + 20)
+
/* -------- Additional API for SPAPR TCE (Server POWERPC) IOMMU -------- */

/*
--
2.26.2


2021-02-24 02:29:04

by kernel test robot

[permalink] [raw]
Subject: Re: [PATCH v12 03/13] vfio: VFIO_IOMMU_SET_MSI_BINDING

Hi Eric,

I love your patch! Yet something to improve:

[auto build test ERROR on v5.11]
[cannot apply to vfio/next next-20210223]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url: https://github.com/0day-ci/linux/commits/Eric-Auger/SMMUv3-Nested-Stage-Setup-VFIO-part/20210224-051641
base: f40ddce88593482919761f74910f42f4b84c004b
config: arm64-randconfig-r003-20210223 (attached as .config)
compiler: clang version 12.0.0 (https://github.com/llvm/llvm-project c9439ca36342fb6013187d0a69aef92736951476)
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# install arm64 cross compiling tool for clang build
# apt-get install binutils-aarch64-linux-gnu
# https://github.com/0day-ci/linux/commit/ec823a68d862693dc787422f168409996f43b10a
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Eric-Auger/SMMUv3-Nested-Stage-Setup-VFIO-part/20210224-051641
git checkout ec823a68d862693dc787422f168409996f43b10a
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross ARCH=arm64

If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <[email protected]>

All errors (new ones prefixed by >>):

In file included from drivers/vfio/vfio_iommu_type1.c:36:
In file included from include/linux/vfio.h:16:
include/uapi/linux/vfio.h:1198:34: error: field has incomplete type 'struct iommu_pasid_table_config'
struct iommu_pasid_table_config config; /* used on SET */
^
include/uapi/linux/vfio.h:1198:9: note: forward declaration of 'struct iommu_pasid_table_config'
struct iommu_pasid_table_config config; /* used on SET */
^
drivers/vfio/vfio_iommu_type1.c:2625:3: error: implicit declaration of function 'iommu_detach_pasid_table' [-Werror,-Wimplicit-function-declaration]
iommu_detach_pasid_table(d->domain);
^
drivers/vfio/vfio_iommu_type1.c:2625:3: note: did you mean 'vfio_detach_pasid_table'?
drivers/vfio/vfio_iommu_type1.c:2619:1: note: 'vfio_detach_pasid_table' declared here
vfio_detach_pasid_table(struct vfio_iommu *iommu)
^
drivers/vfio/vfio_iommu_type1.c:2639:9: error: implicit declaration of function 'iommu_uapi_attach_pasid_table' [-Werror,-Wimplicit-function-declaration]
ret = iommu_uapi_attach_pasid_table(d->domain, (void __user *)arg);
^
drivers/vfio/vfio_iommu_type1.c:2642:5: error: implicit declaration of function 'iommu_detach_pasid_table' [-Werror,-Wimplicit-function-declaration]
iommu_detach_pasid_table(d->domain);
^
>> drivers/vfio/vfio_iommu_type1.c:2668:9: error: implicit declaration of function 'iommu_bind_guest_msi' [-Werror,-Wimplicit-function-declaration]
ret = iommu_bind_guest_msi(d->domain, giova, gpa, size);
^
>> drivers/vfio/vfio_iommu_type1.c:2675:3: error: implicit declaration of function 'iommu_unbind_guest_msi' [-Werror,-Wimplicit-function-declaration]
iommu_unbind_guest_msi(d->domain, giova);
^
drivers/vfio/vfio_iommu_type1.c:2689:3: error: implicit declaration of function 'iommu_unbind_guest_msi' [-Werror,-Wimplicit-function-declaration]
iommu_unbind_guest_msi(d->domain, giova);
^
7 errors generated.


vim +/iommu_bind_guest_msi +2668 drivers/vfio/vfio_iommu_type1.c

2617
2618 static void
> 2619 vfio_detach_pasid_table(struct vfio_iommu *iommu)
2620 {
2621 struct vfio_domain *d;
2622
2623 mutex_lock(&iommu->lock);
2624 list_for_each_entry(d, &iommu->domain_list, next)
2625 iommu_detach_pasid_table(d->domain);
2626
2627 mutex_unlock(&iommu->lock);
2628 }
2629
2630 static int
2631 vfio_attach_pasid_table(struct vfio_iommu *iommu, unsigned long arg)
2632 {
2633 struct vfio_domain *d;
2634 int ret = 0;
2635
2636 mutex_lock(&iommu->lock);
2637
2638 list_for_each_entry(d, &iommu->domain_list, next) {
> 2639 ret = iommu_uapi_attach_pasid_table(d->domain, (void __user *)arg);
2640 if (ret) {
2641 list_for_each_entry_continue_reverse(d, &iommu->domain_list, next)
2642 iommu_detach_pasid_table(d->domain);
2643 break;
2644 }
2645 }
2646
2647 mutex_unlock(&iommu->lock);
2648 return ret;
2649 }
2650 static int vfio_cache_inv_fn(struct device *dev, void *data)
2651 {
2652 struct domain_capsule *dc = (struct domain_capsule *)data;
2653 unsigned long arg = *(unsigned long *)dc->data;
2654
2655 return iommu_uapi_cache_invalidate(dc->domain, dev, (void __user *)arg);
2656 }
2657
2658 static int
2659 vfio_bind_msi(struct vfio_iommu *iommu,
2660 dma_addr_t giova, phys_addr_t gpa, size_t size)
2661 {
2662 struct vfio_domain *d;
2663 int ret = 0;
2664
2665 mutex_lock(&iommu->lock);
2666
2667 list_for_each_entry(d, &iommu->domain_list, next) {
> 2668 ret = iommu_bind_guest_msi(d->domain, giova, gpa, size);
2669 if (ret)
2670 goto unwind;
2671 }
2672 goto unlock;
2673 unwind:
2674 list_for_each_entry_continue_reverse(d, &iommu->domain_list, next) {
> 2675 iommu_unbind_guest_msi(d->domain, giova);
2676 }
2677 unlock:
2678 mutex_unlock(&iommu->lock);
2679 return ret;
2680 }
2681

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/[email protected]


Attachments:
(No filename) (5.89 kB)
.config.gz (42.64 kB)
Download all attachments

2021-03-05 10:47:50

by Jean-Philippe Brucker

[permalink] [raw]
Subject: Re: [PATCH v12 03/13] vfio: VFIO_IOMMU_SET_MSI_BINDING

Hi,

On Tue, Feb 23, 2021 at 10:06:15PM +0100, Eric Auger wrote:
> This patch adds the VFIO_IOMMU_SET_MSI_BINDING ioctl which aim
> to (un)register the guest MSI binding to the host. This latter
> then can use those stage 1 bindings to build a nested stage
> binding targeting the physical MSIs.

Now that RMR is in the IORT spec, could it be used for the nested MSI
problem? For virtio-iommu tables I was planning to do it like this:

MSI is mapped at stage-2 with an arbitrary IPA->doorbell PA. We report
this IPA to userspace through iommu_groups/X/reserved_regions. No change
there. Then to the guest we report a reserved identity mapping at IPA
(using RMR, an equivalent DT binding, or probed RESV_MEM for
virtio-iommu). The guest creates that mapping at stage-1, and that's it.
Unless I overlooked something we'd only reuse existing infrastructure and
avoid the SET_MSI_BINDING interface.

Thanks,
Jean