KVM zPCI passthrough device logic will need a reference to the associated
kvm guest that has access to the device. Let's register a group notifier
for VFIO_GROUP_NOTIFY_SET_KVM to catch this information in order to create
an association between a kvm guest and the host zdev.
Signed-off-by: Matthew Rosato <[email protected]>
---
arch/s390/include/asm/kvm_pci.h | 2 ++
drivers/vfio/pci/vfio_pci_core.c | 2 ++
drivers/vfio/pci/vfio_pci_zdev.c | 46 ++++++++++++++++++++++++++++++++
include/linux/vfio_pci_core.h | 10 +++++++
4 files changed, 60 insertions(+)
diff --git a/arch/s390/include/asm/kvm_pci.h b/arch/s390/include/asm/kvm_pci.h
index fa90729a35cf..97a90b37c87d 100644
--- a/arch/s390/include/asm/kvm_pci.h
+++ b/arch/s390/include/asm/kvm_pci.h
@@ -17,6 +17,7 @@
#include <linux/kvm.h>
#include <linux/pci.h>
#include <linux/mutex.h>
+#include <linux/notifier.h>
#include <asm/pci_insn.h>
#include <asm/pci_dma.h>
@@ -33,6 +34,7 @@ struct kvm_zdev {
u64 rpcit_count;
struct kvm_zdev_ioat ioat;
struct zpci_fib fib;
+ struct notifier_block nb;
};
int kvm_s390_pci_dev_open(struct zpci_dev *zdev);
diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c
index f948e6cd2993..fc57d4d0abbe 100644
--- a/drivers/vfio/pci/vfio_pci_core.c
+++ b/drivers/vfio/pci/vfio_pci_core.c
@@ -452,6 +452,7 @@ void vfio_pci_core_close_device(struct vfio_device *core_vdev)
vfio_pci_vf_token_user_add(vdev, -1);
vfio_spapr_pci_eeh_release(vdev->pdev);
+ vfio_pci_zdev_release(vdev);
vfio_pci_core_disable(vdev);
mutex_lock(&vdev->igate);
@@ -470,6 +471,7 @@ EXPORT_SYMBOL_GPL(vfio_pci_core_close_device);
void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev)
{
vfio_pci_probe_mmaps(vdev);
+ vfio_pci_zdev_open(vdev);
vfio_spapr_pci_eeh_open(vdev->pdev);
vfio_pci_vf_token_user_add(vdev, 1);
}
diff --git a/drivers/vfio/pci/vfio_pci_zdev.c b/drivers/vfio/pci/vfio_pci_zdev.c
index ea4c0d2b0663..5c2bddc57b39 100644
--- a/drivers/vfio/pci/vfio_pci_zdev.c
+++ b/drivers/vfio/pci/vfio_pci_zdev.c
@@ -13,6 +13,7 @@
#include <linux/vfio_zdev.h>
#include <asm/pci_clp.h>
#include <asm/pci_io.h>
+#include <asm/kvm_pci.h>
#include <linux/vfio_pci_core.h>
@@ -136,3 +137,48 @@ int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev,
return ret;
}
+
+static int vfio_pci_zdev_group_notifier(struct notifier_block *nb,
+ unsigned long action, void *data)
+{
+ struct kvm_zdev *kzdev = container_of(nb, struct kvm_zdev, nb);
+
+ if (action == VFIO_GROUP_NOTIFY_SET_KVM) {
+ if (!data || !kzdev->zdev)
+ return NOTIFY_DONE;
+ kvm_s390_pci_attach_kvm(kzdev->zdev, data);
+ }
+
+ return NOTIFY_OK;
+}
+
+void vfio_pci_zdev_open(struct vfio_pci_core_device *vdev)
+{
+ unsigned long events = VFIO_GROUP_NOTIFY_SET_KVM;
+ struct zpci_dev *zdev = to_zpci(vdev->pdev);
+
+ if (!zdev)
+ return;
+
+ if (kvm_s390_pci_dev_open(zdev))
+ return;
+
+ zdev->kzdev->nb.notifier_call = vfio_pci_zdev_group_notifier;
+
+ if (vfio_register_notifier(vdev->vdev.dev, VFIO_GROUP_NOTIFY,
+ &events, &zdev->kzdev->nb))
+ kvm_s390_pci_dev_release(zdev);
+}
+
+void vfio_pci_zdev_release(struct vfio_pci_core_device *vdev)
+{
+ struct zpci_dev *zdev = to_zpci(vdev->pdev);
+
+ if (!zdev || !zdev->kzdev)
+ return;
+
+ vfio_unregister_notifier(vdev->vdev.dev, VFIO_GROUP_NOTIFY,
+ &zdev->kzdev->nb);
+
+ kvm_s390_pci_dev_release(zdev);
+}
diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h
index 5e2bca3b89db..05287f8ac855 100644
--- a/include/linux/vfio_pci_core.h
+++ b/include/linux/vfio_pci_core.h
@@ -198,12 +198,22 @@ static inline int vfio_pci_igd_init(struct vfio_pci_core_device *vdev)
#ifdef CONFIG_VFIO_PCI_ZDEV
extern int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev,
struct vfio_info_cap *caps);
+void vfio_pci_zdev_open(struct vfio_pci_core_device *vdev);
+void vfio_pci_zdev_release(struct vfio_pci_core_device *vdev);
#else
static inline int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev,
struct vfio_info_cap *caps)
{
return -ENODEV;
}
+
+static inline void vfio_pci_zdev_open(struct vfio_pci_core_device *vdev)
+{
+}
+
+static inline void vfio_pci_zdev_release(struct vfio_pci_core_device *vdev)
+{
+}
#endif
/* Will be exported for vfio pci drivers usage */
--
2.27.0
On 1/14/22 21:31, Matthew Rosato wrote:
> KVM zPCI passthrough device logic will need a reference to the associated
> kvm guest that has access to the device. Let's register a group notifier
> for VFIO_GROUP_NOTIFY_SET_KVM to catch this information in order to create
> an association between a kvm guest and the host zdev.
>
> Signed-off-by: Matthew Rosato <[email protected]>
> ---
> arch/s390/include/asm/kvm_pci.h | 2 ++
> drivers/vfio/pci/vfio_pci_core.c | 2 ++
> drivers/vfio/pci/vfio_pci_zdev.c | 46 ++++++++++++++++++++++++++++++++
> include/linux/vfio_pci_core.h | 10 +++++++
> 4 files changed, 60 insertions(+)
>
> diff --git a/arch/s390/include/asm/kvm_pci.h b/arch/s390/include/asm/kvm_pci.h
> index fa90729a35cf..97a90b37c87d 100644
> --- a/arch/s390/include/asm/kvm_pci.h
> +++ b/arch/s390/include/asm/kvm_pci.h
> @@ -17,6 +17,7 @@
> #include <linux/kvm.h>
> #include <linux/pci.h>
> #include <linux/mutex.h>
> +#include <linux/notifier.h>
> #include <asm/pci_insn.h>
> #include <asm/pci_dma.h>
>
> @@ -33,6 +34,7 @@ struct kvm_zdev {
> u64 rpcit_count;
> struct kvm_zdev_ioat ioat;
> struct zpci_fib fib;
> + struct notifier_block nb;
> };
>
> int kvm_s390_pci_dev_open(struct zpci_dev *zdev);
> diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c
> index f948e6cd2993..fc57d4d0abbe 100644
> --- a/drivers/vfio/pci/vfio_pci_core.c
> +++ b/drivers/vfio/pci/vfio_pci_core.c
> @@ -452,6 +452,7 @@ void vfio_pci_core_close_device(struct vfio_device *core_vdev)
>
> vfio_pci_vf_token_user_add(vdev, -1);
> vfio_spapr_pci_eeh_release(vdev->pdev);
> + vfio_pci_zdev_release(vdev);
> vfio_pci_core_disable(vdev);
>
> mutex_lock(&vdev->igate);
> @@ -470,6 +471,7 @@ EXPORT_SYMBOL_GPL(vfio_pci_core_close_device);
> void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev)
> {
> vfio_pci_probe_mmaps(vdev);
> + vfio_pci_zdev_open(vdev);
> vfio_spapr_pci_eeh_open(vdev->pdev);
> vfio_pci_vf_token_user_add(vdev, 1);
> }
> diff --git a/drivers/vfio/pci/vfio_pci_zdev.c b/drivers/vfio/pci/vfio_pci_zdev.c
> index ea4c0d2b0663..5c2bddc57b39 100644
> --- a/drivers/vfio/pci/vfio_pci_zdev.c
> +++ b/drivers/vfio/pci/vfio_pci_zdev.c
> @@ -13,6 +13,7 @@
> #include <linux/vfio_zdev.h>
> #include <asm/pci_clp.h>
> #include <asm/pci_io.h>
> +#include <asm/kvm_pci.h>
>
> #include <linux/vfio_pci_core.h>
>
> @@ -136,3 +137,48 @@ int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev,
>
> return ret;
> }
> +
> +static int vfio_pci_zdev_group_notifier(struct notifier_block *nb,
> + unsigned long action, void *data)
> +{
> + struct kvm_zdev *kzdev = container_of(nb, struct kvm_zdev, nb);
> +
> + if (action == VFIO_GROUP_NOTIFY_SET_KVM) {
> + if (!data || !kzdev->zdev)
> + return NOTIFY_DONE;
> + kvm_s390_pci_attach_kvm(kzdev->zdev, data);
Why not just set kzdev->kvm = data ?
alternatively, define kvm_s390_pci_attach_kvm() as an inline instead of
a global function.
otherwise LGTM
> + }
> +
> + return NOTIFY_OK;
> +}
> +
> +void vfio_pci_zdev_open(struct vfio_pci_core_device *vdev)
> +{
> + unsigned long events = VFIO_GROUP_NOTIFY_SET_KVM;
> + struct zpci_dev *zdev = to_zpci(vdev->pdev);
> +
> + if (!zdev)
> + return;
> +
> + if (kvm_s390_pci_dev_open(zdev))
> + return;
> +
> + zdev->kzdev->nb.notifier_call = vfio_pci_zdev_group_notifier;
> +
> + if (vfio_register_notifier(vdev->vdev.dev, VFIO_GROUP_NOTIFY,
> + &events, &zdev->kzdev->nb))
> + kvm_s390_pci_dev_release(zdev);
> +}
> +
> +void vfio_pci_zdev_release(struct vfio_pci_core_device *vdev)
> +{
> + struct zpci_dev *zdev = to_zpci(vdev->pdev);
> +
> + if (!zdev || !zdev->kzdev)
> + return;
> +
> + vfio_unregister_notifier(vdev->vdev.dev, VFIO_GROUP_NOTIFY,
> + &zdev->kzdev->nb);
> +
> + kvm_s390_pci_dev_release(zdev);
> +}
> diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h
> index 5e2bca3b89db..05287f8ac855 100644
> --- a/include/linux/vfio_pci_core.h
> +++ b/include/linux/vfio_pci_core.h
> @@ -198,12 +198,22 @@ static inline int vfio_pci_igd_init(struct vfio_pci_core_device *vdev)
> #ifdef CONFIG_VFIO_PCI_ZDEV
> extern int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev,
> struct vfio_info_cap *caps);
> +void vfio_pci_zdev_open(struct vfio_pci_core_device *vdev);
> +void vfio_pci_zdev_release(struct vfio_pci_core_device *vdev);
> #else
> static inline int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev,
> struct vfio_info_cap *caps)
> {
> return -ENODEV;
> }
> +
> +static inline void vfio_pci_zdev_open(struct vfio_pci_core_device *vdev)
> +{
> +}
> +
> +static inline void vfio_pci_zdev_release(struct vfio_pci_core_device *vdev)
> +{
> +}
> #endif
>
> /* Will be exported for vfio pci drivers usage */
>
--
Pierre Morel
IBM Lab Boeblingen
On 1/18/22 12:34 PM, Pierre Morel wrote:
>
>
> On 1/14/22 21:31, Matthew Rosato wrote:
>> KVM zPCI passthrough device logic will need a reference to the associated
>> kvm guest that has access to the device. Let's register a group notifier
>> for VFIO_GROUP_NOTIFY_SET_KVM to catch this information in order to
>> create
>> an association between a kvm guest and the host zdev.
>>
>> Signed-off-by: Matthew Rosato <[email protected]>
>> ---
>> arch/s390/include/asm/kvm_pci.h | 2 ++
>> drivers/vfio/pci/vfio_pci_core.c | 2 ++
>> drivers/vfio/pci/vfio_pci_zdev.c | 46 ++++++++++++++++++++++++++++++++
>> include/linux/vfio_pci_core.h | 10 +++++++
>> 4 files changed, 60 insertions(+)
>>
>> diff --git a/arch/s390/include/asm/kvm_pci.h
>> b/arch/s390/include/asm/kvm_pci.h
>> index fa90729a35cf..97a90b37c87d 100644
>> --- a/arch/s390/include/asm/kvm_pci.h
>> +++ b/arch/s390/include/asm/kvm_pci.h
>> @@ -17,6 +17,7 @@
>> #include <linux/kvm.h>
>> #include <linux/pci.h>
>> #include <linux/mutex.h>
>> +#include <linux/notifier.h>
>> #include <asm/pci_insn.h>
>> #include <asm/pci_dma.h>
>> @@ -33,6 +34,7 @@ struct kvm_zdev {
>> u64 rpcit_count;
>> struct kvm_zdev_ioat ioat;
>> struct zpci_fib fib;
>> + struct notifier_block nb;
>> };
>> int kvm_s390_pci_dev_open(struct zpci_dev *zdev);
>> diff --git a/drivers/vfio/pci/vfio_pci_core.c
>> b/drivers/vfio/pci/vfio_pci_core.c
>> index f948e6cd2993..fc57d4d0abbe 100644
>> --- a/drivers/vfio/pci/vfio_pci_core.c
>> +++ b/drivers/vfio/pci/vfio_pci_core.c
>> @@ -452,6 +452,7 @@ void vfio_pci_core_close_device(struct vfio_device
>> *core_vdev)
>> vfio_pci_vf_token_user_add(vdev, -1);
>> vfio_spapr_pci_eeh_release(vdev->pdev);
>> + vfio_pci_zdev_release(vdev);
>> vfio_pci_core_disable(vdev);
>> mutex_lock(&vdev->igate);
>> @@ -470,6 +471,7 @@ EXPORT_SYMBOL_GPL(vfio_pci_core_close_device);
>> void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev)
>> {
>> vfio_pci_probe_mmaps(vdev);
>> + vfio_pci_zdev_open(vdev);
>> vfio_spapr_pci_eeh_open(vdev->pdev);
>> vfio_pci_vf_token_user_add(vdev, 1);
>> }
>> diff --git a/drivers/vfio/pci/vfio_pci_zdev.c
>> b/drivers/vfio/pci/vfio_pci_zdev.c
>> index ea4c0d2b0663..5c2bddc57b39 100644
>> --- a/drivers/vfio/pci/vfio_pci_zdev.c
>> +++ b/drivers/vfio/pci/vfio_pci_zdev.c
>> @@ -13,6 +13,7 @@
>> #include <linux/vfio_zdev.h>
>> #include <asm/pci_clp.h>
>> #include <asm/pci_io.h>
>> +#include <asm/kvm_pci.h>
>> #include <linux/vfio_pci_core.h>
>> @@ -136,3 +137,48 @@ int vfio_pci_info_zdev_add_caps(struct
>> vfio_pci_core_device *vdev,
>> return ret;
>> }
>> +
>> +static int vfio_pci_zdev_group_notifier(struct notifier_block *nb,
>> + unsigned long action, void *data)
>> +{
>> + struct kvm_zdev *kzdev = container_of(nb, struct kvm_zdev, nb);
>> +
>> + if (action == VFIO_GROUP_NOTIFY_SET_KVM) {
>> + if (!data || !kzdev->zdev)
>> + return NOTIFY_DONE;
>> + kvm_s390_pci_attach_kvm(kzdev->zdev, data);
>
> Why not just set kzdev->kvm = data ?
>
> alternatively, define kvm_s390_pci_attach_kvm() as an inline instead of
> a global function.
>
> otherwise LGTM
At some point in the past this function did more than just set a
pointer... You are correct there's no need for this abstraction now,
let's just set kzdev->kvm = data directly here and drop the
kvm_s390_pci_attach_kvm function.