During vfio-pci open_device, pass the KVM associated with the vfio group
(if one exists). This is needed in order to pass a special indicator
(GISA) to firmware to allow zPCI interpretation facilities to be used
for only the specific KVM associated with the vfio-pci device. During
vfio-pci close_device, unregister the notifier.
Signed-off-by: Matthew Rosato <[email protected]>
---
arch/s390/include/asm/pci.h | 2 ++
drivers/vfio/pci/vfio_pci_core.c | 11 ++++++++++-
drivers/vfio/pci/vfio_pci_zdev.c | 27 +++++++++++++++++++++++++++
include/linux/vfio_pci_core.h | 12 ++++++++++++
4 files changed, 51 insertions(+), 1 deletion(-)
diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h
index 85eb0ef9d4c3..67fbce1ea0c9 100644
--- a/arch/s390/include/asm/pci.h
+++ b/arch/s390/include/asm/pci.h
@@ -5,6 +5,7 @@
#include <linux/pci.h>
#include <linux/mutex.h>
#include <linux/iommu.h>
+#include <linux/notifier.h>
#include <linux/pci_hotplug.h>
#include <asm-generic/pci.h>
#include <asm/pci_clp.h>
@@ -195,6 +196,7 @@ struct zpci_dev {
struct s390_domain *s390_domain; /* s390 IOMMU domain data */
struct kvm_zdev *kzdev;
struct mutex kzdev_lock;
+ struct notifier_block nb; /* vfio notifications */
};
static inline bool zdev_enabled(struct zpci_dev *zdev)
diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c
index a0d69ddaf90d..650494410682 100644
--- a/drivers/vfio/pci/vfio_pci_core.c
+++ b/drivers/vfio/pci/vfio_pci_core.c
@@ -316,10 +316,14 @@ int vfio_pci_core_enable(struct vfio_pci_core_device *vdev)
pci_write_config_word(pdev, PCI_COMMAND, cmd);
}
- ret = vfio_config_init(vdev);
+ ret = vfio_pci_zdev_open(vdev);
if (ret)
goto out_free_state;
+ ret = vfio_config_init(vdev);
+ if (ret)
+ goto out_free_zdev;
+
msix_pos = pdev->msix_cap;
if (msix_pos) {
u16 flags;
@@ -340,6 +344,8 @@ int vfio_pci_core_enable(struct vfio_pci_core_device *vdev)
return 0;
+out_free_zdev:
+ vfio_pci_zdev_release(vdev);
out_free_state:
kfree(vdev->pci_saved_state);
vdev->pci_saved_state = NULL;
@@ -418,6 +424,9 @@ void vfio_pci_core_disable(struct vfio_pci_core_device *vdev)
vdev->needs_reset = true;
+ if (vfio_pci_zdev_release(vdev))
+ pci_info(pdev, "%s: Couldn't restore zPCI state\n", __func__);
+
/*
* If we have saved state, restore it. If we can reset the device,
* even better. Resetting with current state seems better than
diff --git a/drivers/vfio/pci/vfio_pci_zdev.c b/drivers/vfio/pci/vfio_pci_zdev.c
index ea4c0d2b0663..d0df85c8b204 100644
--- a/drivers/vfio/pci/vfio_pci_zdev.c
+++ b/drivers/vfio/pci/vfio_pci_zdev.c
@@ -11,6 +11,7 @@
#include <linux/uaccess.h>
#include <linux/vfio.h>
#include <linux/vfio_zdev.h>
+#include <linux/kvm_host.h>
#include <asm/pci_clp.h>
#include <asm/pci_io.h>
@@ -136,3 +137,29 @@ int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev,
return ret;
}
+
+int vfio_pci_zdev_open(struct vfio_pci_core_device *vdev)
+{
+ struct zpci_dev *zdev = to_zpci(vdev->pdev);
+
+ if (!zdev)
+ return -ENODEV;
+
+ if (!vdev->vdev.kvm)
+ return 0;
+
+ return kvm_s390_pci_register_kvm(zdev, vdev->vdev.kvm);
+}
+
+int vfio_pci_zdev_release(struct vfio_pci_core_device *vdev)
+{
+ struct zpci_dev *zdev = to_zpci(vdev->pdev);
+
+ if (!zdev)
+ return -ENODEV;
+
+ if (!vdev->vdev.kvm)
+ return 0;
+
+ return kvm_s390_pci_unregister_kvm(zdev);
+}
diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h
index 63af2897939c..50ffab782435 100644
--- a/include/linux/vfio_pci_core.h
+++ b/include/linux/vfio_pci_core.h
@@ -209,12 +209,24 @@ static inline int vfio_pci_igd_init(struct vfio_pci_core_device *vdev)
#ifdef CONFIG_VFIO_PCI_ZDEV_KVM
extern int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev,
struct vfio_info_cap *caps);
+int vfio_pci_zdev_open(struct vfio_pci_core_device *vdev);
+int vfio_pci_zdev_release(struct vfio_pci_core_device *vdev);
#else
static inline int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev,
struct vfio_info_cap *caps)
{
return -ENODEV;
}
+
+static inline int vfio_pci_zdev_open(struct vfio_pci_core_device *vdev)
+{
+ return 0;
+}
+
+static inline int vfio_pci_zdev_release(struct vfio_pci_core_device *vdev)
+{
+ return 0;
+}
#endif
/* Will be exported for vfio pci drivers usage */
--
2.27.0
On Tue, May 24, 2022 at 02:59:02PM -0400, Matthew Rosato wrote:
> During vfio-pci open_device, pass the KVM associated with the vfio group
> (if one exists). This is needed in order to pass a special indicator
> (GISA) to firmware to allow zPCI interpretation facilities to be used
> for only the specific KVM associated with the vfio-pci device. During
> vfio-pci close_device, unregister the notifier.
>
> Signed-off-by: Matthew Rosato <[email protected]>
> arch/s390/include/asm/pci.h | 2 ++
> drivers/vfio/pci/vfio_pci_core.c | 11 ++++++++++-
> drivers/vfio/pci/vfio_pci_zdev.c | 27 +++++++++++++++++++++++++++
> include/linux/vfio_pci_core.h | 12 ++++++++++++
> 4 files changed, 51 insertions(+), 1 deletion(-)
>
> diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h
> index 85eb0ef9d4c3..67fbce1ea0c9 100644
> +++ b/arch/s390/include/asm/pci.h
> @@ -5,6 +5,7 @@
> #include <linux/pci.h>
> #include <linux/mutex.h>
> #include <linux/iommu.h>
> +#include <linux/notifier.h>
> #include <linux/pci_hotplug.h>
> #include <asm-generic/pci.h>
> #include <asm/pci_clp.h>
> @@ -195,6 +196,7 @@ struct zpci_dev {
> struct s390_domain *s390_domain; /* s390 IOMMU domain data */
> struct kvm_zdev *kzdev;
> struct mutex kzdev_lock;
> + struct notifier_block nb; /* vfio notifications */
This is obsolete now right? Same for the #include ?
> @@ -418,6 +424,9 @@ void vfio_pci_core_disable(struct vfio_pci_core_device *vdev)
>
> vdev->needs_reset = true;
>
> + if (vfio_pci_zdev_release(vdev))
> + pci_info(pdev, "%s: Couldn't restore zPCI state\n", __func__);
> +
> /*
> * If we have saved state, restore it. If we can reset the device,
> * even better. Resetting with current state seems better than
> diff --git a/drivers/vfio/pci/vfio_pci_zdev.c b/drivers/vfio/pci/vfio_pci_zdev.c
> index ea4c0d2b0663..d0df85c8b204 100644
> +++ b/drivers/vfio/pci/vfio_pci_zdev.c
> @@ -11,6 +11,7 @@
> #include <linux/uaccess.h>
> #include <linux/vfio.h>
> #include <linux/vfio_zdev.h>
> +#include <linux/kvm_host.h>
> #include <asm/pci_clp.h>
> #include <asm/pci_io.h>
>
> @@ -136,3 +137,29 @@ int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev,
>
> return ret;
> }
> +
> +int vfio_pci_zdev_open(struct vfio_pci_core_device *vdev)
> +{
> + struct zpci_dev *zdev = to_zpci(vdev->pdev);
> +
> + if (!zdev)
> + return -ENODEV;
> +
> + if (!vdev->vdev.kvm)
> + return 0;
> +
> + return kvm_s390_pci_register_kvm(zdev, vdev->vdev.kvm);
> +}
> +
> +int vfio_pci_zdev_release(struct vfio_pci_core_device *vdev)
> +{
> + struct zpci_dev *zdev = to_zpci(vdev->pdev);
Keeping these functions named open_device/close_device wouuld probably
be clearer
> + if (!zdev)
> + return -ENODEV;
> +
> + if (!vdev->vdev.kvm)
> + return 0;
> +
> + return kvm_s390_pci_unregister_kvm(zdev);
> +}
Again this cannot fail, you should make it return void, not ignore the
failure - or at least push the ignoring the failure down to the place
that is causing this.
Otherwise it looks fine to me, thanks
Jason
On 5/24/22 5:08 PM, Jason Gunthorpe wrote:
> On Tue, May 24, 2022 at 02:59:02PM -0400, Matthew Rosato wrote:
>> During vfio-pci open_device, pass the KVM associated with the vfio group
>> (if one exists). This is needed in order to pass a special indicator
>> (GISA) to firmware to allow zPCI interpretation facilities to be used
>> for only the specific KVM associated with the vfio-pci device. During
>> vfio-pci close_device, unregister the notifier.
>>
>> Signed-off-by: Matthew Rosato <[email protected]>
>> arch/s390/include/asm/pci.h | 2 ++
>> drivers/vfio/pci/vfio_pci_core.c | 11 ++++++++++-
>> drivers/vfio/pci/vfio_pci_zdev.c | 27 +++++++++++++++++++++++++++
>> include/linux/vfio_pci_core.h | 12 ++++++++++++
>> 4 files changed, 51 insertions(+), 1 deletion(-)
>>
>> diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h
>> index 85eb0ef9d4c3..67fbce1ea0c9 100644
>> +++ b/arch/s390/include/asm/pci.h
>> @@ -5,6 +5,7 @@
>> #include <linux/pci.h>
>> #include <linux/mutex.h>
>> #include <linux/iommu.h>
>> +#include <linux/notifier.h>
>> #include <linux/pci_hotplug.h>
>> #include <asm-generic/pci.h>
>> #include <asm/pci_clp.h>
>> @@ -195,6 +196,7 @@ struct zpci_dev {
>> struct s390_domain *s390_domain; /* s390 IOMMU domain data */
>> struct kvm_zdev *kzdev;
>> struct mutex kzdev_lock;
>> + struct notifier_block nb; /* vfio notifications */
>
> This is obsolete now right? Same for the #include ?
Of course, I forgot to remove them... Will do
>
>> @@ -418,6 +424,9 @@ void vfio_pci_core_disable(struct vfio_pci_core_device *vdev)
>>
>> vdev->needs_reset = true;
>>
>> + if (vfio_pci_zdev_release(vdev))
>> + pci_info(pdev, "%s: Couldn't restore zPCI state\n", __func__);
>> +
>> /*
>> * If we have saved state, restore it. If we can reset the device,
>> * even better. Resetting with current state seems better than
>> diff --git a/drivers/vfio/pci/vfio_pci_zdev.c b/drivers/vfio/pci/vfio_pci_zdev.c
>> index ea4c0d2b0663..d0df85c8b204 100644
>> +++ b/drivers/vfio/pci/vfio_pci_zdev.c
>> @@ -11,6 +11,7 @@
>> #include <linux/uaccess.h>
>> #include <linux/vfio.h>
>> #include <linux/vfio_zdev.h>
>> +#include <linux/kvm_host.h>
>> #include <asm/pci_clp.h>
>> #include <asm/pci_io.h>
>>
>> @@ -136,3 +137,29 @@ int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev,
>>
>> return ret;
>> }
>> +
>> +int vfio_pci_zdev_open(struct vfio_pci_core_device *vdev)
>> +{
>> + struct zpci_dev *zdev = to_zpci(vdev->pdev);
>> +
>> + if (!zdev)
>> + return -ENODEV;
>> +
>> + if (!vdev->vdev.kvm)
>> + return 0;
>> +
>> + return kvm_s390_pci_register_kvm(zdev, vdev->vdev.kvm);
>> +}
>> +
>> +int vfio_pci_zdev_release(struct vfio_pci_core_device *vdev)
>> +{
>> + struct zpci_dev *zdev = to_zpci(vdev->pdev);
>
> Keeping these functions named open_device/close_device wouuld probably
> be clearer
Agreed, will rename vfio_pci_zdev_{open,close}_device
>
>> + if (!zdev)
>> + return -ENODEV;
>> +
>> + if (!vdev->vdev.kvm)
>> + return 0;
>> +
>> + return kvm_s390_pci_unregister_kvm(zdev);
>> +}
>
> Again this cannot fail, you should make it return void, not ignore the
> failure - or at least push the ignoring the failure down to the place
> that is causing this.
I'll move that handling into kvm_s390_pci_unregister_kvm and make
vfio_pci_zdev_close_device return void / will remove the check for the
rc in vfio_pci_core_disable.
>
> Otherwise it looks fine to me, thanks
>
Thanks