LinuxLists.cc - [PATCH v9 00/22] guest dedicated crypto adapters

[permalink] [raw]

Subject: [PATCH v9 02/22] s390/zcrypt: Review inline assembler constraints.

From: Harald Freudenberger <[email protected]>

Reviewed and adapted the register use and asm constraints
of the C inline assembler functions in accordance to the
the AP instructions specifications.

Signed-off-by: Harald Freudenberger <[email protected]>
Signed-off-by: Christian Borntraeger <[email protected]>
Signed-off-by: Tony Krowiak <[email protected]>
---
drivers/s390/crypto/ap_asm.h | 40 +++++++++++++++++++++++-----------------
1 files changed, 23 insertions(+), 17 deletions(-)

diff --git a/drivers/s390/crypto/ap_asm.h b/drivers/s390/crypto/ap_asm.h
index b22d30a..e22ee12 100644
--- a/drivers/s390/crypto/ap_asm.h
+++ b/drivers/s390/crypto/ap_asm.h
@@ -20,14 +20,16 @@ static inline int ap_instructions_available(void)
{
register unsigned long reg0 asm ("0") = AP_MKQID(0, 0);
register unsigned long reg1 asm ("1") = -ENODEV;
- register unsigned long reg2 asm ("2") = 0UL;
+ register unsigned long reg2 asm ("2");

asm volatile(
" .long 0xb2af0000\n" /* PQAP(TAPQ) */
- "0: la %1,0\n"
+ "0: la %0,0\n"
"1:\n"
EX_TABLE(0b, 1b)
- : "+d" (reg0), "+d" (reg1), "+d" (reg2) : : "cc");
+ : "+d" (reg1), "=d" (reg2)
+ : "d" (reg0)
+ : "cc");
return reg1;
}

@@ -42,10 +44,12 @@ static inline struct ap_queue_status ap_tapq(ap_qid_t qid, unsigned long *info)
{
register unsigned long reg0 asm ("0") = qid;
register struct ap_queue_status reg1 asm ("1");
- register unsigned long reg2 asm ("2") = 0UL;
+ register unsigned long reg2 asm ("2");

asm volatile(".long 0xb2af0000" /* PQAP(TAPQ) */
- : "+d" (reg0), "=d" (reg1), "+d" (reg2) : : "cc");
+ : "=d" (reg1), "=d" (reg2)
+ : "d" (reg0)
+ : "cc");
if (info)
*info = reg2;
return reg1;
@@ -59,13 +63,14 @@ static inline struct ap_queue_status ap_tapq(ap_qid_t qid, unsigned long *info)
*/
static inline struct ap_queue_status ap_rapq(ap_qid_t qid)
{
- register unsigned long reg0 asm ("0") = qid | 0x01000000UL;
+ register unsigned long reg0 asm ("0") = qid | (1UL << 24);
register struct ap_queue_status reg1 asm ("1");
- register unsigned long reg2 asm ("2") = 0UL;

asm volatile(
".long 0xb2af0000" /* PQAP(RAPQ) */
- : "+d" (reg0), "=d" (reg1), "+d" (reg2) : : "cc");
+ : "=d" (reg1)
+ : "d" (reg0)
+ : "cc");
return reg1;
}

@@ -107,8 +112,8 @@ static inline struct ap_queue_status ap_aqic(ap_qid_t qid,

asm volatile(
".long 0xb2af0000" /* PQAP(AQIC) */
- : "+d" (reg0), "+d" (reg1_in), "=d" (reg1_out), "+d" (reg2)
- :
+ : "=d" (reg1_out)
+ : "d" (reg0), "d" (reg1_in), "d" (reg2)
: "cc");
return reg1_out;
}
@@ -120,17 +125,17 @@ static inline struct ap_queue_status ap_aqic(ap_qid_t qid,
*/
static inline int ap_qci(void *config)
{
- register unsigned long reg0 asm ("0") = 0x04000000UL;
+ register unsigned long reg0 asm ("0") = 4UL << 24;
register unsigned long reg1 asm ("1") = -EINVAL;
register void *reg2 asm ("2") = (void *) config;

asm volatile(
".long 0xb2af0000\n" /* PQAP(QCI) */
- "0: la %1,0\n"
+ "0: la %0,0\n"
"1:\n"
EX_TABLE(0b, 1b)
- : "+d" (reg0), "+d" (reg1), "+d" (reg2)
- :
+ : "+d" (reg1)
+ : "d" (reg0), "d" (reg2)
: "cc", "memory");

return reg1;
@@ -169,12 +174,13 @@ static inline struct ap_queue_status ap_qact(ap_qid_t qid, int ifbit,
| ((ifbit & 0x01) << 22);
register unsigned long reg1_in asm ("1") = apinfo->val;
register struct ap_queue_status reg1_out asm ("1");
- register unsigned long reg2 asm ("2") = 0;
+ register unsigned long reg2 asm ("2");

asm volatile(
".long 0xb2af0000" /* PQAP(QACT) */
- : "+d" (reg0), "+d" (reg1_in), "=d" (reg1_out), "+d" (reg2)
- : : "cc");
+ : "+d" (reg1_in), "=d" (reg1_out), "=d" (reg2)
+ : "d" (reg0)
+ : "cc");
apinfo->val = reg2;
return reg1_out;
}
--
1.7.1

2018-08-13 21:50:08

[permalink] [raw]

Subject: [PATCH v9 07/22] KVM: s390: refactor crypto initialization

From: Tony Krowiak <[email protected]>

This patch refactors the code that initializes and sets up the
crypto configuration for a guest. The following changes are
implemented via this patch:

1. Prior to the introduction of AP device virtualization, it
was not necessary to provide guest access to the CRYCB
unless the MSA extension 3 (MSAX3) facility was installed
on the host system. With the introduction of AP device
virtualization, the CRYCB must be made accessible to the
guest if the AP instructions are installed on the host
and are to be provided to the guest.

2. Introduces a flag indicating AP instructions executed on
the guest shall be interpreted by the firmware. It is
initialized to indicate AP instructions are to be
to be interpreted and is used to set the SIE bit for
each vcpu during vcpu setup.

Signed-off-by: Tony Krowiak <[email protected]>
Reviewed-by: Halil Pasic <[email protected]>
Acked-by: Christian Borntraeger <[email protected]>
Acked-by: Janosch Frank <[email protected]>
Tested-by: Michael Mueller <[email protected]>
Tested-by: Farhan Ali <[email protected]>
Signed-off-by: Christian Borntraeger <[email protected]>
---
arch/s390/include/asm/kvm_host.h | 2 +
arch/s390/include/uapi/asm/kvm.h | 1 +
arch/s390/kvm/kvm-s390.c | 81 +++++++++++++++++++-------------------
3 files changed, 44 insertions(+), 40 deletions(-)

diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
index af39561..4a739d4 100644
--- a/arch/s390/include/asm/kvm_host.h
+++ b/arch/s390/include/asm/kvm_host.h
@@ -187,6 +187,7 @@ struct kvm_s390_sie_block {
#define ECA_AIV 0x00200000
#define ECA_VX 0x00020000
#define ECA_PROTEXCI 0x00002000
+#define ECA_APIE 0x00000008
#define ECA_SII 0x00000001
__u32 eca; /* 0x004c */
#define ICPT_INST 0x04
@@ -256,6 +257,7 @@ struct kvm_s390_sie_block {
__u8 reservede4[4]; /* 0x00e4 */
__u64 tecmc; /* 0x00e8 */
__u8 reservedf0[12]; /* 0x00f0 */
+#define CRYCB_FORMAT_MASK 0x00000003
#define CRYCB_FORMAT1 0x00000001
#define CRYCB_FORMAT2 0x00000003
__u32 crycbd; /* 0x00fc */
diff --git a/arch/s390/include/uapi/asm/kvm.h b/arch/s390/include/uapi/asm/kvm.h
index 4cdaa55..a580dec 100644
--- a/arch/s390/include/uapi/asm/kvm.h
+++ b/arch/s390/include/uapi/asm/kvm.h
@@ -130,6 +130,7 @@ struct kvm_s390_vm_cpu_machine {
#define KVM_S390_VM_CPU_FEAT_PFMFI 11
#define KVM_S390_VM_CPU_FEAT_SIGPIF 12
#define KVM_S390_VM_CPU_FEAT_KSS 13
+#define KVM_S390_VM_CPU_FEAT_AP 14
struct kvm_s390_vm_cpu_feat {
__u64 feat[16];
};
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index 19f4f44..f7de123 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -40,6 +40,7 @@
#include <asm/sclp.h>
#include <asm/cpacf.h>
#include <asm/timex.h>
+#include <asm/ap.h>
#include "kvm-s390.h"
#include "gaccess.h"

@@ -1881,49 +1882,37 @@ long kvm_arch_vm_ioctl(struct file *filp,
return r;
}

-static int kvm_s390_query_ap_config(u8 *config)
-{
- u32 fcn_code = 0x04000000UL;
- u32 cc = 0;
-
- memset(config, 0, 128);
- asm volatile(
- "lgr 0,%1\n"
- "lgr 2,%2\n"
- ".long 0xb2af0000\n" /* PQAP(QCI) */
- "0: ipm %0\n"
- "srl %0,28\n"
- "1:\n"
- EX_TABLE(0b, 1b)
- : "+r" (cc)
- : "r" (fcn_code), "r" (config)
- : "cc", "0", "2", "memory"
- );
-
- return cc;
-}
-
static int kvm_s390_apxa_installed(void)
{
- u8 config[128];
- int cc;
+ struct ap_config_info info;

- if (test_facility(12)) {
- cc = kvm_s390_query_ap_config(config);
-
- if (cc)
- pr_err("PQAP(QCI) failed with cc=%d", cc);
- else
- return config[0] & 0x40;
+ if (ap_instructions_available()) {
+ if (ap_qci(&info) == 0)
+ return info.apxa;
}

return 0;
}

+/*
+ * The format of the crypto control block (CRYCB) is specified in the 3 low
+ * order bits of the CRYCB designation (CRYCBD) field as follows:
+ * Format 0: Neither the message security assist extension 3 (MSAX3) nor the
+ * AP extended addressing (APXA) facility are installed.
+ * Format 1: The APXA facility is not installed but the MSAX3 facility is.
+ * Format 2: Both the APXA and MSAX3 facilities are installed
+ */
static void kvm_s390_set_crycb_format(struct kvm *kvm)
{
kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;

+ /* Clear the CRYCB format bits - i.e., set format 0 by default */
+ kvm->arch.crypto.crycbd &= ~(CRYCB_FORMAT_MASK);
+
+ /* Check whether MSAX3 is installed */
+ if (!test_kvm_facility(kvm, 76))
+ return;
+
if (kvm_s390_apxa_installed())
kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
else
@@ -1941,12 +1930,12 @@ static u64 kvm_s390_get_initial_cpuid(void)

static void kvm_s390_crypto_init(struct kvm *kvm)
{
- if (!test_kvm_facility(kvm, 76))
- return;
-
kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
kvm_s390_set_crycb_format(kvm);

+ if (!test_kvm_facility(kvm, 76))
+ return;
+
/* Enable AES/DEA protected key functions by default */
kvm->arch.crypto.aes_kw = 1;
kvm->arch.crypto.dea_kw = 1;
@@ -2474,17 +2463,29 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)

static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
{
- if (!test_kvm_facility(vcpu->kvm, 76))
+ /*
+ * If neither the AP instructions nor the MSAX3 facility are installed
+ * on the host, then there is no need for a CRYCB in SIE because they
+ * will not be installed on the guest either.
+ */
+ if (!test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_AP) &&
+ !test_facility(76))
return;

+ vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);

- if (vcpu->kvm->arch.crypto.aes_kw)
- vcpu->arch.sie_block->ecb3 |= ECB3_AES;
- if (vcpu->kvm->arch.crypto.dea_kw)
- vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
+ vcpu->arch.sie_block->eca &= ~ECA_APIE;
+ if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_AP))
+ vcpu->arch.sie_block->eca |= ECA_APIE;

- vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
+ /* If MSAX3 is installed on the guest, set up protected key support */
+ if (test_kvm_facility(vcpu->kvm, 76)) {
+ if (vcpu->kvm->arch.crypto.aes_kw)
+ vcpu->arch.sie_block->ecb3 |= ECB3_AES;
+ if (vcpu->kvm->arch.crypto.dea_kw)
+ vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
+ }
}

void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
--
1.7.1

2018-08-13 21:50:32

[permalink] [raw]

Subject: [PATCH v9 08/22] s390: vfio-ap: base implementation of VFIO AP device driver

From: Tony Krowiak <[email protected]>

Introduces a new AP device driver. This device driver
is built on the VFIO mediated device framework. The framework
provides sysfs interfaces that facilitate passthrough
access by guests to devices installed on the linux host.

The VFIO AP device driver will serve two purposes:

1. Provide the interfaces to reserve AP devices for exclusive
use by KVM guests. This is accomplished by unbinding the
devices to be reserved for guest usage from the zcrypt
device driver and binding them to the VFIO AP device driver.

2. Implements the functions, callbacks and sysfs attribute
interfaces required to create one or more VFIO mediated
devices each of which will be used to configure the AP
matrix for a guest and serve as a file descriptor
for facilitating communication between QEMU and the
VFIO AP device driver.

When the VFIO AP device driver is initialized:

* It registers with the AP bus for control of type 10 (CEX4
and newer) AP queue devices. This limitation was imposed
due to:

1. A desire to keep the code as simple as possible;

2. Some older models are no longer supported by the kernel
and others are getting close to end of service.

The probe and remove callbacks will be provided to support
the binding/unbinding of AP queue devices to/from the VFIO
AP device driver.

* Creates a matrix device, /sys/devices/vfio_ap/matrix,
to serve as the parent of the mediated devices created, one
for each guest, and to hold the APQNs of the AP devices bound to
the VFIO AP device driver.

Signed-off-by: Tony Krowiak <[email protected]>
Reviewed-by: Halil Pasic <[email protected]>
Tested-by: Michael Mueller <[email protected]>
Tested-by: Farhan Ali <[email protected]>
Signed-off-by: Christian Borntraeger <[email protected]>
---
MAINTAINERS | 10 +++
arch/s390/Kconfig | 11 +++
drivers/s390/crypto/Makefile | 4 +
drivers/s390/crypto/vfio_ap_drv.c | 118 +++++++++++++++++++++++++++++++++
drivers/s390/crypto/vfio_ap_private.h | 30 ++++++++
5 files changed, 173 insertions(+), 0 deletions(-)
create mode 100644 drivers/s390/crypto/vfio_ap_drv.c
create mode 100644 drivers/s390/crypto/vfio_ap_private.h

diff --git a/MAINTAINERS b/MAINTAINERS
index 544cac8..e84c559 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -12418,6 +12418,16 @@ W: http://www.ibm.com/developerworks/linux/linux390/
S: Supported
F: drivers/s390/crypto/

+S390 VFIO AP DRIVER
+M: Tony Krowiak <[email protected]>
+M: Pierre Morel <[email protected]>
+M: Halil Pasic <[email protected]>
+L: [email protected]
+W: http://www.ibm.com/developerworks/linux/linux390/
+S: Supported
+F: drivers/s390/crypto/vfio_ap_drv.c
+F: drivers/s390/crypto/vfio_ap_private.h
+
S390 ZFCP DRIVER
M: Steffen Maier <[email protected]>
M: Benjamin Block <[email protected]>
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index 4fe5b2a..1097b28 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -778,6 +778,17 @@ config VFIO_CCW
To compile this driver as a module, choose M here: the
module will be called vfio_ccw.

+config VFIO_AP
+ def_tristate n
+ prompt "VFIO support for AP devices"
+ depends on ZCRYPT && VFIO_MDEV_DEVICE && KVM
+ help
+ This driver grants access to Adjunct Processor (AP) devices
+ via the VFIO mediated device interface.
+
+ To compile this driver as a module, choose M here: the module
+ will be called vfio_ap.
+
endmenu

menu "Dump support"
diff --git a/drivers/s390/crypto/Makefile b/drivers/s390/crypto/Makefile
index b59af54..48e466e 100644
--- a/drivers/s390/crypto/Makefile
+++ b/drivers/s390/crypto/Makefile
@@ -15,3 +15,7 @@ obj-$(CONFIG_ZCRYPT) += zcrypt_pcixcc.o zcrypt_cex2a.o zcrypt_cex4.o
# pkey kernel module
pkey-objs := pkey_api.o
obj-$(CONFIG_PKEY) += pkey.o
+
+# adjunct processor matrix
+vfio_ap-objs := vfio_ap_drv.o
+obj-$(CONFIG_VFIO_AP) += vfio_ap.o
diff --git a/drivers/s390/crypto/vfio_ap_drv.c b/drivers/s390/crypto/vfio_ap_drv.c
new file mode 100644
index 0000000..5069580
--- /dev/null
+++ b/drivers/s390/crypto/vfio_ap_drv.c
@@ -0,0 +1,118 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * VFIO based AP device driver
+ *
+ * Copyright IBM Corp. 2018
+ *
+ * Author(s): Tony Krowiak <[email protected]>
+ */
+
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include "vfio_ap_private.h"
+
+#define VFIO_AP_ROOT_NAME "vfio_ap"
+#define VFIO_AP_DEV_TYPE_NAME "ap_matrix"
+#define VFIO_AP_DEV_NAME "matrix"
+
+MODULE_AUTHOR("IBM Corporation");
+MODULE_DESCRIPTION("VFIO AP device driver, Copyright IBM Corp. 2018");
+MODULE_LICENSE("GPL v2");
+
+static struct ap_driver vfio_ap_drv;
+
+static struct device_type vfio_ap_dev_type = {
+ .name = VFIO_AP_DEV_TYPE_NAME,
+};
+
+struct ap_matrix_dev matrix_dev;
+
+/* Only type 10 adapters (CEX4 and later) are supported
+ * by the AP matrix device driver
+ */
+static struct ap_device_id ap_queue_ids[] = {
+ { .dev_type = AP_DEVICE_TYPE_CEX4,
+ .match_flags = AP_DEVICE_ID_MATCH_QUEUE_TYPE },
+ { .dev_type = AP_DEVICE_TYPE_CEX5,
+ .match_flags = AP_DEVICE_ID_MATCH_QUEUE_TYPE },
+ { .dev_type = AP_DEVICE_TYPE_CEX6,
+ .match_flags = AP_DEVICE_ID_MATCH_QUEUE_TYPE },
+ { /* end of sibling */ },
+};
+
+MODULE_DEVICE_TABLE(vfio_ap, ap_queue_ids);
+
+static int vfio_ap_queue_dev_probe(struct ap_device *apdev)
+{
+ return 0;
+}
+
+static void vfio_ap_queue_dev_remove(struct ap_device *apdev)
+{
+ /* Nothing to do yet */
+}
+
+static int vfio_ap_matrix_dev_init(void)
+{
+ int ret;
+ struct device *root_device;
+
+ root_device = root_device_register(VFIO_AP_ROOT_NAME);
+ if (IS_ERR(root_device)) {
+ ret = PTR_ERR(root_device);
+ return ret;
+ }
+
+ matrix_dev.device.type = &vfio_ap_dev_type;
+ dev_set_name(&matrix_dev.device, "%s", VFIO_AP_DEV_NAME);
+ matrix_dev.device.type = &vfio_ap_dev_type;
+ matrix_dev.device.parent = root_device;
+ matrix_dev.device.driver = &vfio_ap_drv.driver;
+
+ ret = device_register(&matrix_dev.device);
+ if (ret) {
+ root_device_unregister(root_device);
+ return ret;
+ }
+
+ return 0;
+}
+
+static void vfio_ap_matrix_dev_destroy(void)
+{
+ device_unregister(&matrix_dev.device);
+ root_device_unregister(matrix_dev.device.parent);
+}
+
+int __init vfio_ap_init(void)
+{
+ int ret;
+
+ ret = vfio_ap_matrix_dev_init();
+ if (ret)
+ return ret;
+
+ memset(&vfio_ap_drv, 0, sizeof(vfio_ap_drv));
+ vfio_ap_drv.probe = vfio_ap_queue_dev_probe;
+ vfio_ap_drv.remove = vfio_ap_queue_dev_remove;
+ vfio_ap_drv.ids = ap_queue_ids;
+
+ ret = ap_driver_register(&vfio_ap_drv, THIS_MODULE, VFIO_AP_DRV_NAME);
+ if (ret) {
+ vfio_ap_matrix_dev_destroy();
+ return ret;
+ }
+
+ return 0;
+}
+
+void __exit vfio_ap_exit(void)
+{
+ ap_driver_unregister(&vfio_ap_drv);
+ vfio_ap_matrix_dev_destroy();
+}
+
+module_init(vfio_ap_init);
+module_exit(vfio_ap_exit);
diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h
new file mode 100644
index 0000000..30c3e33
--- /dev/null
+++ b/drivers/s390/crypto/vfio_ap_private.h
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Private data and functions for adjunct processor VFIO matrix driver.
+ *
+ * Author(s): Tony Krowiak <[email protected]>
+ *
+ * Copyright IBM Corp. 2018
+ */
+
+#ifndef _VFIO_AP_PRIVATE_H_
+#define _VFIO_AP_PRIVATE_H_
+
+#include <linux/types.h>
+#include <linux/device.h>
+#include <linux/mdev.h>
+#include <linux/delay.h>
+#include <linux/mutex.h>
+
+#include "ap_bus.h"
+
+#define VFIO_AP_MODULE_NAME "vfio_ap"
+#define VFIO_AP_DRV_NAME "vfio_ap"
+
+struct ap_matrix_dev {
+ struct device device;
+};
+
+extern struct ap_matrix_dev matrix_dev;
+
+#endif /* _VFIO_AP_PRIVATE_H_ */
--
1.7.1

2018-08-13 21:50:50

[permalink] [raw]

Subject: [PATCH v9 09/22] s390: vfio-ap: register matrix device with VFIO mdev framework

From: Tony Krowiak <[email protected]>

Registers the matrix device created by the VFIO AP device
driver with the VFIO mediated device framework.
Registering the matrix device will create the sysfs
structures needed to create mediated matrix devices
each of which will be used to configure the AP matrix
for a guest and connect it to the VFIO AP device driver.

Registering the matrix device with the VFIO mediated device
framework will create the following sysfs structures:

/sys/devices/vfio_ap/matrix/
...... [mdev_supported_types]
......... [vfio_ap-passthrough]
............ create

To create a mediated device for the AP matrix device, write a UUID
to the create file:

uuidgen > create

A symbolic link to the mediated device's directory will be created in the
devices subdirectory named after the generated $uuid:

/sys/devices/vfio_ap/matrix/
...... [mdev_supported_types]
......... [vfio_ap-passthrough]
............ [devices]
............... [$uuid]

A symbolic link to the mediated device will also be created
in the vfio_ap matrix's directory:

/sys/devices/vfio_ap/matrix/[$uuid]

Signed-off-by: Tony Krowiak <[email protected]>
Reviewed-by: Halil Pasic <[email protected]>
Tested-by: Michael Mueller <[email protected]>
Tested-by: Farhan Ali <[email protected]>
Signed-off-by: Christian Borntraeger <[email protected]>
---
MAINTAINERS | 1 +
drivers/s390/crypto/Makefile | 2 +-
drivers/s390/crypto/vfio_ap_drv.c | 23 ++++++
drivers/s390/crypto/vfio_ap_ops.c | 124 +++++++++++++++++++++++++++++++++
drivers/s390/crypto/vfio_ap_private.h | 45 ++++++++++++
include/uapi/linux/vfio.h | 1 +
6 files changed, 195 insertions(+), 1 deletions(-)
create mode 100644 drivers/s390/crypto/vfio_ap_ops.c

diff --git a/MAINTAINERS b/MAINTAINERS
index e84c559..f60dd56 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -12427,6 +12427,7 @@ W: http://www.ibm.com/developerworks/linux/linux390/
S: Supported
F: drivers/s390/crypto/vfio_ap_drv.c
F: drivers/s390/crypto/vfio_ap_private.h
+F: drivers/s390/crypto/vfio_ap_ops.c

S390 ZFCP DRIVER
M: Steffen Maier <[email protected]>
diff --git a/drivers/s390/crypto/Makefile b/drivers/s390/crypto/Makefile
index 48e466e..8d36b05 100644
--- a/drivers/s390/crypto/Makefile
+++ b/drivers/s390/crypto/Makefile
@@ -17,5 +17,5 @@ pkey-objs := pkey_api.o
obj-$(CONFIG_PKEY) += pkey.o

# adjunct processor matrix
-vfio_ap-objs := vfio_ap_drv.o
+vfio_ap-objs := vfio_ap_drv.o vfio_ap_ops.o
obj-$(CONFIG_VFIO_AP) += vfio_ap.o
diff --git a/drivers/s390/crypto/vfio_ap_drv.c b/drivers/s390/crypto/vfio_ap_drv.c
index 5069580..fa04c5a 100644
--- a/drivers/s390/crypto/vfio_ap_drv.c
+++ b/drivers/s390/crypto/vfio_ap_drv.c
@@ -11,6 +11,7 @@
#include <linux/mod_devicetable.h>
#include <linux/slab.h>
#include <linux/string.h>
+#include <asm/zcrypt.h>
#include "vfio_ap_private.h"

#define VFIO_AP_ROOT_NAME "vfio_ap"
@@ -65,6 +66,19 @@ static int vfio_ap_matrix_dev_init(void)
return ret;
}

+ mutex_init(&matrix_dev.lock);
+ INIT_LIST_HEAD(&matrix_dev.mdev_list);
+
+ /* Test if PQAP(QCI) instruction is available */
+ if (test_facility(12)) {
+ ret = ap_qci(&matrix_dev.info);
+ if (ret) {
+ root_device_unregister(root_device);
+ return ret;
+ }
+ }
+
+ atomic_set(&matrix_dev.available_instances, MAX_ZDEV_ENTRIES_EXT);
matrix_dev.device.type = &vfio_ap_dev_type;
dev_set_name(&matrix_dev.device, "%s", VFIO_AP_DEV_NAME);
matrix_dev.device.type = &vfio_ap_dev_type;
@@ -105,11 +119,20 @@ int __init vfio_ap_init(void)
return ret;
}

+ ret = vfio_ap_mdev_register();
+ if (ret) {
+ ap_driver_unregister(&vfio_ap_drv);
+ vfio_ap_matrix_dev_destroy();
+
+ return ret;
+ }
+
return 0;
}

void __exit vfio_ap_exit(void)
{
+ vfio_ap_mdev_unregister();
ap_driver_unregister(&vfio_ap_drv);
vfio_ap_matrix_dev_destroy();
}
diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c
new file mode 100644
index 0000000..8018c2d
--- /dev/null
+++ b/drivers/s390/crypto/vfio_ap_ops.c
@@ -0,0 +1,124 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Adjunct processor matrix VFIO device driver callbacks.
+ *
+ * Copyright IBM Corp. 2018
+ *
+ * Author(s): Tony Krowiak <[email protected]>
+ * Halil Pasic <[email protected]>
+ * Pierre Morel <[email protected]>
+ */
+#include <linux/string.h>
+#include <linux/vfio.h>
+#include <linux/device.h>
+#include <linux/list.h>
+#include <linux/ctype.h>
+
+#include "vfio_ap_private.h"
+
+#define VFIO_AP_MDEV_TYPE_HWVIRT "passthrough"
+#define VFIO_AP_MDEV_NAME_HWVIRT "VFIO AP Passthrough Device"
+
+static void vfio_ap_matrix_init(struct ap_config_info *info,
+ struct ap_matrix *matrix)
+{
+ matrix->apm_max = info->apxa ? info->Na : 63;
+ matrix->aqm_max = info->apxa ? info->Nd : 15;
+ matrix->adm_max = info->apxa ? info->Nd : 15;
+}
+
+static int vfio_ap_mdev_create(struct kobject *kobj, struct mdev_device *mdev)
+{
+ struct ap_matrix_mdev *matrix_mdev;
+
+ matrix_mdev = kzalloc(sizeof(*matrix_mdev), GFP_KERNEL);
+ if (!matrix_mdev)
+ return -ENOMEM;
+
+ matrix_mdev->name = dev_name(mdev_dev(mdev));
+ vfio_ap_matrix_init(&matrix_dev.info, &matrix_mdev->matrix);
+ mdev_set_drvdata(mdev, matrix_mdev);
+
+ if (atomic_dec_if_positive(&matrix_dev.available_instances) < 0) {
+ kfree(matrix_mdev);
+ return -EPERM;
+ }
+
+ mutex_lock(&matrix_dev.lock);
+ list_add(&matrix_mdev->list, &matrix_dev.mdev_list);
+ mutex_unlock(&matrix_dev.lock);
+
+ return 0;
+}
+
+static int vfio_ap_mdev_remove(struct mdev_device *mdev)
+{
+ struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
+
+ mutex_lock(&matrix_dev.lock);
+ list_del(&matrix_mdev->list);
+ mutex_unlock(&matrix_dev.lock);
+ kfree(matrix_mdev);
+ mdev_set_drvdata(mdev, NULL);
+ atomic_inc(&matrix_dev.available_instances);
+
+ return 0;
+}
+
+static ssize_t name_show(struct kobject *kobj, struct device *dev, char *buf)
+{
+ return sprintf(buf, "%s\n", VFIO_AP_MDEV_NAME_HWVIRT);
+}
+
+MDEV_TYPE_ATTR_RO(name);
+
+static ssize_t available_instances_show(struct kobject *kobj,
+ struct device *dev, char *buf)
+{
+ return sprintf(buf, "%d\n",
+ atomic_read(&matrix_dev.available_instances));
+}
+
+MDEV_TYPE_ATTR_RO(available_instances);
+
+static ssize_t device_api_show(struct kobject *kobj, struct device *dev,
+ char *buf)
+{
+ return sprintf(buf, "%s\n", VFIO_DEVICE_API_AP_STRING);
+}
+
+MDEV_TYPE_ATTR_RO(device_api);
+
+static struct attribute *vfio_ap_mdev_type_attrs[] = {
+ &mdev_type_attr_name.attr,
+ &mdev_type_attr_device_api.attr,
+ &mdev_type_attr_available_instances.attr,
+ NULL,
+};
+
+static struct attribute_group vfio_ap_mdev_hwvirt_type_group = {
+ .name = VFIO_AP_MDEV_TYPE_HWVIRT,
+ .attrs = vfio_ap_mdev_type_attrs,
+};
+
+static struct attribute_group *vfio_ap_mdev_type_groups[] = {
+ &vfio_ap_mdev_hwvirt_type_group,
+ NULL,
+};
+
+static const struct mdev_parent_ops vfio_ap_matrix_ops = {
+ .owner = THIS_MODULE,
+ .supported_type_groups = vfio_ap_mdev_type_groups,
+ .create = vfio_ap_mdev_create,
+ .remove = vfio_ap_mdev_remove,
+};
+
+int vfio_ap_mdev_register(void)
+{
+ return mdev_register_device(&matrix_dev.device, &vfio_ap_matrix_ops);
+}
+
+void vfio_ap_mdev_unregister(void)
+{
+ mdev_unregister_device(&matrix_dev.device);
+}
diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h
index 30c3e33..aa0d195 100644
--- a/drivers/s390/crypto/vfio_ap_private.h
+++ b/drivers/s390/crypto/vfio_ap_private.h
@@ -5,6 +5,9 @@
* Author(s): Tony Krowiak <[email protected]>
*
* Copyright IBM Corp. 2018
+ *
+ * Author(s): Tony Krowiak <[email protected]>
+ * Halil Pasic <[email protected]>
*/

#ifndef _VFIO_AP_PRIVATE_H_
@@ -23,8 +26,50 @@

struct ap_matrix_dev {
struct device device;
+ atomic_t available_instances;
+ struct ap_config_info info;
+ struct list_head mdev_list;
+ struct mutex lock;
};

+/**
+ * Locking strategy: take the matrix_dev.lock mutex each time we fiddle
+ * with state managed by the vfio_ap driver (be it using the
+ * mdev_list or be it reading or writing the state of a single
+ * ap_matrix_mdev device). It's quite coarse but we don't expect
+ * much contention.
+ */
extern struct ap_matrix_dev matrix_dev;

+/**
+ * The AP matrix is comprised of three bit masks identifying the adapters,
+ * queues (domains) and control domains that belong to an AP matrix. The bits in
+ * each mask, from least significant to most significant bit, correspond to IDs
+ * 0 to 255. When a bit is set, the corresponding ID belongs to the matrix.
+ *
+ * @apm_max: max adapter number in @apm
+ * @apm identifies the AP adapters in the matrix
+ * @aqm_max: max domain number in @aqm
+ * @aqm identifies the AP queues (domains) in the matrix
+ * @adm_max: max domain number in @adm
+ * @adm identifies the AP control domains in the matrix
+ */
+struct ap_matrix {
+ unsigned long apm_max;
+ DECLARE_BITMAP(apm, 256);
+ unsigned long aqm_max;
+ DECLARE_BITMAP(aqm, 256);
+ unsigned long adm_max;
+ DECLARE_BITMAP(adm, 256);
+};
+
+struct ap_matrix_mdev {
+ const char *name;
+ struct list_head list;
+ struct ap_matrix matrix;
+};
+
+extern int vfio_ap_mdev_register(void);
+extern void vfio_ap_mdev_unregister(void);
+
#endif /* _VFIO_AP_PRIVATE_H_ */
diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
index 1aa7b82..bfbe2be 100644
--- a/include/uapi/linux/vfio.h
+++ b/include/uapi/linux/vfio.h
@@ -215,6 +215,7 @@ struct vfio_device_info {
#define VFIO_DEVICE_API_PLATFORM_STRING "vfio-platform"
#define VFIO_DEVICE_API_AMBA_STRING "vfio-amba"
#define VFIO_DEVICE_API_CCW_STRING "vfio-ccw"
+#define VFIO_DEVICE_API_AP_STRING "vfio-ap"

/**
* VFIO_DEVICE_GET_REGION_INFO - _IOWR(VFIO_TYPE, VFIO_BASE + 8,
--
1.7.1

2018-08-13 21:50:51

[permalink] [raw]

Subject: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

From: Tony Krowiak <[email protected]>

Introduces a new CPU model feature and two CPU model
facilities to support AP virtualization for KVM guests.

CPU model feature:

The KVM_S390_VM_CPU_FEAT_AP feature indicates that
AP instructions are available on the guest. This
feature will be enabled by the kernel only if the AP
instructions are installed on the linux host. This feature
must be specifically turned on for the KVM guest from
userspace to use the VFIO AP device driver for guest
access to AP devices.

CPU model facilities:

1. AP Query Configuration Information (QCI) facility is installed.

This is indicated by setting facilities bit 12 for
the guest. The kernel will not enable this facility
for the guest if it is not set on the host.

If this facility is not set for the KVM guest, then only
APQNs with an APQI less than 16 will be used by a Linux
guest regardless of the matrix configuration for the virtual
machine. This is a limitation of the Linux AP bus.

2. AP Facilities Test facility (APFT) is installed.

This is indicated by setting facilities bit 15 for
the guest. The kernel will not enable this facility for
the guest if it is not set on the host.

If this facility is not set for the KVM guest, then no
AP devices will be available to the guest regardless of
the guest's matrix configuration for the virtual
machine. This is a limitation of the Linux AP bus.

Signed-off-by: Tony Krowiak <[email protected]>
Reviewed-by: Christian Borntraeger <[email protected]>
Reviewed-by: Halil Pasic <[email protected]>
Tested-by: Michael Mueller <[email protected]>
Tested-by: Farhan Ali <[email protected]>
Signed-off-by: Christian Borntraeger <[email protected]>
---
arch/s390/kvm/kvm-s390.c | 5 +++++
arch/s390/tools/gen_facilities.c | 2 ++
2 files changed, 7 insertions(+), 0 deletions(-)

diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index 1e8cb67..d5e04d2 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -367,6 +367,11 @@ static void kvm_s390_cpu_feat_init(void)

if (MACHINE_HAS_ESOP)
allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
+
+ /* Check if AP instructions installed on host */
+ if (ap_instructions_available())
+ allow_cpu_feat(KVM_S390_VM_CPU_FEAT_AP);
+
/*
* We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
* 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
diff --git a/arch/s390/tools/gen_facilities.c b/arch/s390/tools/gen_facilities.c
index 90a8c9e..a52290b 100644
--- a/arch/s390/tools/gen_facilities.c
+++ b/arch/s390/tools/gen_facilities.c
@@ -106,6 +106,8 @@ struct facility_def {

.name = "FACILITIES_KVM_CPUMODEL",
.bits = (int[]){
+ 12, /* AP Query Configuration Information */
+ 15, /* AP Facilities Test */
-1 /* END */
}
},
--
1.7.1

2018-08-13 21:51:12

[permalink] [raw]

Subject: [PATCH v9 19/22] KVM: s390: Clear Crypto Control Block when using vSIE

From: Pierre Morel <[email protected]>

When we clear the Crypto Control Block (CRYCB) used by a guest
level 2, the vSIE shadow CRYCB for guest level 3 must be updated
before the guest uses it.

We achieve this by using the KVM_REQ_VSIE_RESTART synchronous
request for each vCPU belonging to the guest to force the reload
of the shadow CRYCB before rerunning the guest level 3.

Signed-off-by: Pierre Morel <[email protected]>
Signed-off-by: Tony Krowiak <[email protected]>
---
arch/s390/kvm/kvm-s390.c | 2 ++
1 files changed, 2 insertions(+), 0 deletions(-)

diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index 8d8a65a..1e8cb67 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -1929,6 +1929,8 @@ void kvm_arch_crypto_clear_masks(struct kvm *kvm)
memset(&kvm->arch.crypto.crycb->apcb1, 0,
sizeof(kvm->arch.crypto.crycb->apcb1));

+ /* recreate the shadow crycb for each vcpu */
+ kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
kvm_s390_vcpu_unblock_all(kvm);
mutex_unlock(&kvm->lock);
}
--
1.7.1

2018-08-13 21:51:15

[permalink] [raw]

Subject: [PATCH v9 20/22] KVM: s390: Handling of Cypto control block in VSIE

From: Pierre Morel <[email protected]>

Shadowing the crypto control block now supports APCB shadowing.

AP instruction interpretation for guest 3 through ECA.28 is shadowed when
guest 2 ECA.28 is set.

CRYCB is shadowed for APCB and wrapping keys.

CRYCB format 0 is now supported for both guests 2 and 3.

Shadow CRYCB always uses the guest 2 CRYCB format and it
follows that:

* Guest 3 CRYCB format 0 is supported with guest 2 CRYCB format 0,1 or 2
* Guest 3 CRYCB format 1 is supported with guest 2 CRYCB format 1 or 2
* Guest 3 CRYCB format 2 is supported with guest 2 CRYCB format 2

Signed-off-by: Pierre Morel <[email protected]>
Signed-off-by: Tony Krowiak <[email protected]>
Acked-by: Halil Pasic <[email protected]>
Tested-by: Michael Mueller <[email protected]>
Tested-by: Farhan Ali <[email protected]>
Signed-off-by: Christian Borntraeger <[email protected]>
---
arch/s390/kvm/vsie.c | 222 ++++++++++++++++++++++++++++++++++++++++++++------
1 files changed, 198 insertions(+), 24 deletions(-)

diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c
index aa30b48..e15240e 100644
--- a/arch/s390/kvm/vsie.c
+++ b/arch/s390/kvm/vsie.c
@@ -136,17 +136,8 @@ static int prepare_cpuflags(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
return 0;
}

-/*
- * Create a shadow copy of the crycb block and setup key wrapping, if
- * requested for guest 3 and enabled for guest 2.
- *
- * We only accept format-1 (no AP in g2), but convert it into format-2
- * There is nothing to do for format-0.
- *
- * Returns: - 0 if shadowed or nothing to do
- * - > 0 if control has to be given to guest 2
- */
-static int shadow_crycb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
+/* Copy keys into shadow crycb, is only called if MSA3 is available. */
+static int copy_key_masks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
{
struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
@@ -155,30 +146,17 @@ static int shadow_crycb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
unsigned long *b1, *b2;
u8 ecb3_flags;

- scb_s->crycbd = 0;
- if (!(crycbd_o & vcpu->arch.sie_block->crycbd & CRYCB_FORMAT1))
- return 0;
- /* format-1 is supported with message-security-assist extension 3 */
- if (!test_kvm_facility(vcpu->kvm, 76))
- return 0;
/* we may only allow it if enabled for guest 2 */
ecb3_flags = scb_o->ecb3 & vcpu->arch.sie_block->ecb3 &
(ECB3_AES | ECB3_DEA);
if (!ecb3_flags)
return 0;

- if ((crycb_addr & PAGE_MASK) != ((crycb_addr + 128) & PAGE_MASK))
- return set_validity_icpt(scb_s, 0x003CU);
- else if (!crycb_addr)
- return set_validity_icpt(scb_s, 0x0039U);
-
/* copy only the wrapping keys */
if (read_guest_real(vcpu, crycb_addr + 72, &vsie_page->crycb, 56))
return set_validity_icpt(scb_s, 0x0035U);

scb_s->ecb3 |= ecb3_flags;
- scb_s->crycbd = ((__u32)(__u64) &vsie_page->crycb) | CRYCB_FORMAT1 |
- CRYCB_FORMAT2;

/* xor both blocks in one run */
b1 = (unsigned long *) vsie_page->crycb.dea_wrapping_key_mask;
@@ -189,6 +167,202 @@ static int shadow_crycb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
return 0;
}

+/* Copy masks into apcb when g2 and g3 use format 1 */
+static int copy_apcb1(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
+{
+ struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
+ struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
+ const uint32_t crycbd_o = READ_ONCE(scb_o->crycbd);
+ const u32 crycb_o = crycbd_o & 0x7ffffff8U;
+ struct kvm_s390_crypto_cb *crycb_h = &vcpu->kvm->arch.sie_page2->crycb;
+ struct kvm_s390_crypto_cb *crycb_s = &vsie_page->crycb;
+ unsigned long *apcb_s = (unsigned long *) &crycb_s->apcb1;
+ unsigned long *apcb_h = (unsigned long *) &crycb_h->apcb1;
+ int i;
+ u32 src;
+
+ src = crycb_o + offsetof(struct kvm_s390_crypto_cb, apcb1);
+ if (read_guest_real(vcpu, src, apcb_s, sizeof(struct kvm_s390_apcb1)))
+ return set_validity_icpt(scb_s, 0x0035U);
+
+ for (i = 0; i < sizeof(struct kvm_s390_apcb1); i += sizeof(*apcb_s))
+ *apcb_s &= *apcb_h;
+
+ return 0;
+}
+
+/*
+ * Copy masks into apcb when g2 use format 1 and g3 use format 0
+ * In this case the shadow APCB uses format 1
+ */
+static int copy_apcb01(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
+{
+ struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
+ struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
+ const uint32_t crycbd_o = READ_ONCE(scb_o->crycbd);
+ const u32 crycb_o = crycbd_o & 0x7ffffff8U;
+ struct kvm_s390_apcb1 *apcb_h = &vcpu->kvm->arch.sie_page2->crycb.apcb1;
+ struct kvm_s390_apcb1 *apcb_s = &vsie_page->crycb.apcb1;
+ u32 src;
+
+ memset(apcb_s, 0, sizeof(*apcb_s));
+
+ src = crycb_o + offsetof(struct kvm_s390_crypto_cb, apcb0.apm[0]);
+ if (read_guest_real(vcpu, src, &apcb_s->apm[0], sizeof(__u64)))
+ return set_validity_icpt(scb_s, 0x0035U);
+
+ src = crycb_o + offsetof(struct kvm_s390_crypto_cb, apcb0.aqm[0]);
+ if (read_guest_real(vcpu, src, &apcb_s->aqm[0], sizeof(__u64)))
+ return set_validity_icpt(scb_s, 0x0035U);
+
+ src = crycb_o + offsetof(struct kvm_s390_crypto_cb, apcb0.adm[0]);
+ if (read_guest_real(vcpu, src, &apcb_s->adm[0], sizeof(__u64)))
+ return set_validity_icpt(scb_s, 0x0035U);
+
+ apcb_s->apm[0] &= apcb_h->apm[0];
+ apcb_s->aqm[0] &= apcb_h->aqm[0];
+ apcb_s->adm[0] &= apcb_h->adm[0];
+
+ return 0;
+}
+
+/* Copy masks into apcb when g2 and g3 use format 0 */
+static int copy_apcb0(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
+{
+ struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
+ struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
+ const uint32_t crycbd_o = READ_ONCE(scb_o->crycbd);
+ const u32 crycb_o = crycbd_o & 0x7ffffff8U;
+ struct kvm_s390_apcb0 *apcb_h = &vcpu->kvm->arch.sie_page2->crycb.apcb0;
+ struct kvm_s390_apcb0 *apcb_s = &vsie_page->crycb.apcb0;
+ u32 src;
+
+ src = crycb_o + offsetof(struct kvm_s390_crypto_cb, apcb0.apm[0]);
+ if (read_guest_real(vcpu, src, &apcb_s->apm[0], sizeof(__u64)))
+ return set_validity_icpt(scb_s, 0x0035U);
+
+ src = crycb_o + offsetof(struct kvm_s390_crypto_cb, apcb0.aqm[0]);
+ if (read_guest_real(vcpu, src, &apcb_s->aqm[0], sizeof(__u64)))
+ return set_validity_icpt(scb_s, 0x0035U);
+
+ src = crycb_o + offsetof(struct kvm_s390_crypto_cb, apcb0.adm[0]);
+ if (read_guest_real(vcpu, src, &apcb_s->adm[0], sizeof(__u64)))
+ return set_validity_icpt(scb_s, 0x0035U);
+
+ apcb_s->apm[0] &= apcb_h->apm[0];
+ apcb_s->aqm[0] &= apcb_h->aqm[0];
+ apcb_s->adm[0] &= apcb_h->adm[0];
+
+ return 0;
+}
+
+/* Shadowing APCB depends on G2 and G3 CRYCB format */
+static int copy_apcb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page,
+ int g2_fmt, int g3_fmt)
+{
+ struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
+ int ret = 0;
+
+ switch (g2_fmt) {
+ case CRYCB_FORMAT0:
+ switch (g3_fmt) {
+ case CRYCB_FORMAT0:
+ ret = copy_apcb0(vcpu, vsie_page);
+ break;
+ default:
+ return set_validity_icpt(scb_s, 0x0020U);
+ }
+ break;
+ case CRYCB_FORMAT1:
+ switch (g3_fmt) {
+ case CRYCB_FORMAT1:
+ case CRYCB_FORMAT0: /* Fall through to copy APCB */
+ ret = copy_apcb0(vcpu, vsie_page);
+ break;
+ default:
+ return set_validity_icpt(scb_s, 0x0020U);
+ }
+ break;
+ case CRYCB_FORMAT2:
+ switch (g3_fmt) {
+ case CRYCB_FORMAT0:
+ case CRYCB_FORMAT1:
+ ret = copy_apcb01(vcpu, vsie_page);
+ break;
+ case CRYCB_FORMAT2:
+ ret = copy_apcb1(vcpu, vsie_page);
+ break;
+ }
+ break;
+ default:
+ /*
+ * Guest 2 format is valid or we can not get to here.
+ */
+ break;
+ }
+
+ return ret;
+}
+
+/*
+ * Create a shadow copy of the crycb block.
+ * - Setup key wrapping, if requested for guest 3 and enabled for guest 2.
+ * - Shadow APCB if requested by guest 3 and enabled for guest 2 through
+ * ECA_APIE.
+ *
+ * We only accept format-1 (no AP in g2), but convert it into format-2
+ * There is nothing to do for format-0.
+ *
+ * Returns: - 0 if shadowed or nothing to do
+ * - > 0 if control has to be given to guest 2
+ * - < 0 if something went wrong on copy
+ */
+static int shadow_crycb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
+{
+ struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
+ struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
+ const uint32_t crycbd_o = READ_ONCE(scb_o->crycbd);
+ const u32 crycb_addr = crycbd_o & 0x7ffffff8U;
+ int g2_fmt = vcpu->arch.sie_block->crycbd & CRYCB_FORMAT_MASK;
+ int g3_fmt = crycbd_o & CRYCB_FORMAT_MASK;
+ int g2_apie, g2_msa3, g3_apie, g3_msa3;
+ int size, ret;
+
+ /* crycb should not cross a page boundary */
+ size = (g3_fmt == CRYCB_FORMAT2) ? 0x100 : 0x80;
+ if ((crycb_addr & PAGE_MASK) != ((crycb_addr + size) & PAGE_MASK))
+ return set_validity_icpt(scb_s, 0x003CU);
+
+ g2_apie = vcpu->arch.sie_block->eca & ECA_APIE;
+ g3_apie = scb_o->eca & g2_apie;
+
+ g2_msa3 = test_kvm_facility(vcpu->kvm, 76);
+ g3_msa3 = (g3_fmt != CRYCB_FORMAT0) & g2_msa3;
+
+ scb_s->crycbd = 0;
+ /* If no AP instructions and no keys we just set crycbd to 0 */
+ if (!(g3_apie || g3_msa3))
+ return 0;
+
+ if (!crycb_addr)
+ return set_validity_icpt(scb_s, 0x0039U);
+
+ if (g3_apie) {
+ ret = copy_apcb(vcpu, vsie_page, g2_fmt, g3_fmt);
+ if (ret)
+ return ret;
+ scb_s->eca |= g3_apie;
+ }
+
+ if (g3_msa3)
+ ret = copy_key_masks(vcpu, vsie_page);
+
+ if (!ret)
+ scb_s->crycbd = ((__u32)(__u64) &vsie_page->crycb) | g2_fmt;
+
+ return ret;
+}
+
/* shadow (round up/down) the ibc to avoid validity icpt */
static void prepare_ibc(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
{
--
1.7.1

2018-08-13 21:51:33

[permalink] [raw]

Subject: [PATCH v9 15/22] s390: vfio-ap: implement mediated device open callback

From: Tony Krowiak <[email protected]>

Implements the open callback on the mediated matrix device.
The function registers a group notifier to receive notification
of the VFIO_GROUP_NOTIFY_SET_KVM event. When notified,
the vfio_ap device driver will get access to the guest's
kvm structure. The open callback must ensure that only one
mediated device shall be opened per guest.

Signed-off-by: Tony Krowiak <[email protected]>
Acked-by: Halil Pasic <[email protected]>
Tested-by: Michael Mueller <[email protected]>
Tested-by: Farhan Ali <[email protected]>
Tested-by: Pierre Morel <[email protected]>
Acked-by: Pierre Morel <[email protected]>
Signed-off-by: Christian Borntraeger <[email protected]>
---
drivers/s390/crypto/vfio_ap_ops.c | 174 ++++++++++++++++++++++++++++++++-
drivers/s390/crypto/vfio_ap_private.h | 2 +
2 files changed, 175 insertions(+), 1 deletions(-)

diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c
index af3b55f..280bd17 100644
--- a/drivers/s390/crypto/vfio_ap_ops.c
+++ b/drivers/s390/crypto/vfio_ap_ops.c
@@ -13,6 +13,10 @@
#include <linux/device.h>
#include <linux/list.h>
#include <linux/ctype.h>
+#include <linux/bitops.h>
+#include <linux/kvm_host.h>
+#include <linux/module.h>
+#include <asm/kvm.h>

#include "vfio_ap_private.h"

@@ -55,6 +59,9 @@ static int vfio_ap_mdev_remove(struct mdev_device *mdev)
{
struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);

+ if (matrix_mdev->kvm)
+ return -EBUSY;
+
mutex_lock(&matrix_dev.lock);
list_del(&matrix_mdev->list);
mutex_unlock(&matrix_dev.lock);
@@ -291,6 +298,10 @@ static ssize_t assign_adapter_store(struct device *dev,
struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
unsigned long max_apid = matrix_mdev->matrix.apm_max;

+ /* If the guest is running, disallow assignment of adapter */
+ if (matrix_mdev->kvm)
+ return -EBUSY;
+
ret = kstrtoul(buf, 0, &apid);
if (ret)
return ret;
@@ -348,6 +359,10 @@ static ssize_t unassign_adapter_store(struct device *dev,
struct mdev_device *mdev = mdev_from_dev(dev);
struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);

+ /* If the guest is running, disallow un-assignment of adapter */
+ if (matrix_mdev->kvm)
+ return -EBUSY;
+
ret = kstrtoul(buf, 0, &apid);
if (ret)
return ret;
@@ -393,6 +408,10 @@ static ssize_t assign_domain_store(struct device *dev,
struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
unsigned long max_apqi = matrix_mdev->matrix.aqm_max;

+ /* If the guest is running, disallow assignment of domain */
+ if (matrix_mdev->kvm)
+ return -EBUSY;
+
ret = kstrtoul(buf, 0, &apqi);
if (ret)
return ret;
@@ -432,6 +451,10 @@ static ssize_t unassign_domain_store(struct device *dev,
struct mdev_device *mdev = mdev_from_dev(dev);
struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);

+ /* If the guest is running, disallow un-assignment of domain */
+ if (matrix_mdev->kvm)
+ return -EBUSY;
+
ret = kstrtoul(buf, 0, &apqi);
if (ret)
return ret;
@@ -470,6 +493,10 @@ static ssize_t assign_control_domain_store(struct device *dev,
struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
unsigned long maxid = matrix_mdev->matrix.adm_max;

+ /* If the guest is running, disallow assignment of control domain */
+ if (matrix_mdev->kvm)
+ return -EBUSY;
+
ret = kstrtoul(buf, 0, &id);
if (ret)
return ret;
@@ -514,6 +541,10 @@ static ssize_t unassign_control_domain_store(struct device *dev,
struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
unsigned long max_domid = matrix_mdev->matrix.adm_max;

+ /* If the guest is running, disallow un-assignment of control domain */
+ if (matrix_mdev->kvm)
+ return -EBUSY;
+
ret = kstrtoul(buf, 0, &domid);
if (ret)
return ret;
@@ -602,7 +633,6 @@ static ssize_t matrix_show(struct device *dev, struct device_attribute *attr,
}
DEVICE_ATTR_RO(matrix);

-
static struct attribute *vfio_ap_mdev_attrs[] = {
&dev_attr_assign_adapter.attr,
&dev_attr_unassign_adapter.attr,
@@ -624,12 +654,154 @@ static ssize_t matrix_show(struct device *dev, struct device_attribute *attr,
NULL
};

+/**
+ * Verify that the AP instructions are available on the guest. This is indicated
+ * via the KVM_S390_VM_CPU_FEAT_AP CPU model feature.
+ */
+static int kvm_ap_validate_crypto_setup(struct kvm *kvm)
+{
+ if (test_bit_inv(KVM_S390_VM_CPU_FEAT_AP, kvm->arch.cpu_feat))
+ return 0;
+
+ return -EOPNOTSUPP;
+}
+
+static void kvm_ap_merge_bitmasks(unsigned long *dst, unsigned long *mask1,
+ unsigned long *mask2, unsigned long nbits)
+{
+ int i;
+
+ for (i = 0; i < BITS_TO_LONGS(nbits); i++)
+ dst[i] = mask1[i] | mask2[i];
+}
+
+static void vfio_ap_mdev_copy_masks(struct ap_matrix_mdev *matrix_mdev)
+{
+ int nbytes;
+ unsigned long *apm, *aqm, *adm;
+ struct kvm_s390_crypto_cb *crycb = matrix_mdev->kvm->arch.crypto.crycb;
+
+ switch (matrix_mdev->kvm->arch.crypto.crycbd & CRYCB_FORMAT_MASK) {
+ case CRYCB_FORMAT2:
+ apm = (unsigned long *)crycb->apcb1.apm;
+ aqm = (unsigned long *)crycb->apcb1.aqm;
+ adm = (unsigned long *)crycb->apcb1.adm;
+ break;
+ case CRYCB_FORMAT1:
+ case CRYCB_FORMAT0:
+ default:
+ apm = (unsigned long *)crycb->apcb0.apm;
+ aqm = (unsigned long *)crycb->apcb0.aqm;
+ adm = (unsigned long *)crycb->apcb0.adm;
+ break;
+ }
+
+ nbytes = DIV_ROUND_UP(matrix_mdev->matrix.apm_max + 1, BITS_PER_BYTE);
+ memcpy(apm, matrix_mdev->matrix.apm, nbytes);
+ nbytes = DIV_ROUND_UP(matrix_mdev->matrix.aqm_max + 1, BITS_PER_BYTE);
+ memcpy(aqm, matrix_mdev->matrix.aqm, nbytes);
+ kvm_ap_merge_bitmasks(adm, aqm, adm, matrix_mdev->matrix.adm_max + 1);
+}
+
+/**
+ * vfio_ap_mdev_open_once
+ *
+ * @matrix_mdev: a mediated matrix device
+ *
+ * Return 0 if no other mediated matrix device has been opened for the
+ * KVM guest assigned to @matrix_mdev; otherwise, returns an error.
+ */
+static int vfio_ap_mdev_open_once(struct ap_matrix_mdev *matrix_mdev,
+ struct kvm *kvm)
+{
+ struct ap_matrix_mdev *m;
+
+ mutex_lock(&matrix_dev.lock);
+
+ list_for_each_entry(m, &matrix_dev.mdev_list, list) {
+ if ((m != matrix_mdev) && (m->kvm == matrix_mdev->kvm)) {
+ mutex_unlock(&matrix_dev.lock);
+ return -EPERM;
+ }
+ }
+
+ mutex_unlock(&matrix_dev.lock);
+
+ return 0;
+}
+
+static int vfio_ap_mdev_group_notifier(struct notifier_block *nb,
+ unsigned long action, void *data)
+{
+ int ret;
+ struct ap_matrix_mdev *matrix_mdev;
+
+ if (action != VFIO_GROUP_NOTIFY_SET_KVM)
+ return NOTIFY_OK;
+
+ matrix_mdev = container_of(nb, struct ap_matrix_mdev, group_notifier);
+
+ matrix_mdev->kvm = data;
+ if (data == NULL)
+ return NOTIFY_OK;
+
+ ret = vfio_ap_mdev_open_once(matrix_mdev, data);
+ if (ret)
+ return ret;
+
+ ret = kvm_ap_validate_crypto_setup(matrix_mdev->kvm);
+ if (ret)
+ return ret;
+
+ vfio_ap_mdev_copy_masks(matrix_mdev);
+
+ return NOTIFY_OK;
+}
+
+static int vfio_ap_mdev_open(struct mdev_device *mdev)
+{
+ struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
+ unsigned long events;
+ int ret;
+
+
+ if (!try_module_get(THIS_MODULE))
+ return -ENODEV;
+
+ matrix_mdev->group_notifier.notifier_call = vfio_ap_mdev_group_notifier;
+ events = VFIO_GROUP_NOTIFY_SET_KVM;
+
+ ret = vfio_register_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY,
+ &events, &matrix_mdev->group_notifier);
+ if (ret) {
+ module_put(THIS_MODULE);
+ return ret;
+ }
+
+ return 0;
+}
+
+static void vfio_ap_mdev_release(struct mdev_device *mdev)
+{
+ struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
+
+ if (matrix_mdev->kvm)
+ kvm_arch_crypto_clear_masks(matrix_mdev->kvm);
+
+ vfio_unregister_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY,
+ &matrix_mdev->group_notifier);
+ matrix_mdev->kvm = NULL;
+ module_put(THIS_MODULE);
+}
+
static const struct mdev_parent_ops vfio_ap_matrix_ops = {
.owner = THIS_MODULE,
.supported_type_groups = vfio_ap_mdev_type_groups,
.mdev_attr_groups = vfio_ap_mdev_attr_groups,
.create = vfio_ap_mdev_create,
.remove = vfio_ap_mdev_remove,
+ .open = vfio_ap_mdev_open,
+ .release = vfio_ap_mdev_release,
};

int vfio_ap_mdev_register(void)
diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h
index aa0d195..3e8534b 100644
--- a/drivers/s390/crypto/vfio_ap_private.h
+++ b/drivers/s390/crypto/vfio_ap_private.h
@@ -67,6 +67,8 @@ struct ap_matrix_mdev {
const char *name;
struct list_head list;
struct ap_matrix matrix;
+ struct notifier_block group_notifier;
+ struct kvm *kvm;
};

extern int vfio_ap_mdev_register(void);
--
1.7.1

2018-08-13 21:51:38

[permalink] [raw]

Subject: [PATCH v9 16/22] s390: vfio-ap: implement VFIO_DEVICE_GET_INFO ioctl

From: Tony Krowiak <[email protected]>

Introduces ioctl access to the VFIO AP Matrix device driver
by implementing the VFIO_DEVICE_GET_INFO ioctl. This ioctl
provides the VFIO AP Matrix device driver information to the
guest machine.

Signed-off-by: Tony Krowiak <[email protected]>
Reviewed-by: Pierre Morel <[email protected]>
Acked-by: Halil Pasic <[email protected]>
Tested-by: Michael Mueller <[email protected]>
Tested-by: Farhan Ali <[email protected]>
Tested-by: Pierre Morel <[email protected]>
Signed-off-by: Christian Borntraeger <[email protected]>
---
drivers/s390/crypto/vfio_ap_ops.c | 38 +++++++++++++++++++++++++++++++++++++
include/uapi/linux/vfio.h | 1 +
2 files changed, 39 insertions(+), 0 deletions(-)

diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c
index 280bd17..368b559 100644
--- a/drivers/s390/crypto/vfio_ap_ops.c
+++ b/drivers/s390/crypto/vfio_ap_ops.c
@@ -794,6 +794,43 @@ static void vfio_ap_mdev_release(struct mdev_device *mdev)
module_put(THIS_MODULE);
}

+static int vfio_ap_mdev_get_device_info(unsigned long arg)
+{
+ unsigned long minsz;
+ struct vfio_device_info info;
+
+ minsz = offsetofend(struct vfio_device_info, num_irqs);
+
+ if (copy_from_user(&info, (void __user *)arg, minsz))
+ return -EFAULT;
+
+ if (info.argsz < minsz)
+ return -EINVAL;
+
+ info.flags = VFIO_DEVICE_FLAGS_AP;
+ info.num_regions = 0;
+ info.num_irqs = 0;
+
+ return copy_to_user((void __user *)arg, &info, minsz);
+}
+
+static ssize_t vfio_ap_mdev_ioctl(struct mdev_device *mdev,
+ unsigned int cmd, unsigned long arg)
+{
+ int ret;
+
+ switch (cmd) {
+ case VFIO_DEVICE_GET_INFO:
+ ret = vfio_ap_mdev_get_device_info(arg);
+ break;
+ default:
+ ret = -EOPNOTSUPP;
+ break;
+ }
+
+ return ret;
+}
+
static const struct mdev_parent_ops vfio_ap_matrix_ops = {
.owner = THIS_MODULE,
.supported_type_groups = vfio_ap_mdev_type_groups,
@@ -802,6 +839,7 @@ static void vfio_ap_mdev_release(struct mdev_device *mdev)
.remove = vfio_ap_mdev_remove,
.open = vfio_ap_mdev_open,
.release = vfio_ap_mdev_release,
+ .ioctl = vfio_ap_mdev_ioctl,
};

int vfio_ap_mdev_register(void)
diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
index bfbe2be..f378b98 100644
--- a/include/uapi/linux/vfio.h
+++ b/include/uapi/linux/vfio.h
@@ -200,6 +200,7 @@ struct vfio_device_info {
#define VFIO_DEVICE_FLAGS_PLATFORM (1 << 2) /* vfio-platform device */
#define VFIO_DEVICE_FLAGS_AMBA (1 << 3) /* vfio-amba device */
#define VFIO_DEVICE_FLAGS_CCW (1 << 4) /* vfio-ccw device */
+#define VFIO_DEVICE_FLAGS_AP (1 << 5) /* vfio-ap device */
__u32 num_regions; /* Max region index + 1 */
__u32 num_irqs; /* Max IRQ index + 1 */
};
--
1.7.1

2018-08-13 21:51:44

[permalink] [raw]

Subject: [PATCH v9 18/22] s390: vfio-ap: implement VFIO_DEVICE_RESET ioctl

From: Tony Krowiak <[email protected]>

Implements the VFIO_DEVICE_RESET ioctl. This ioctl zeroizes
all of the AP queues assigned to the guest.

Signed-off-by: Tony Krowiak <[email protected]>
Reviewed-by: Halil Pasic <[email protected]>
Tested-by: Michael Mueller <[email protected]>
Tested-by: Farhan Ali <[email protected]>
Reviewed-by: Pierre Morel <[email protected]>
Tested-by: Pierre Morel <[email protected]>
Signed-off-by: Christian Borntraeger <[email protected]>
---
drivers/s390/crypto/vfio_ap_ops.c | 5 ++++-
1 files changed, 4 insertions(+), 1 deletions(-)

diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c
index cc90323..d4a065b 100644
--- a/drivers/s390/crypto/vfio_ap_ops.c
+++ b/drivers/s390/crypto/vfio_ap_ops.c
@@ -832,7 +832,7 @@ static int vfio_ap_mdev_get_device_info(unsigned long arg)
if (info.argsz < minsz)
return -EINVAL;

- info.flags = VFIO_DEVICE_FLAGS_AP;
+ info.flags = VFIO_DEVICE_FLAGS_AP | VFIO_DEVICE_FLAGS_RESET;
info.num_regions = 0;
info.num_irqs = 0;

@@ -848,6 +848,9 @@ static ssize_t vfio_ap_mdev_ioctl(struct mdev_device *mdev,
case VFIO_DEVICE_GET_INFO:
ret = vfio_ap_mdev_get_device_info(arg);
break;
+ case VFIO_DEVICE_RESET:
+ ret = vfio_ap_mdev_reset_queues(mdev, true);
+ break;
default:
ret = -EOPNOTSUPP;
break;
--
1.7.1

2018-08-13 21:51:51

[permalink] [raw]

Subject: [PATCH v9 14/22] KVM: s390: interfaces to clear CRYCB masks

From: Tony Krowiak <[email protected]>

Introduces two new KVM interface to clear the APM, AQM and ADM masks in
the guest's CRYCB. The VCPUs are taken out of SIE to ensure the VCPUs do
not get out of sync.

Signed-off-by: Tony Krowiak <[email protected]>
Acked-by: Halil Pasic <[email protected]>
Tested-by: Michael Mueller <[email protected]>
Tested-by: Farhan Ali <[email protected]>
Tested-by: Pierre Morel <[email protected]>
Signed-off-by: Christian Borntraeger <[email protected]>
---
arch/s390/include/asm/kvm_host.h | 3 +++
arch/s390/kvm/kvm-s390.c | 15 +++++++++++++++
2 files changed, 18 insertions(+), 0 deletions(-)

diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
index 4a739d4..07e58d8 100644
--- a/arch/s390/include/asm/kvm_host.h
+++ b/arch/s390/include/asm/kvm_host.h
@@ -258,6 +258,7 @@ struct kvm_s390_sie_block {
__u64 tecmc; /* 0x00e8 */
__u8 reservedf0[12]; /* 0x00f0 */
#define CRYCB_FORMAT_MASK 0x00000003
+#define CRYCB_FORMAT0 0x00000000
#define CRYCB_FORMAT1 0x00000001
#define CRYCB_FORMAT2 0x00000003
__u32 crycbd; /* 0x00fc */
@@ -861,6 +862,8 @@ void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
struct kvm_async_pf *work);

+void kvm_arch_crypto_clear_masks(struct kvm *kvm);
+
extern int sie64a(struct kvm_s390_sie_block *, u64 *);
extern char sie_exit;

diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index f7de123..8d8a65a 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -1919,6 +1919,21 @@ static void kvm_s390_set_crycb_format(struct kvm *kvm)
kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
}

+void kvm_arch_crypto_clear_masks(struct kvm *kvm)
+{
+ mutex_lock(&kvm->lock);
+ kvm_s390_vcpu_block_all(kvm);
+
+ memset(&kvm->arch.crypto.crycb->apcb0, 0,
+ sizeof(kvm->arch.crypto.crycb->apcb0));
+ memset(&kvm->arch.crypto.crycb->apcb1, 0,
+ sizeof(kvm->arch.crypto.crycb->apcb1));
+
+ kvm_s390_vcpu_unblock_all(kvm);
+ mutex_unlock(&kvm->lock);
+}
+EXPORT_SYMBOL(kvm_arch_crypto_clear_masks);
+
static u64 kvm_s390_get_initial_cpuid(void)
{
struct cpuid cpuid;
--
1.7.1

2018-08-13 21:52:12

[permalink] [raw]

Subject: [PATCH v9 11/22] s390: vfio-ap: sysfs interfaces to configure domains

From: Tony Krowiak <[email protected]>

Introduces two new sysfs attributes for the VFIO mediated
matrix device for assigning AP domains to and unassigning
AP domains from a mediated matrix device. The IDs of the
AP domains assigned to the mediated matrix device will be
stored in an AP queue mask (AQM).

The bits in the AQM, from most significant to least
significant bit, correspond to AP queue index (APQI) 0 to
255 (note that an APQI is synonymous with with a domain ID).
On some systems, the maximum allowable domain number may be
less than 255 - depending upon the host's AP configuration -
and assignment may be rejected if the input domain ID exceeds
the limit.

When a domain is assigned, the bit corresponding to the APQI
will be set in the AQM. Likewise, when a domain is unassigned,
the bit corresponding to the APQI will be cleared from the AQM.

In order to successfully assign a domain, the APQNs derived from
the domain ID being assigned and the adapter numbers of all
adapters previously assigned:

1. Must be bound to the vfio_ap device driver.

2. Must not be assigned to any other mediated matrix device.

If there are no adapters assigned to the mdev, then there must
be an AP queue bound to the vfio_ap device driver with an
APQN containing the domain ID (i.e., APQI), otherwise all
adapters subsequently assigned will fail because there will be no
AP queues bound with an APQN containing the APQI.

Assigning or un-assigning an AP domain will also be rejected if
a guest using the mediated matrix device is running.

The relevant sysfs structures are:

/sys/devices/vfio_ap/matrix/
...... [mdev_supported_types]
......... [vfio_ap-passthrough]
............ [devices]
...............[$uuid]
.................. assign_domain
.................. unassign_domain

To assign a domain to the $uuid mediated matrix device,
write the domain's ID to the assign_domain file. To
unassign a domain, write the domain's ID to the
unassign_domain file. The ID is specified using
conventional semantics: If it begins with 0x, the number
will be parsed as a hexadecimal (case insensitive) number;
if it begins with 0, it will be parsed as an octal number;
otherwise, it will be parsed as a decimal number.

For example, to assign domain 173 (0xad) to the mediated matrix
device $uuid:

echo 173 > assign_domain

or

echo 0255 > assign_domain

or

echo 0xad > assign_domain

To unassign domain 173 (0xad):

echo 173 > unassign_domain

or

echo 0255 > unassign_domain

or

echo 0xad > unassign_domain

Signed-off-by: Tony Krowiak <[email protected]>
Reviewed-by: Halil Pasic <[email protected]>
Tested-by: Michael Mueller <[email protected]>
Tested-by: Farhan Ali <[email protected]>
Tested-by: Pierre Morel <[email protected]>
Signed-off-by: Christian Borntraeger <[email protected]>
---
drivers/s390/crypto/vfio_ap_ops.c | 88 ++++++++++++++++++++++++++++++++++++-
1 files changed, 87 insertions(+), 1 deletions(-)

diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c
index dfb434c..c4d7504 100644
--- a/drivers/s390/crypto/vfio_ap_ops.c
+++ b/drivers/s390/crypto/vfio_ap_ops.c
@@ -363,10 +363,96 @@ static ssize_t unassign_adapter_store(struct device *dev,
}
DEVICE_ATTR_WO(unassign_adapter);

+static int
+vfio_ap_mdev_verify_queues_reserved_for_apqi(struct ap_matrix_mdev *matrix_mdev,
+ unsigned long apqi)
+{
+ int ret;
+ unsigned long apid;
+ unsigned long nbits = matrix_mdev->matrix.apm_max + 1;
+
+ if (find_first_bit_inv(matrix_mdev->matrix.apm, nbits) >= nbits)
+ return vfio_ap_verify_queue_reserved(NULL, &apqi);
+
+ for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, nbits) {
+ ret = vfio_ap_verify_queue_reserved(&apid, &apqi);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
+static ssize_t assign_domain_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ int ret;
+ unsigned long apqi;
+ struct mdev_device *mdev = mdev_from_dev(dev);
+ struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
+ unsigned long max_apqi = matrix_mdev->matrix.aqm_max;
+
+ ret = kstrtoul(buf, 0, &apqi);
+ if (ret)
+ return ret;
+ if (apqi > max_apqi)
+ return -EINVAL;
+
+ mutex_lock(&matrix_dev.lock);
+
+ ret = vfio_ap_mdev_verify_queues_reserved_for_apqi(matrix_mdev, apqi);
+ if (ret)
+ goto done;
+
+ set_bit_inv(apqi, matrix_mdev->matrix.aqm);
+
+ ret = vfio_ap_mdev_verify_no_sharing(matrix_mdev);
+ if (ret)
+ goto share_err;
+
+ ret = count;
+ goto done;
+
+share_err:
+ clear_bit_inv(apqi, matrix_mdev->matrix.aqm);
+done:
+ mutex_unlock(&matrix_dev.lock);
+
+ return ret;
+}
+DEVICE_ATTR_WO(assign_domain);
+
+static ssize_t unassign_domain_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ int ret;
+ unsigned long apqi;
+ struct mdev_device *mdev = mdev_from_dev(dev);
+ struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
+
+ ret = kstrtoul(buf, 0, &apqi);
+ if (ret)
+ return ret;
+
+ if (apqi > matrix_mdev->matrix.aqm_max)
+ return -EINVAL;
+
+ mutex_lock(&matrix_dev.lock);
+ clear_bit_inv((unsigned long)apqi, matrix_mdev->matrix.aqm);
+ mutex_unlock(&matrix_dev.lock);
+
+ return count;
+}
+DEVICE_ATTR_WO(unassign_domain);
+
static struct attribute *vfio_ap_mdev_attrs[] = {
&dev_attr_assign_adapter.attr,
&dev_attr_unassign_adapter.attr,
- NULL
+ &dev_attr_assign_domain.attr,
+ &dev_attr_unassign_domain.attr,
+ NULL,
};

static struct attribute_group vfio_ap_mdev_attr_group = {
--
1.7.1

2018-08-13 21:52:23

[permalink] [raw]

Subject: [PATCH v9 10/22] s390: vfio-ap: sysfs interfaces to configure adapters

From: Tony Krowiak <[email protected]>

Introduces two new sysfs attributes for the VFIO mediated
matrix device for assigning AP adapters to and unassigning
AP adapters from a mediated matrix device. The IDs of the
AP adapters assigned to the mediated matrix device will be
stored in an AP mask (APM).

The bits in the APM, from most significant to least significant
bit, correspond to AP adapter IDs (APID) 0 to 255. On
some systems, the maximum allowable adapter number may be less
than 255 - depending upon the host's AP configuration - and
assignment may be rejected if the input adapter ID exceeds the
limit.

When an adapter is assigned, the bit corresponding to the APID
will be set in the APM. Likewise, when an adapter is
unassigned, the bit corresponding to the APID will be cleared
from the APM.

In order to successfully assign an adapter, the APQNs derived from
the adapter ID being assigned and the queue indexes of all domains
previously assigned:

1. Must be bound to the vfio_ap device driver.

2. Must not be assigned to any other mediated matrix device

If there are no domains assigned to the mdev, then there must
be an AP queue bound to the vfio_ap device driver with an
APQN containing the APID, otherwise all domains
subsequently assigned will fail because there will be no
AP queues bound with an APQN containing the adapter ID.

Assigning or un-assigning an AP adapter will be rejected if
a guest using the mediated matrix device is running.

The relevant sysfs structures are:

/sys/devices/vfio_ap/matrix/
...... [mdev_supported_types]
......... [vfio_ap-passthrough]
............ [devices]
...............[$uuid]
.................. assign_adapter
.................. unassign_adapter

To assign an adapter to the $uuid mediated matrix device's APM,
write the APID to the assign_adapter file. To unassign an adapter,
write the APID to the unassign_adapter file. The APID is specified
using conventional semantics: If it begins with 0x the number will
be parsed as a hexadecimal number; if it begins with a 0 the number
will be parsed as an octal number; otherwise, it will be parsed as a
decimal number.

For example, to assign adapter 173 (0xad) to the mediated matrix
device $uuid:

echo 173 > assign_adapter

or

echo 0xad > assign_adapter

or

echo 0255 > assign_adapter

To unassign adapter 173 (0xad):

echo 173 > unassign_adapter

or

echo 0xad > unassign_adapter

or

echo 0255 > unassign_adapter

Signed-off-by: Tony Krowiak <[email protected]>
Reviewed-by: Halil Pasic <[email protected]>
Tested-by: Michael Mueller <[email protected]>
Tested-by: Farhan Ali <[email protected]>
Tested-by: Pierre Morel <[email protected]>
Signed-off-by: Christian Borntraeger <[email protected]>
---
drivers/s390/crypto/vfio_ap_ops.c | 273 +++++++++++++++++++++++++++++++++++++
1 files changed, 273 insertions(+), 0 deletions(-)

diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c
index 8018c2d..dfb434c 100644
--- a/drivers/s390/crypto/vfio_ap_ops.c
+++ b/drivers/s390/crypto/vfio_ap_ops.c
@@ -106,9 +106,282 @@ static ssize_t device_api_show(struct kobject *kobj, struct device *dev,
NULL,
};

+struct vfio_ap_queue_reserved {
+ unsigned long *apid;
+ unsigned long *apqi;
+ bool reserved;
+};
+
+/**
+ * vfio_ap_has_queue
+ *
+ * @dev: an AP queue device
+ * @data: a struct vfio_ap_queue_reserved reference
+ *
+ * Flags whether the AP queue device (@dev) has a queue ID containing the APQN,
+ * apid or apqi specified in @data:
+ *
+ * - If @data contains both an apid and apqi value, then @data will be flagged
+ * as reserved if the APID and APQI fields for the AP queue device matches
+ *
+ * - If @data contains only an apid value, @data will be flagged as
+ * reserved if the APID field in the AP queue device matches
+ *
+ * - If @data contains only an apqi value, @data will be flagged as
+ * reserved if the APQI field in the AP queue device matches
+ *
+ * Returns 0 to indicate the input to function succeeded. Returns -EINVAL if
+ * @data does not contain either an apid or apqi.
+ */
+static int vfio_ap_has_queue(struct device *dev, void *data)
+{
+ struct vfio_ap_queue_reserved *qres = data;
+ struct ap_queue *ap_queue = to_ap_queue(dev);
+ ap_qid_t qid;
+ unsigned long id;
+
+ if (qres->apid && qres->apqi) {
+ qid = AP_MKQID(*qres->apid, *qres->apqi);
+ if (qid == ap_queue->qid)
+ qres->reserved = true;
+ } else if (qres->apid && !qres->apqi) {
+ id = AP_QID_CARD(ap_queue->qid);
+ if (id == *qres->apid)
+ qres->reserved = true;
+ } else if (!qres->apid && qres->apqi) {
+ id = AP_QID_QUEUE(ap_queue->qid);
+ if (id == *qres->apqi)
+ qres->reserved = true;
+ } else {
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+/**
+ * vfio_ap_verify_queue_reserved
+ *
+ * @matrix_dev: a mediated matrix device
+ * @apid: an AP adapter ID
+ * @apqi: an AP queue index
+ *
+ * Verifies that the AP queue with @apid/@apqi is reserved by the VFIO AP device
+ * driver according to the following rules:
+ *
+ * - If both @apid and @apqi are not NULL, then there must be an AP queue
+ * device bound to the vfio_ap driver with the APQN identified by @apid and
+ * @apqi
+ *
+ * - If only @apid is not NULL, then there must be an AP queue device bound
+ * to the vfio_ap driver with an APQN containing @apid
+ *
+ * - If only @apqi is not NULL, then there must be an AP queue device bound
+ * to the vfio_ap driver with an APQN containing @apqi
+ *
+ * Returns 0 if the AP queue is reserved; otherwise, returns -EADDRNOTAVAIL.
+ */
+static int vfio_ap_verify_queue_reserved(unsigned long *apid,
+ unsigned long *apqi)
+{
+ int ret;
+ struct vfio_ap_queue_reserved qres;
+
+ qres.apid = apid;
+ qres.apqi = apqi;
+ qres.reserved = false;
+
+ ret = driver_for_each_device(matrix_dev.device.driver, NULL, &qres,
+ vfio_ap_has_queue);
+ if (ret)
+ return ret;
+
+ if (qres.reserved)
+ return 0;
+
+ return -EADDRNOTAVAIL;
+}
+
+static int
+vfio_ap_mdev_verify_queues_reserved_for_apid(struct ap_matrix_mdev *matrix_mdev,
+ unsigned long apid)
+{
+ int ret;
+ unsigned long apqi;
+ unsigned long nbits = matrix_mdev->matrix.aqm_max + 1;
+
+ if (find_first_bit_inv(matrix_mdev->matrix.aqm, nbits) >= nbits)
+ return vfio_ap_verify_queue_reserved(&apid, NULL);
+
+ for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, nbits) {
+ ret = vfio_ap_verify_queue_reserved(&apid, &apqi);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
+/**
+ * vfio_ap_mdev_verify_no_sharing
+ *
+ * Verifies that the APQNs derived from the cross product of the AP adapter IDs
+ * and AP queue indexes comprising the AP matrix are not configured for another
+ * mediated device. AP queue sharing is not allowed.
+ *
+ * @kvm: the KVM guest
+ * @matrix: the AP matrix
+ *
+ * Returns 0 if the APQNs are not shared, otherwise; returns -EADDRINUSE.
+ */
+static int vfio_ap_mdev_verify_no_sharing(struct ap_matrix_mdev *matrix_mdev)
+{
+ int nbits;
+ struct ap_matrix_mdev *lstdev;
+ unsigned long apm[BITS_TO_LONGS(matrix_mdev->matrix.apm_max + 1)];
+ unsigned long aqm[BITS_TO_LONGS(matrix_mdev->matrix.aqm_max + 1)];
+
+ list_for_each_entry(lstdev, &matrix_dev.mdev_list, list) {
+ if (matrix_mdev == lstdev)
+ continue;
+
+ memset(apm, 0, sizeof(apm));
+ memset(aqm, 0, sizeof(aqm));
+
+ /*
+ * We work on full longs, as we can only exclude the leftover
+ * bits in non-inverse order. The leftover is all zeros.
+ */
+ nbits = sizeof(apm) * BITS_PER_BYTE;
+ if (!bitmap_and(apm, matrix_mdev->matrix.apm,
+ lstdev->matrix.apm, nbits))
+ continue;
+
+ nbits = sizeof(aqm) * BITS_PER_BYTE;
+ if (!bitmap_and(aqm, matrix_mdev->matrix.aqm,
+ lstdev->matrix.aqm, nbits))
+ continue;
+
+ return -EADDRINUSE;
+ }
+
+ return 0;
+}
+
+/**
+ * assign_adapter_store
+ *
+ * @dev: the matrix device
+ * @attr: a mediated matrix device attribute
+ * @buf: a buffer containing the adapter ID (APID) to be assigned
+ * @count: the number of bytes in @buf
+ *
+ * Parses the APID from @buf and assigns it to the mediated matrix device.
+ *
+ * Returns the number of bytes processed if the APID is valid; otherwise returns
+ * an error.
+ */
+static ssize_t assign_adapter_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ int ret = 0;
+ unsigned long apid;
+ struct mdev_device *mdev = mdev_from_dev(dev);
+ struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
+ unsigned long max_apid = matrix_mdev->matrix.apm_max;
+
+ ret = kstrtoul(buf, 0, &apid);
+ if (ret)
+ return ret;
+ if (apid > max_apid)
+ return -EINVAL;
+
+ /* Set the bit in the AP mask (APM) corresponding to the AP adapter
+ * number (APID). The bits in the mask, from most significant to least
+ * significant bit, correspond to APIDs 0-255.
+ */
+ mutex_lock(&matrix_dev.lock);
+
+ ret = vfio_ap_mdev_verify_queues_reserved_for_apid(matrix_mdev, apid);
+ if (ret)
+ goto done;
+
+ set_bit_inv(apid, matrix_mdev->matrix.apm);
+
+ ret = vfio_ap_mdev_verify_no_sharing(matrix_mdev);
+ if (ret)
+ goto share_err;
+
+ ret = count;
+ goto done;
+
+share_err:
+ clear_bit_inv(apid, matrix_mdev->matrix.apm);
+done:
+ mutex_unlock(&matrix_dev.lock);
+
+ return ret;
+}
+static DEVICE_ATTR_WO(assign_adapter);
+
+/**
+ * unassign_adapter_store
+ *
+ * @dev: the matrix device
+ * @attr: a mediated matrix device attribute
+ * @buf: a buffer containing the adapter ID (APID) to be assigned
+ * @count: the number of bytes in @buf
+ *
+ * Parses the APID from @buf and unassigns it from the mediated matrix device.
+ * The APID must be a valid value
+ *
+ * Returns the number of bytes processed if the APID is valid; otherwise returns
+ * an error.
+ */
+static ssize_t unassign_adapter_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ int ret;
+ unsigned long apid;
+ struct mdev_device *mdev = mdev_from_dev(dev);
+ struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
+
+ ret = kstrtoul(buf, 0, &apid);
+ if (ret)
+ return ret;
+
+ if (apid > matrix_mdev->matrix.apm_max)
+ return -EINVAL;
+
+ mutex_lock(&matrix_dev.lock);
+ clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm);
+ mutex_unlock(&matrix_dev.lock);
+
+ return count;
+}
+DEVICE_ATTR_WO(unassign_adapter);
+
+static struct attribute *vfio_ap_mdev_attrs[] = {
+ &dev_attr_assign_adapter.attr,
+ &dev_attr_unassign_adapter.attr,
+ NULL
+};
+
+static struct attribute_group vfio_ap_mdev_attr_group = {
+ .attrs = vfio_ap_mdev_attrs
+};
+
+static const struct attribute_group *vfio_ap_mdev_attr_groups[] = {
+ &vfio_ap_mdev_attr_group,
+ NULL
+};
+
static const struct mdev_parent_ops vfio_ap_matrix_ops = {
.owner = THIS_MODULE,
.supported_type_groups = vfio_ap_mdev_type_groups,
+ .mdev_attr_groups = vfio_ap_mdev_attr_groups,
.create = vfio_ap_mdev_create,
.remove = vfio_ap_mdev_remove,
};
--
1.7.1

2018-08-13 21:52:35

[permalink] [raw]

Subject: [PATCH v9 13/22] s390: vfio-ap: sysfs interface to view matrix mdev matrix

From: Tony Krowiak <[email protected]>

Provides a sysfs interface to view the AP matrix configured for the
mediated matrix device.

The relevant sysfs structures are:

/sys/devices/vfio_ap/matrix/
...... [mdev_supported_types]
......... [vfio_ap-passthrough]
............ [devices]
...............[$uuid]
.................. matrix

To view the matrix configured for the mediated matrix device,
print the matrix file:

cat matrix

Below are examples of the output from the above command:

Example 1: Adapters and domains assigned
Assignments:
Adapters 5 and 6
Domains 4 and 71 (0x47)

Output
05.0004
05.0047
06.0004
06.0047

Examples 2: Only adapters assigned
Assignments:
Adapters 5 and 6

Output:
05.
06.

Examples 3: Only domains assigned
Assignments:
Domains 4 and 71 (0x47)

Output:
.0004
.0047

Signed-off-by: Tony Krowiak <[email protected]>
Reviewed-by: Halil Pasic <[email protected]>
Tested-by: Michael Mueller <[email protected]>
Tested-by: Farhan Ali <[email protected]>
Tested-by: Pierre Morel <[email protected]>
Signed-off-by: Christian Borntraeger <[email protected]>
---
drivers/s390/crypto/vfio_ap_ops.c | 52 +++++++++++++++++++++++++++++++++++++
1 files changed, 52 insertions(+), 0 deletions(-)

diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c
index f732177..af3b55f 100644
--- a/drivers/s390/crypto/vfio_ap_ops.c
+++ b/drivers/s390/crypto/vfio_ap_ops.c
@@ -552,6 +552,57 @@ static ssize_t control_domains_show(struct device *dev,
}
DEVICE_ATTR_RO(control_domains);

+static ssize_t matrix_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct mdev_device *mdev = mdev_from_dev(dev);
+ struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
+ char *bufpos = buf;
+ unsigned long apid;
+ unsigned long apqi;
+ unsigned long apid1;
+ unsigned long apqi1;
+ unsigned long napm_bits = matrix_mdev->matrix.apm_max + 1;
+ unsigned long naqm_bits = matrix_mdev->matrix.aqm_max + 1;
+ int nchars = 0;
+ int n;
+
+ apid1 = find_first_bit_inv(matrix_mdev->matrix.apm, napm_bits);
+ apqi1 = find_first_bit_inv(matrix_mdev->matrix.aqm, naqm_bits);
+
+ mutex_lock(&matrix_dev.lock);
+
+ if ((apid1 < napm_bits) && (apqi1 < naqm_bits)) {
+ for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, napm_bits) {
+ for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm,
+ naqm_bits) {
+ n = sprintf(bufpos, "%02lx.%04lx\n", apid,
+ apqi);
+ bufpos += n;
+ nchars += n;
+ }
+ }
+ } else if (apid1 < napm_bits) {
+ for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, napm_bits) {
+ n = sprintf(bufpos, "%02lx.\n", apid);
+ bufpos += n;
+ nchars += n;
+ }
+ } else if (apqi1 < naqm_bits) {
+ for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, naqm_bits) {
+ n = sprintf(bufpos, ".%04lx\n", apqi);
+ bufpos += n;
+ nchars += n;
+ }
+ }
+
+ mutex_unlock(&matrix_dev.lock);
+
+ return nchars;
+}
+DEVICE_ATTR_RO(matrix);
+
+
static struct attribute *vfio_ap_mdev_attrs[] = {
&dev_attr_assign_adapter.attr,
&dev_attr_unassign_adapter.attr,
@@ -560,6 +611,7 @@ static ssize_t control_domains_show(struct device *dev,
&dev_attr_assign_control_domain.attr,
&dev_attr_unassign_control_domain.attr,
&dev_attr_control_domains.attr,
+ &dev_attr_matrix.attr,
NULL,
};

--
1.7.1

2018-08-13 21:52:47

[permalink] [raw]

Subject: [PATCH v9 17/22] s390: vfio-ap: zeroize the AP queues.

From: Tony Krowiak <[email protected]>

Let's call PAPQ(ZAPQ) to zeroize a queue:

* For each queue configured for a mediated matrix device
when it is released.

Zeroizing a queue resets the queue, clears all pending
messages for the queue entries and disables adapter interruptions
associated with the queue.

Signed-off-by: Tony Krowiak <[email protected]>
Reviewed-by: Halil Pasic <[email protected]>
Tested-by: Michael Mueller <[email protected]>
Tested-by: Farhan Ali <[email protected]>
Signed-off-by: Christian Borntraeger <[email protected]>
---
drivers/s390/crypto/vfio_ap_ops.c | 25 +++++++++++++++++++++++++
drivers/s390/crypto/vfio_ap_private.h | 25 +++++++++++++++++++++++++
2 files changed, 50 insertions(+), 0 deletions(-)

diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c
index 368b559..cc90323 100644
--- a/drivers/s390/crypto/vfio_ap_ops.c
+++ b/drivers/s390/crypto/vfio_ap_ops.c
@@ -758,6 +758,30 @@ static int vfio_ap_mdev_group_notifier(struct notifier_block *nb,
return NOTIFY_OK;
}

+static int vfio_ap_mdev_reset_queues(struct mdev_device *mdev, bool force)
+{
+ int ret;
+ int rc = 0;
+ unsigned long apid, apqi;
+ struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
+
+ for_each_set_bit_inv(apid, matrix_mdev->matrix.apm,
+ matrix_mdev->matrix.apm_max + 1) {
+ for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm,
+ matrix_mdev->matrix.aqm_max + 1) {
+ ret = vfio_ap_reset_queue(apid, apqi, 1);
+ if (ret) {
+ if (force)
+ rc = ret;
+ else
+ return ret;
+ }
+ }
+ }
+
+ return rc;
+}
+
static int vfio_ap_mdev_open(struct mdev_device *mdev)
{
struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
@@ -788,6 +812,7 @@ static void vfio_ap_mdev_release(struct mdev_device *mdev)
if (matrix_mdev->kvm)
kvm_arch_crypto_clear_masks(matrix_mdev->kvm);

+ vfio_ap_mdev_reset_queues(mdev, true);
vfio_unregister_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY,
&matrix_mdev->group_notifier);
matrix_mdev->kvm = NULL;
diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h
index 3e8534b..34f982a 100644
--- a/drivers/s390/crypto/vfio_ap_private.h
+++ b/drivers/s390/crypto/vfio_ap_private.h
@@ -74,4 +74,29 @@ struct ap_matrix_mdev {
extern int vfio_ap_mdev_register(void);
extern void vfio_ap_mdev_unregister(void);

+static inline int vfio_ap_reset_queue(unsigned int apid, unsigned int apqi,
+ unsigned int retry)
+{
+ struct ap_queue_status status;
+
+ do {
+ status = ap_zapq(AP_MKQID(apid, apqi));
+ switch (status.response_code) {
+ case AP_RESPONSE_NORMAL:
+ return 0;
+ case AP_RESPONSE_RESET_IN_PROGRESS:
+ case AP_RESPONSE_BUSY:
+ msleep(20);
+ break;
+ default:
+ pr_warn("%s: error zeroizing %02x.%04x: response code %d\n",
+ VFIO_AP_MODULE_NAME, apid, apqi,
+ status.response_code);
+ return -EIO;
+ }
+ } while (retry--);
+
+ return -EBUSY;
+}
+
#endif /* _VFIO_AP_PRIVATE_H_ */
--
1.7.1

2018-08-13 21:53:04

[permalink] [raw]

Subject: [PATCH v9 05/22] KVM: s390: vsie: simulate VCPU SIE entry/exit

From: David Hildenbrand <[email protected]>

VCPU requests and VCPU blocking right now don't take care of the vSIE
(as it was not necessary until now). But we want to have VCPU requests
that will also be handled before running the vSIE again.

So let's simulate a SIE entry when entering the vSIE loop and check
for PROG_ flags. The existing infrastructure (e.g. exit_sie()) will then
detect that the SIE (in form of the vSIE execution loop) is running and
properly kick the vSIE CPU, resulting in it leaving the vSIE loop and
therefore the vSIE interception handler, allowing it to handle VCPU
requests.

E.g. if we want to modify the crycb of the VCPU and make sure that any
masks also get applied to the VSIE crycb shadow (which uses masks from the
VCPU crycb), we will need a way to hinder the vSIE from running and make
sure to process the updated crycb before reentering the vSIE again.

Signed-off-by: David Hildenbrand <[email protected]>
Signed-off-by: Tony Krowiak <[email protected]>
Reviewed-by: Pierre Morel <[email protected]>
---
arch/s390/kvm/kvm-s390.c | 9 ++++++++-
arch/s390/kvm/kvm-s390.h | 1 +
arch/s390/kvm/vsie.c | 20 ++++++++++++++++++--
3 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index 3b7a515..6df2d12 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -2655,18 +2655,25 @@ static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
exit_sie(vcpu);
}

+bool kvm_s390_vcpu_sie_inhibited(struct kvm_vcpu *vcpu)
+{
+ return atomic_read(&vcpu->arch.sie_block->prog20) &
+ (PROG_BLOCK_SIE | PROG_REQUEST);
+}
+
static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
{
atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
}

/*
- * Kick a guest cpu out of SIE and wait until SIE is not running.
+ * Kick a guest cpu out of (v)SIE and wait until (v)SIE is not running.
* If the CPU is not running (e.g. waiting as idle) the function will
* return immediately. */
void exit_sie(struct kvm_vcpu *vcpu)
{
kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
+ kvm_s390_vsie_kick(vcpu);
while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
cpu_relax();
}
diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/kvm-s390.h
index 981e3ba..1f6e36c 100644
--- a/arch/s390/kvm/kvm-s390.h
+++ b/arch/s390/kvm/kvm-s390.h
@@ -290,6 +290,7 @@ void kvm_s390_set_tod_clock(struct kvm *kvm,
void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu);
void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu);
void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu);
+bool kvm_s390_vcpu_sie_inhibited(struct kvm_vcpu *vcpu);
void exit_sie(struct kvm_vcpu *vcpu);
void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu);
int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu);
diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c
index 84c89cb..aa30b48 100644
--- a/arch/s390/kvm/vsie.c
+++ b/arch/s390/kvm/vsie.c
@@ -982,6 +982,17 @@ static int vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
int rc = 0;

+ /*
+ * Simulate a SIE entry of the VCPU (see sie64a), so VCPU blocking
+ * and VCPU requests can hinder the whole vSIE loop from running
+ * and lead to an immediate exit. We do it at this point (not
+ * earlier), so kvm_s390_vsie_kick() works correctly already.
+ */
+ vcpu->arch.sie_block->prog0c |= PROG_IN_SIE;
+ barrier();
+ if (kvm_s390_vcpu_sie_inhibited(vcpu))
+ return 0;
+
while (1) {
rc = acquire_gmap_shadow(vcpu, vsie_page);
if (!rc)
@@ -997,10 +1008,14 @@ static int vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
if (rc == -EAGAIN)
rc = 0;
if (rc || scb_s->icptcode || signal_pending(current) ||
- kvm_s390_vcpu_has_irq(vcpu, 0))
+ kvm_s390_vcpu_has_irq(vcpu, 0) ||
+ kvm_s390_vcpu_sie_inhibited(vcpu))
break;
}

+ barrier();
+ vcpu->arch.sie_block->prog0c &= ~PROG_IN_SIE;
+
if (rc == -EFAULT) {
/*
* Addressing exceptions are always presentes as intercepts.
@@ -1114,7 +1129,8 @@ int kvm_s390_handle_vsie(struct kvm_vcpu *vcpu)
if (unlikely(scb_addr & 0x1ffUL))
return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);

- if (signal_pending(current) || kvm_s390_vcpu_has_irq(vcpu, 0))
+ if (signal_pending(current) || kvm_s390_vcpu_has_irq(vcpu, 0) ||
+ kvm_s390_vcpu_sie_inhibited(vcpu))
return 0;

vsie_page = get_vsie_page(vcpu->kvm, scb_addr);
--
1.7.1

2018-08-13 21:53:20

[permalink] [raw]

Subject: [PATCH v9 04/22] s390/zcrypt: Integrate ap_asm.h into include/asm/ap.h.

From: Harald Freudenberger <[email protected]>

Move all the inline functions from the ap bus header
file ap_asm.h into the in-kernel api header file
arch/s390/include/asm/ap.h so that KVM can make use
of all the low level AP functions.

Signed-off-by: Harald Freudenberger <[email protected]>
Signed-off-by: Christian Borntraeger <[email protected]>
Signed-off-by: Tony Krowiak <[email protected]>
---
arch/s390/include/asm/ap.h | 284 ++++++++++++++++++++++++++++++++++++----
drivers/s390/crypto/ap_bus.c | 23 +---
drivers/s390/crypto/ap_bus.h | 1 +
drivers/s390/crypto/ap_card.c | 1 -
drivers/s390/crypto/ap_queue.c | 1 -
5 files changed, 260 insertions(+), 50 deletions(-)

diff --git a/arch/s390/include/asm/ap.h b/arch/s390/include/asm/ap.h
index c1bedb4..887494a 100644
--- a/arch/s390/include/asm/ap.h
+++ b/arch/s390/include/asm/ap.h
@@ -47,6 +47,50 @@ struct ap_queue_status {
};

/**
+ * ap_intructions_available() - Test if AP instructions are available.
+ *
+ * Returns 1 if the AP instructions are installed, otherwise 0.
+ */
+static inline int ap_instructions_available(void)
+{
+ register unsigned long reg0 asm ("0") = AP_MKQID(0, 0);
+ register unsigned long reg1 asm ("1") = 0;
+ register unsigned long reg2 asm ("2") = 0;
+
+ asm volatile(
+ " .long 0xb2af0000\n" /* PQAP(TAPQ) */
+ "0: la %0,1\n"
+ "1:\n"
+ EX_TABLE(0b, 1b)
+ : "+d" (reg1), "+d" (reg2)
+ : "d" (reg0)
+ : "cc");
+ return reg1;
+}
+
+/**
+ * ap_tapq(): Test adjunct processor queue.
+ * @qid: The AP queue number
+ * @info: Pointer to queue descriptor
+ *
+ * Returns AP queue status structure.
+ */
+static inline struct ap_queue_status ap_tapq(ap_qid_t qid, unsigned long *info)
+{
+ register unsigned long reg0 asm ("0") = qid;
+ register struct ap_queue_status reg1 asm ("1");
+ register unsigned long reg2 asm ("2");
+
+ asm volatile(".long 0xb2af0000" /* PQAP(TAPQ) */
+ : "=d" (reg1), "=d" (reg2)
+ : "d" (reg0)
+ : "cc");
+ if (info)
+ *info = reg2;
+ return reg1;
+}
+
+/**
* ap_test_queue(): Test adjunct processor queue.
* @qid: The AP queue number
* @tbit: Test facilities bit
@@ -54,10 +98,57 @@ struct ap_queue_status {
*
* Returns AP queue status structure.
*/
-struct ap_queue_status ap_test_queue(ap_qid_t qid,
- int tbit,
- unsigned long *info);
+static inline struct ap_queue_status ap_test_queue(ap_qid_t qid,
+ int tbit,
+ unsigned long *info)
+{
+ if (tbit)
+ qid |= 1UL << 23; /* set T bit*/
+ return ap_tapq(qid, info);
+}

+/**
+ * ap_pqap_rapq(): Reset adjunct processor queue.
+ * @qid: The AP queue number
+ *
+ * Returns AP queue status structure.
+ */
+static inline struct ap_queue_status ap_rapq(ap_qid_t qid)
+{
+ register unsigned long reg0 asm ("0") = qid | (1UL << 24);
+ register struct ap_queue_status reg1 asm ("1");
+
+ asm volatile(
+ ".long 0xb2af0000" /* PQAP(RAPQ) */
+ : "=d" (reg1)
+ : "d" (reg0)
+ : "cc");
+ return reg1;
+}
+
+/**
+ * ap_pqap_zapq(): Reset and zeroize adjunct processor queue.
+ * @qid: The AP queue number
+ *
+ * Returns AP queue status structure.
+ */
+static inline struct ap_queue_status ap_zapq(ap_qid_t qid)
+{
+ register unsigned long reg0 asm ("0") = qid | (2UL << 24);
+ register struct ap_queue_status reg1 asm ("1");
+
+ asm volatile(
+ ".long 0xb2af0000" /* PQAP(ZAPQ) */
+ : "=d" (reg1)
+ : "d" (reg0)
+ : "cc");
+ return reg1;
+}
+
+/**
+ * struct ap_config_info - convenience struct for AP crypto
+ * config info as returned by the ap_qci() function.
+ */
struct ap_config_info {
unsigned int apsc : 1; /* S bit */
unsigned int apxa : 1; /* N bit */
@@ -74,50 +165,189 @@ struct ap_config_info {
unsigned char _reserved4[16];
} __aligned(8);

-/*
- * ap_query_configuration(): Fetch cryptographic config info
+/**
+ * ap_qci(): Get AP configuration data
*
- * Returns the ap configuration info fetched via PQAP(QCI).
- * On success 0 is returned, on failure a negative errno
- * is returned, e.g. if the PQAP(QCI) instruction is not
- * available, the return value will be -EOPNOTSUPP.
+ * Returns 0 on success, or -EOPNOTSUPP.
*/
-int ap_query_configuration(struct ap_config_info *info);
+static inline int ap_qci(struct ap_config_info *config)
+{
+ register unsigned long reg0 asm ("0") = 4UL << 24;
+ register unsigned long reg1 asm ("1") = -EOPNOTSUPP;
+ register struct ap_config_info *reg2 asm ("2") = config;
+
+ asm volatile(
+ ".long 0xb2af0000\n" /* PQAP(QCI) */
+ "0: la %0,0\n"
+ "1:\n"
+ EX_TABLE(0b, 1b)
+ : "+d" (reg1)
+ : "d" (reg0), "d" (reg2)
+ : "cc", "memory");
+
+ return reg1;
+}

/*
* struct ap_qirq_ctrl - convenient struct for easy invocation
- * of the ap_queue_irq_ctrl() function. This struct is passed
- * as GR1 parameter to the PQAP(AQIC) instruction. For details
- * please see the AR documentation.
+ * of the ap_aqic() function. This struct is passed as GR1
+ * parameter to the PQAP(AQIC) instruction. For details please
+ * see the AR documentation.
*/
struct ap_qirq_ctrl {
unsigned int _res1 : 8;
- unsigned int zone : 8; /* zone info */
- unsigned int ir : 1; /* ir flag: enable (1) or disable (0) irq */
+ unsigned int zone : 8; /* zone info */
+ unsigned int ir : 1; /* ir flag: enable (1) or disable (0) irq */
unsigned int _res2 : 4;
- unsigned int gisc : 3; /* guest isc field */
+ unsigned int gisc : 3; /* guest isc field */
unsigned int _res3 : 6;
- unsigned int gf : 2; /* gisa format */
+ unsigned int gf : 2; /* gisa format */
unsigned int _res4 : 1;
- unsigned int gisa : 27; /* gisa origin */
+ unsigned int gisa : 27; /* gisa origin */
unsigned int _res5 : 1;
- unsigned int isc : 3; /* irq sub class */
+ unsigned int isc : 3; /* irq sub class */
};

/**
- * ap_queue_irq_ctrl(): Control interruption on a AP queue.
+ * ap_aqic(): Control interruption for a specific AP.
* @qid: The AP queue number
- * @qirqctrl: struct ap_qirq_ctrl, see above
+ * @qirqctrl: struct ap_qirq_ctrl (64 bit value)
* @ind: The notification indicator byte
*
* Returns AP queue status.
+ */
+static inline struct ap_queue_status ap_aqic(ap_qid_t qid,
+ struct ap_qirq_ctrl qirqctrl,
+ void *ind)
+{
+ register unsigned long reg0 asm ("0") = qid | (3UL << 24);
+ register struct ap_qirq_ctrl reg1_in asm ("1") = qirqctrl;
+ register struct ap_queue_status reg1_out asm ("1");
+ register void *reg2 asm ("2") = ind;
+
+ asm volatile(
+ ".long 0xb2af0000" /* PQAP(AQIC) */
+ : "=d" (reg1_out)
+ : "d" (reg0), "d" (reg1_in), "d" (reg2)
+ : "cc");
+ return reg1_out;
+}
+
+/*
+ * union ap_qact_ap_info - used together with the
+ * ap_aqic() function to provide a convenient way
+ * to handle the ap info needed by the qact function.
+ */
+union ap_qact_ap_info {
+ unsigned long val;
+ struct {
+ unsigned int : 3;
+ unsigned int mode : 3;
+ unsigned int : 26;
+ unsigned int cat : 8;
+ unsigned int : 8;
+ unsigned char ver[2];
+ };
+};
+
+/**
+ * ap_qact(): Query AP combatibility type.
+ * @qid: The AP queue number
+ * @apinfo: On input the info about the AP queue. On output the
+ * alternate AP queue info provided by the qact function
+ * in GR2 is stored in.
*
- * Control interruption on the given AP queue.
- * Just a simple wrapper function for the low level PQAP(AQIC)
- * instruction available for other kernel modules.
+ * Returns AP queue status. Check response_code field for failures.
*/
-struct ap_queue_status ap_queue_irq_ctrl(ap_qid_t qid,
- struct ap_qirq_ctrl qirqctrl,
- void *ind);
+static inline struct ap_queue_status ap_qact(ap_qid_t qid, int ifbit,
+ union ap_qact_ap_info *apinfo)
+{
+ register unsigned long reg0 asm ("0") = qid | (5UL << 24)
+ | ((ifbit & 0x01) << 22);
+ register unsigned long reg1_in asm ("1") = apinfo->val;
+ register struct ap_queue_status reg1_out asm ("1");
+ register unsigned long reg2 asm ("2");
+
+ asm volatile(
+ ".long 0xb2af0000" /* PQAP(QACT) */
+ : "+d" (reg1_in), "=d" (reg1_out), "=d" (reg2)
+ : "d" (reg0)
+ : "cc");
+ apinfo->val = reg2;
+ return reg1_out;
+}
+
+/**
+ * ap_nqap(): Send message to adjunct processor queue.
+ * @qid: The AP queue number
+ * @psmid: The program supplied message identifier
+ * @msg: The message text
+ * @length: The message length
+ *
+ * Returns AP queue status structure.
+ * Condition code 1 on NQAP can't happen because the L bit is 1.
+ * Condition code 2 on NQAP also means the send is incomplete,
+ * because a segment boundary was reached. The NQAP is repeated.
+ */
+static inline struct ap_queue_status ap_nqap(ap_qid_t qid,
+ unsigned long long psmid,
+ void *msg, size_t length)
+{
+ register unsigned long reg0 asm ("0") = qid | 0x40000000UL;
+ register struct ap_queue_status reg1 asm ("1");
+ register unsigned long reg2 asm ("2") = (unsigned long) msg;
+ register unsigned long reg3 asm ("3") = (unsigned long) length;
+ register unsigned long reg4 asm ("4") = (unsigned int) (psmid >> 32);
+ register unsigned long reg5 asm ("5") = psmid & 0xffffffff;
+
+ asm volatile (
+ "0: .long 0xb2ad0042\n" /* NQAP */
+ " brc 2,0b"
+ : "+d" (reg0), "=d" (reg1), "+d" (reg2), "+d" (reg3)
+ : "d" (reg4), "d" (reg5)
+ : "cc", "memory");
+ return reg1;
+}
+
+/**
+ * ap_dqap(): Receive message from adjunct processor queue.
+ * @qid: The AP queue number
+ * @psmid: Pointer to program supplied message identifier
+ * @msg: The message text
+ * @length: The message length
+ *
+ * Returns AP queue status structure.
+ * Condition code 1 on DQAP means the receive has taken place
+ * but only partially. The response is incomplete, hence the
+ * DQAP is repeated.
+ * Condition code 2 on DQAP also means the receive is incomplete,
+ * this time because a segment boundary was reached. Again, the
+ * DQAP is repeated.
+ * Note that gpr2 is used by the DQAP instruction to keep track of
+ * any 'residual' length, in case the instruction gets interrupted.
+ * Hence it gets zeroed before the instruction.
+ */
+static inline struct ap_queue_status ap_dqap(ap_qid_t qid,
+ unsigned long long *psmid,
+ void *msg, size_t length)
+{
+ register unsigned long reg0 asm("0") = qid | 0x80000000UL;
+ register struct ap_queue_status reg1 asm ("1");
+ register unsigned long reg2 asm("2") = 0UL;
+ register unsigned long reg4 asm("4") = (unsigned long) msg;
+ register unsigned long reg5 asm("5") = (unsigned long) length;
+ register unsigned long reg6 asm("6") = 0UL;
+ register unsigned long reg7 asm("7") = 0UL;
+
+
+ asm volatile(
+ "0: .long 0xb2ae0064\n" /* DQAP */
+ " brc 6,0b\n"
+ : "+d" (reg0), "=d" (reg1), "+d" (reg2),
+ "+d" (reg4), "+d" (reg5), "+d" (reg6), "+d" (reg7)
+ : : "cc", "memory");
+ *psmid = (((unsigned long long) reg6) << 32) + reg7;
+ return reg1;
+}

#endif /* _ASM_S390_AP_H_ */
diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c
index 35a0c2b..aa1bbe6 100644
--- a/drivers/s390/crypto/ap_bus.c
+++ b/drivers/s390/crypto/ap_bus.c
@@ -36,7 +36,6 @@
#include <linux/debugfs.h>

#include "ap_bus.h"
-#include "ap_asm.h"
#include "ap_debug.h"

/*
@@ -174,24 +173,6 @@ static inline int ap_qact_available(void)
return 0;
}

-/**
- * ap_test_queue(): Test adjunct processor queue.
- * @qid: The AP queue number
- * @tbit: Test facilities bit
- * @info: Pointer to queue descriptor
- *
- * Returns AP queue status structure.
- */
-struct ap_queue_status ap_test_queue(ap_qid_t qid,
- int tbit,
- unsigned long *info)
-{
- if (tbit)
- qid |= 1UL << 23; /* set T bit*/
- return ap_tapq(qid, info);
-}
-EXPORT_SYMBOL(ap_test_queue);
-
/*
* ap_query_configuration(): Fetch cryptographic config info
*
@@ -200,7 +181,7 @@ struct ap_queue_status ap_test_queue(ap_qid_t qid,
* is returned, e.g. if the PQAP(QCI) instruction is not
* available, the return value will be -EOPNOTSUPP.
*/
-int ap_query_configuration(struct ap_config_info *info)
+static inline int ap_query_configuration(struct ap_config_info *info)
{
if (!ap_configuration_available())
return -EOPNOTSUPP;
@@ -1220,7 +1201,7 @@ static int __init ap_module_init(void)
if (rc)
return rc;

- if (ap_instructions_available() != 0) {
+ if (!ap_instructions_available()) {
pr_warn("The hardware system does not support AP instructions\n");
return -ENODEV;
}
diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h
index 6a273c5..9365419 100644
--- a/drivers/s390/crypto/ap_bus.h
+++ b/drivers/s390/crypto/ap_bus.h
@@ -15,6 +15,7 @@

#include <linux/device.h>
#include <linux/types.h>
+#include <asm/isc.h>
#include <asm/ap.h>

#define AP_DEVICES 256 /* Number of AP devices. */
diff --git a/drivers/s390/crypto/ap_card.c b/drivers/s390/crypto/ap_card.c
index 2c726df..c13e432 100644
--- a/drivers/s390/crypto/ap_card.c
+++ b/drivers/s390/crypto/ap_card.c
@@ -14,7 +14,6 @@
#include <asm/facility.h>

#include "ap_bus.h"
-#include "ap_asm.h"

/*
* AP card related attributes.
diff --git a/drivers/s390/crypto/ap_queue.c b/drivers/s390/crypto/ap_queue.c
index ba3a2e1..d83c1fa 100644
--- a/drivers/s390/crypto/ap_queue.c
+++ b/drivers/s390/crypto/ap_queue.c
@@ -14,7 +14,6 @@
#include <asm/facility.h>

#include "ap_bus.h"
-#include "ap_asm.h"

/**
* ap_queue_irq_ctrl(): Control interruption on a AP queue.
--
1.7.1

2018-08-13 21:53:24

[permalink] [raw]

Subject: [PATCH v9 06/22] KVM: s390: introduce and use KVM_REQ_VSIE_RESTART

From: David Hildenbrand <[email protected]>

When we change the crycb (or execution controls), we also have to make sure
that the vSIE shadow datastructures properly consider the changed
values before rerunning the vSIE. We can achieve that by simply using a
VCPU request now.

This has to be a synchronous request (== handled before entering the
(v)SIE again).

The request will make sure that the vSIE handler is left, and that the
request will be processed (NOP), therefore forcing a reload of all
vSIE data (including rebuilding the crycb) when re-entering the vSIE
interception handler the next time.

Signed-off-by: David Hildenbrand <[email protected]>
Signed-off-by: Tony Krowiak <[email protected]>
Reviewed-by: Pierre Morel <[email protected]>
---
arch/s390/include/asm/kvm_host.h | 1 +
arch/s390/kvm/kvm-s390.c | 7 ++++++-
2 files changed, 7 insertions(+), 1 deletions(-)

diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
index a2188e3..af39561 100644
--- a/arch/s390/include/asm/kvm_host.h
+++ b/arch/s390/include/asm/kvm_host.h
@@ -44,6 +44,7 @@
#define KVM_REQ_ICPT_OPEREXC KVM_ARCH_REQ(2)
#define KVM_REQ_START_MIGRATION KVM_ARCH_REQ(3)
#define KVM_REQ_STOP_MIGRATION KVM_ARCH_REQ(4)
+#define KVM_REQ_VSIE_RESTART KVM_ARCH_REQ(5)

#define SIGP_CTRL_C 0x80
#define SIGP_CTRL_SCN_MASK 0x3f
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index 6df2d12..19f4f44 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -798,8 +798,11 @@ void kvm_s390_vcpu_crypto_reset_all(struct kvm *kvm)

kvm_s390_vcpu_block_all(kvm);

- kvm_for_each_vcpu(i, vcpu, kvm)
+ kvm_for_each_vcpu(i, vcpu, kvm) {
kvm_s390_vcpu_crypto_setup(vcpu);
+ /* recreate the shadow crycb by leaving the VSIE handler */
+ kvm_s390_sync_request(KVM_REQ_VSIE_RESTART, vcpu);
+ }

kvm_s390_vcpu_unblock_all(kvm);
}
@@ -3090,6 +3093,8 @@ static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)

/* nothing to do, just clear the request */
kvm_clear_request(KVM_REQ_UNHALT, vcpu);
+ /* we left the vsie handler, nothing to do, just clear the request */
+ kvm_clear_request(KVM_REQ_VSIE_RESTART, vcpu);

return 0;
}
--
1.7.1

2018-08-13 21:53:55

[permalink] [raw]

Subject: [PATCH v9 01/22] s390/zcrypt: Add ZAPQ inline function.

From: Harald Freudenberger <[email protected]>

Added new inline function ap_pqap_zapq()
which is a C inline function wrapper for
the AP PQAP(ZAPQ) instruction.

Signed-off-by: Harald Freudenberger <[email protected]>
Signed-off-by: Christian Borntraeger <[email protected]>
Signed-off-by: Tony Krowiak <[email protected]>
---
drivers/s390/crypto/ap_asm.h | 19 +++++++++++++++++++
1 files changed, 19 insertions(+), 0 deletions(-)

diff --git a/drivers/s390/crypto/ap_asm.h b/drivers/s390/crypto/ap_asm.h
index 16b59ce..b22d30a 100644
--- a/drivers/s390/crypto/ap_asm.h
+++ b/drivers/s390/crypto/ap_asm.h
@@ -70,6 +70,25 @@ static inline struct ap_queue_status ap_rapq(ap_qid_t qid)
}

/**
+ * ap_pqap_zapq(): Reset and zeroize adjunct processor queue.
+ * @qid: The AP queue number
+ *
+ * Returns AP queue status structure.
+ */
+static inline struct ap_queue_status ap_zapq(ap_qid_t qid)
+{
+ register unsigned long reg0 asm ("0") = qid | (2UL << 24);
+ register struct ap_queue_status reg1 asm ("1");
+
+ asm volatile(
+ ".long 0xb2af0000" /* PQAP(ZAPQ) */
+ : "=d" (reg1)
+ : "d" (reg0)
+ : "cc");
+ return reg1;
+}
+
+/**
* ap_aqic(): Control interruption for a specific AP.
* @qid: The AP queue number
* @qirqctrl: struct ap_qirq_ctrl (64 bit value)
--
1.7.1

2018-08-13 21:54:00

[permalink] [raw]

Subject: [PATCH v9 03/22] s390/zcrypt: Show load of cards and queues in sysfs

From: Harald Freudenberger <[email protected]>

Show the current load value of cards and queues in sysfs.
The load value for each card and queue is maintained by
the zcrypt device driver for dispatching and load
balancing requests over the available devices.

This patch provides the load value to userspace via a
new read only sysfs attribute 'load' per card and queue.

Signed-off-by: Harald Freudenberger <[email protected]>
Signed-off-by: Christian Borntraeger <[email protected]>
Signed-off-by: Tony Krowiak <[email protected]>
---
drivers/s390/crypto/zcrypt_card.c | 12 ++++++++++++
drivers/s390/crypto/zcrypt_queue.c | 12 ++++++++++++
2 files changed, 24 insertions(+), 0 deletions(-)

diff --git a/drivers/s390/crypto/zcrypt_card.c b/drivers/s390/crypto/zcrypt_card.c
index 233e1e6..da2c8df 100644
--- a/drivers/s390/crypto/zcrypt_card.c
+++ b/drivers/s390/crypto/zcrypt_card.c
@@ -83,9 +83,21 @@ static ssize_t zcrypt_card_online_store(struct device *dev,
static DEVICE_ATTR(online, 0644, zcrypt_card_online_show,
zcrypt_card_online_store);

+static ssize_t zcrypt_card_load_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct zcrypt_card *zc = to_ap_card(dev)->private;
+
+ return snprintf(buf, PAGE_SIZE, "%d\n", atomic_read(&zc->load));
+}
+
+static DEVICE_ATTR(load, 0444, zcrypt_card_load_show, NULL);
+
static struct attribute *zcrypt_card_attrs[] = {
&dev_attr_type.attr,
&dev_attr_online.attr,
+ &dev_attr_load.attr,
NULL,
};

diff --git a/drivers/s390/crypto/zcrypt_queue.c b/drivers/s390/crypto/zcrypt_queue.c
index 720434e..91a52f2 100644
--- a/drivers/s390/crypto/zcrypt_queue.c
+++ b/drivers/s390/crypto/zcrypt_queue.c
@@ -75,8 +75,20 @@ static ssize_t zcrypt_queue_online_store(struct device *dev,
static DEVICE_ATTR(online, 0644, zcrypt_queue_online_show,
zcrypt_queue_online_store);

+static ssize_t zcrypt_queue_load_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct zcrypt_queue *zq = to_ap_queue(dev)->private;
+
+ return snprintf(buf, PAGE_SIZE, "%d\n", atomic_read(&zq->load));
+}
+
+static DEVICE_ATTR(load, 0444, zcrypt_queue_load_show, NULL);
+
static struct attribute *zcrypt_queue_attrs[] = {
&dev_attr_online.attr,
+ &dev_attr_load.attr,
NULL,
};

--
1.7.1

2018-08-13 22:31:35

[permalink] [raw]

Subject: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

From: Tony Krowiak <[email protected]>

Provides the sysfs interfaces for:

1. Assigning AP control domains to the mediated matrix device

2. Unassigning AP control domains from a mediated matrix device

3. Displaying the control domains assigned to a mediated matrix
device

The IDs of the AP control domains assigned to the mediated matrix
device are stored in an AP domain mask (ADM). The bits in the ADM,
from most significant to least significant bit, correspond to
AP domain numbers 0 to 255. On some systems, the maximum allowable
domain number may be less than 255 - depending upon the host's
AP configuration - and assignment may be rejected if the input
domain ID exceeds the limit.

When a control domain is assigned, the bit corresponding its domain
ID will be set in the ADM. Likewise, when a domain is unassigned,
the bit corresponding to its domain ID will be cleared in the ADM.

The relevant sysfs structures are:

/sys/devices/vfio_ap/matrix/
...... [mdev_supported_types]
......... [vfio_ap-passthrough]
............ [devices]
...............[$uuid]
.................. assign_control_domain
.................. unassign_control_domain

To assign a control domain to the $uuid mediated matrix device's
ADM, write its domain number to the assign_control_domain file.
To unassign a domain, write its domain number to the
unassign_control_domain file. The domain number is specified
using conventional semantics: If it begins with 0x the number
will be parsed as a hexadecimal (case insensitive) number;
if it begins with 0, it is parsed as an octal number;
otherwise, it will be parsed as a decimal number.

For example, to assign control domain 173 (0xad) to the mediated
matrix device $uuid:

echo 173 > assign_control_domain

or

echo 0255 > assign_control_domain

or

echo 0xad > assign_control_domain

To unassign control domain 173 (0xad):

echo 173 > unassign_control_domain

or

echo 0255 > unassign_control_domain

or

echo 0xad > unassign_control_domain

The assignment will be rejected if the APQI exceeds the maximum
value for an AP domain:
* If the AP Extended Addressing (APXA) facility is installed,
the max value is 255
* Else the max value is 15

Signed-off-by: Tony Krowiak <[email protected]>
Reviewed-by: Halil Pasic <[email protected]>
Tested-by: Michael Mueller <[email protected]>
Tested-by: Farhan Ali <[email protected]>
Tested-by: Pierre Morel <[email protected]>
Signed-off-by: Christian Borntraeger <[email protected]>
---
drivers/s390/crypto/vfio_ap_ops.c | 108 +++++++++++++++++++++++++++++++++++++
1 files changed, 108 insertions(+), 0 deletions(-)

diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c
index c4d7504..f732177 100644
--- a/drivers/s390/crypto/vfio_ap_ops.c
+++ b/drivers/s390/crypto/vfio_ap_ops.c
@@ -447,11 +447,119 @@ static ssize_t unassign_domain_store(struct device *dev,
}
DEVICE_ATTR_WO(unassign_domain);

+/**
+ * assign_control_domain_store
+ *
+ * @dev: the matrix device
+ * @attr: the mediated matrix device's assign_control_domain attribute
+ * @buf: a buffer containing the domain ID to be assigned
+ * @count: the number of bytes in @buf
+ *
+ * Parses the domain ID from @buf and assigns it to the mediated matrix device.
+ *
+ * Returns the number of bytes processed if the domain ID is valid; otherwise
+ * returns an error.
+ */
+static ssize_t assign_control_domain_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ int ret;
+ unsigned long id;
+ struct mdev_device *mdev = mdev_from_dev(dev);
+ struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
+ unsigned long maxid = matrix_mdev->matrix.adm_max;
+
+ ret = kstrtoul(buf, 0, &id);
+ if (ret)
+ return ret;
+
+ if (id > maxid)
+ ret = -EINVAL;
+
+ /* Set the bit in the ADM (bitmask) corresponding to the AP control
+ * domain number (id). The bits in the mask, from most significant to
+ * least significant, correspond to IDs 0 up to the one less than the
+ * number of control domains that can be assigned.
+ */
+ mutex_lock(&matrix_dev.lock);
+ set_bit_inv(id, matrix_mdev->matrix.adm);
+ mutex_unlock(&matrix_dev.lock);
+
+ return count;
+}
+DEVICE_ATTR_WO(assign_control_domain);
+
+/**
+ * unassign_control_domain_store
+ *
+ * @dev: the matrix device
+ * @attr: the mediated matrix device's unassign_control_domain attribute
+ * @buf: a buffer containing the domain ID to be assigned
+ * @count: the number of bytes in @buf
+ *
+ * Parses the domain ID from @buf and unassigns it from the mediated matrix
+ * device.
+ *
+ * Returns the number of bytes processed if the domain ID is valid; otherwise
+ * returns an error.
+ */
+static ssize_t unassign_control_domain_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ int ret;
+ unsigned long domid;
+ struct mdev_device *mdev = mdev_from_dev(dev);
+ struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
+ unsigned long max_domid = matrix_mdev->matrix.adm_max;
+
+ ret = kstrtoul(buf, 0, &domid);
+ if (ret)
+ return ret;
+ if (domid > max_domid)
+ return -EINVAL;
+
+ mutex_lock(&matrix_dev.lock);
+ clear_bit_inv(domid, matrix_mdev->matrix.adm);
+ mutex_unlock(&matrix_dev.lock);
+
+ return count;
+}
+DEVICE_ATTR_WO(unassign_control_domain);
+
+static ssize_t control_domains_show(struct device *dev,
+ struct device_attribute *dev_attr,
+ char *buf)
+{
+ unsigned long id;
+ int nchars = 0;
+ int n;
+ char *bufpos = buf;
+ struct mdev_device *mdev = mdev_from_dev(dev);
+ struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
+ unsigned long max_domid = matrix_mdev->matrix.adm_max;
+
+ mutex_lock(&matrix_dev.lock);
+ for_each_set_bit_inv(id, matrix_mdev->matrix.adm, max_domid + 1) {
+ n = sprintf(bufpos, "%04lx\n", id);
+ bufpos += n;
+ nchars += n;
+ }
+ mutex_unlock(&matrix_dev.lock);
+
+ return nchars;
+}
+DEVICE_ATTR_RO(control_domains);
+
static struct attribute *vfio_ap_mdev_attrs[] = {
&dev_attr_assign_adapter.attr,
&dev_attr_unassign_adapter.attr,
&dev_attr_assign_domain.attr,
&dev_attr_unassign_domain.attr,
+ &dev_attr_assign_control_domain.attr,
+ &dev_attr_unassign_control_domain.attr,
+ &dev_attr_control_domains.attr,
NULL,
};

--
1.7.1

2018-08-13 22:31:58

[permalink] [raw]

Subject: [PATCH v9 22/22] s390: doc: detailed specifications for AP virtualization

From: Tony Krowiak <[email protected]>

This patch provides documentation describing the AP architecture and
design concepts behind the virtualization of AP devices. It also
includes an example of how to configure AP devices for exclusive
use of KVM guests.

Signed-off-by: Tony Krowiak <[email protected]>
Reviewed-by: Halil Pasic <[email protected]>
Signed-off-by: Christian Borntraeger <[email protected]>
---
Documentation/s390/vfio-ap.txt | 615 ++++++++++++++++++++++++++++++++++++++++
MAINTAINERS | 1 +
2 files changed, 616 insertions(+), 0 deletions(-)
create mode 100644 Documentation/s390/vfio-ap.txt

diff --git a/Documentation/s390/vfio-ap.txt b/Documentation/s390/vfio-ap.txt
new file mode 100644
index 0000000..524951a
--- /dev/null
+++ b/Documentation/s390/vfio-ap.txt
@@ -0,0 +1,615 @@
+Introduction:
+============
+The Adjunct Processor (AP) facility is an IBM Z cryptographic facility comprised
+of three AP instructions and from 1 up to 256 PCIe cryptographic adapter cards.
+The AP devices provide cryptographic functions to all CPUs assigned to a
+linux system running in an IBM Z system LPAR.
+
+The AP adapter cards are exposed via the AP bus. The motivation for vfio-ap
+is to make AP cards available to KVM guests using the VFIO mediated device
+framework. This implementation relies considerably on the s390 virtualization
+facilities which do most of the hard work of providing direct access to AP
+devices.
+
+AP Architectural Overview:
+=========================
+To facilitate the comprehension of the design, let's start with some
+definitions:
+
+* AP adapter
+
+ An AP adapter is an IBM Z adapter card that can perform cryptographic
+ functions. There can be from 0 to 256 adapters assigned to an LPAR. Adapters
+ assigned to the LPAR in which a linux host is running will be available to
+ the linux host. Each adapter is identified by a number from 0 to 255. When
+ installed, an AP adapter is accessed by AP instructions executed by any CPU.
+
+ The AP adapter cards are assigned to a given LPAR via the system's Activation
+ Profile which can be edited via the HMC. When the system is IPL'd, the AP bus
+ module is loaded and detects the AP adapter cards assigned to the LPAR. The AP
+ bus creates a sysfs device for each adapter as they are detected. For example,
+ if AP adapters 4 and 10 (0x0a) are assigned to the LPAR, the AP bus will
+ create the following sysfs entries:
+
+ /sys/devices/ap/card04
+ /sys/devices/ap/card0a
+
+ Symbolic links to these devices will also be created in the AP bus devices
+ sub-directory:
+
+ /sys/bus/ap/devices/[card04]
+ /sys/bus/ap/devices/[card04]
+
+* AP domain
+
+ An adapter is partitioned into domains. Each domain can be thought of as
+ a set of hardware registers for processing AP instructions. An adapter can
+ hold up to 256 domains. Each domain is identified by a number from 0 to 255.
+ Domains can be further classified into two types:
+
+ * Usage domains are domains that can be accessed directly to process AP
+ commands.
+
+ * Control domains are domains that are accessed indirectly by AP
+ commands sent to a usage domain to control or change the domain; for
+ example, to set a secure private key for the domain.
+
+ The AP usage and control domains are assigned to a given LPAR via the system's
+ Activation Profile which can be edited via the HMC. When the system is IPL'd,
+ the AP bus module is loaded and detects the AP usage and control domains
+ assigned to the LPAR. The domain number of each usage domain will be coupled
+ with the adapter number of each AP adapter assigned to the LPAR to identify
+ the AP queues (see AP Queue section below). The domain number of each control
+ domain will be represented in a bitmask and stored in a sysfs file
+ /sys/bus/ap/ap_control_domain_mask created by the bus. The bits in the mask,
+ from most to least significant bit, correspond to domains 0-255.
+
+ A domain may be assigned to a system as both a usage and control domain, or
+ as a control domain only. Consequently, all domains assigned as both a usage
+ and control domain can both process AP commands as well as be changed by an AP
+ command sent to any usage domain assigned to the same system. Domains assigned
+ only as control domains can not process AP commands but can be changed by AP
+ commands sent to any usage domain assigned to the system.
+
+* AP Queue
+
+ An AP queue is the means by which an AP command-request message is sent to a
+ usage domain inside a specific adapter. An AP queue is identified by a tuple
+ comprised of an AP adapter ID (APID) and an AP queue index (APQI). The
+ APQI corresponds to a given usage domain number within the adapter. This tuple
+ forms an AP Queue Number (APQN) uniquely identifying an AP queue. AP
+ instructions include a field containing the APQN to identify the AP queue to
+ which the AP command-request message is to be sent for processing.
+
+ The AP bus will create a sysfs device for each APQN that can be derived from
+ the cross product of the AP adapter and usage domain numbers detected when the
+ AP bus module is loaded. For example, if adapters 4 and 10 (0x0a) and usage
+ domains 6 and 71 (0x47) are assigned to the LPAR, the AP bus will create the
+ following sysfs entries:
+
+ /sys/devices/ap/card04/04.0006
+ /sys/devices/ap/card04/04.0047
+ /sys/devices/ap/card0a/0a.0006
+ /sys/devices/ap/card0a/0a.0047
+
+ The following symbolic links to these devices will be created in the AP bus
+ devices subdirectory:
+
+ /sys/bus/ap/devices/[04.0006]
+ /sys/bus/ap/devices/[04.0047]
+ /sys/bus/ap/devices/[0a.0006]
+ /sys/bus/ap/devices/[0a.0047]
+
+* AP Instructions:
+
+ There are three AP instructions:
+
+ * NQAP: to enqueue an AP command-request message to a queue
+ * DQAP: to dequeue an AP command-reply message from a queue
+ * PQAP: to administer the queues
+
+AP and SIE:
+==========
+Let's now take a look at how AP instructions executed on a guest are interpreted
+by the hardware.
+
+A satellite control block called the Crypto Control Block (CRYCB) is attached to
+our main hardware virtualization control block. The CRYCB contains three fields
+to identify the adapters, usage domains and control domains assigned to the KVM
+guest:
+
+* The AP Mask (APM) field is a bit mask that identifies the AP adapters assigned
+ to the KVM guest. Each bit in the mask, from most significant to least
+ significant bit, corresponds to an APID from 0-255. If a bit is set, the
+ corresponding adapter is valid for use by the KVM guest.
+
+* The AP Queue Mask (AQM) field is a bit mask identifying the AP usage domains
+ assigned to the KVM guest. Each bit in the mask, from most significant to
+ least significant bit, corresponds to an AP queue index (APQI) from 0-255. If
+ a bit is set, the corresponding queue is valid for use by the KVM guest.
+
+* The AP Domain Mask field is a bit mask that identifies the AP control domains
+ assigned to the KVM guest. The ADM bit mask controls which domains can be
+ changed by an AP command-request message sent to a usage domain from the
+ guest. Each bit in the mask, from least significant to most significant bit,
+ corresponds to a domain from 0-255. If a bit is set, the corresponding domain
+ can be modified by an AP command-request message sent to a usage domain
+ configured for the KVM guest.
+
+If you recall from the description of an AP Queue, AP instructions include
+an APQN to identify the AP adapter and AP queue to which an AP command-request
+message is to be sent (NQAP and PQAP instructions), or from which a
+command-reply message is to be received (DQAP instruction). The validity of an
+APQN is defined by the matrix calculated from the APM and AQM; it is the
+cross product of all assigned adapter numbers (APM) with all assigned queue
+indexes (AQM). For example, if adapters 1 and 2 and usage domains 5 and 6 are
+assigned to a guest, the APQNs (1,5), (1,6), (2,5) and (2,6) will be valid for
+the guest.
+
+The APQNs can provide secure key functionality - i.e., a private key is stored
+on the adapter card for each of its domains - so each APQN must be assigned to
+at most one guest or to the linux host.
+
+ Example 1: Valid configuration:
+ ------------------------------
+ Guest1: adapters 1,2 domains 5,6
+ Guest2: adapter 1,2 domain 7
+
+ This is valid because both guests have a unique set of APQNs: Guest1 has
+ APQNs (1,5), (1,6), (2,5) and (2,6); Guest2 has APQNs (1,7) and (2,7).
+
+ Example 2: Invalid configuration:
+ Guest1: adapters 1,2 domains 5,6
+ Guest2: adapter 1 domains 6,7
+
+ This is an invalid configuration because both guests have access to
+ APQN (1,6).
+
+The Design:
+===========
+The design introduces three new objects:
+
+1. AP matrix device
+2. VFIO AP device driver (vfio_ap.ko)
+3. AP mediated matrix passthrough device
+
+The VFIO AP device driver
+-------------------------
+The VFIO AP (vfio_ap) device driver serves the following purposes:
+
+1. Provides the interfaces to bind APQNs for exclusive use of KVM guests.
+
+2. Sets up the VFIO mediated device interfaces to manage a mediated matrix
+ device and creates the sysfs interfaces for assigning adapters, usage
+ domains, and control domains comprising the matrix for a KVM guest.
+
+3. Configures the APM, AQM and ADM in the CRYCB referenced by a KVM guest's
+ SIE state description to grant the guest access to a matrix of AP devices
+
+Reserve APQNs for exclusive use of KVM guests
+---------------------------------------------
+The following block diagram illustrates the mechanism by which APQNs are
+reserved:
+
+ +------------------+
+ remove | |
+ +------------------->+ cex4queue driver +
+ | | |
+ | +------------------+
+ |
+ |
+ | remove +------------------+
+ | +-----------------+ |<---------------+
+ | | probe | Device core | |
+ | | +--------------+ +<-----------+ |
+ | | | +--------+---------+ | |
+ | | | ^ | |
+ | | | register | | |
+ | | | vfio_ap device | bind | | unbind
+ | v v | vfio_ap | | cex4queue
++--------+-----+---+ +--------+---------+ +-+---+---+--+
+| | register | | | |
+| ap_bus +<---------+ vfio_ap driver + + admin |
+| +--------->+ | | |
++------------------+ probe +---+--------+-----+ +------------+
+ | |
+ create | | assign
+ | | adapters/domains/control domains
+ v v
+ +---+--------+-----+
+ | |
+ | mediated device |
+ | |
+ +------------------+
+
+The process for reserving an AP queue for use by a KVM guest is:
+
+* The vfio-ap driver during its initialization will perform the following:
+ * Create a single 'matrix' device, /sys/devices/vfio_ap/matrix. This will
+ serve as the parent device for all mediated matrix devices used to configure
+ an AP matrix which is the cross product of all AP adapter IDs (APID) and
+ queue indexes (APQI) assigned to a guest.
+ * Register the matrix device with the device core
+* Register with the ap_bus for AP queue devices of type 10 (CEX4 and
+ newer) and to provide the vfio_ap driver's probe and remove callback
+ interfaces. Devices older than CEX4 queues are not supported to simplify the
+ implementation and because older devices will be going out of service in the
+ relatively near future.
+* The admin needs to unbind AP Queues to be reserved for use by guests from
+ the cex4queue device driver and bind them to the vfio_ap device driver.
+
+
+Set up the VFIO mediated device interfaces
+------------------------------------------
+The VFIO AP device driver utilizes the common interface of the VFIO mediated
+device core driver to:
+* Register an AP mediated bus driver to add a mediated matrix device to and
+ remove it from a VFIO group.
+* Create and destroy a mediated matrix device
+* Add a mediated matrix device to and remove it from the AP mediated bus driver
+* Add a mediated matrix device to and remove it from an IOMMU group
+
+The following high-level block diagram shows the main components and interfaces
+of the VFIO AP mediated matrix device driver:
+
+ +-------------+
+ | |
+ | +---------+ | mdev_register_driver() +--------------+
+ | | Mdev | +<-----------------------+ |
+ | | bus | | | vfio_mdev.ko |
+ | | driver | +----------------------->+ |<-> VFIO user
+ | +---------+ | probe()/remove() +--------------+ APIs
+ | |
+ | MDEV CORE |
+ | MODULE |
+ | mdev.ko |
+ | +---------+ | mdev_register_device() +--------------+
+ | |Physical | +<-----------------------+ |
+ | | device | | | vfio_ap.ko |<-> matrix
+ | |interface| +----------------------->+ | device
+ | +---------+ | callback +--------------+
+ +-------------+
+
+During initialization of the vfio_ap module, the matrix device is registered
+with an 'mdev_parent_ops' structure that provides the sysfs attribute
+structures, mdev functions and callback interfaces for managing the mediated
+matrix device.
+
+* sysfs attribute structures:
+ * supported_type_groups
+ The VFIO mediated device framework supports creation of user-defined
+ mediated device types. These mediated device types are specified
+ via the 'supported_type_groups' structure when a device is registered
+ with the mediated device framework. The registration process creates the
+ sysfs structures for each mediated device type specified in the
+ 'mdev_supported_types' sub-directory of the device being registered. Along
+ with the device type, the sysfs attributes of the mediated device type are
+ provided.
+
+ The VFIO AP device driver will register one mediated device type for
+ passthrough devices:
+ /sys/devices/vfio_ap/matrix/mdev_supported_types/vfio_ap-passthrough
+ Only the read-only attributes required by the VFIO mdev framework will
+ be provided:
+ ... name
+ ... device_api
+ ... available_instances
+ ... device_api
+ Where:
+ * name: specifies the name of the mediated device type
+ * device_api: the mediated device type's API
+ * available_instances: the number of mediated matrix passthrough devices
+ that can be created
+ * device_api: specifies the VFIO API
+ * mdev_attr_groups
+ This attribute group identifies the user-defined sysfs attributes of the
+ mediated device. When a device is registered with the VFIO mediated device
+ framework, the sysfs attributes files identified in the 'mdev_attr_groups'
+ structure will be created in the mediated matrix device's directory. The
+ sysfs attributes for a mediated matrix device are:
+ * assign_adapter:
+ * unassign_adapter:
+ Write-only attributes for assigning/unassigning an AP adapter to/from the
+ mediated matrix device. To assign/unassign an adapter, the APID of the
+ adapter is written to the respective attribute file.
+ * assign_domain:
+ * unassign_domain:
+ Write-only attributes for assigning/unassigning an AP usage domain to/from
+ the mediated matrix device. To assign/unassign a domain, the APQI of the
+ AP queue corresponding to a usage domain is written to the respective
+ attribute file.
+ * matrix:
+ A read-only file for displaying the APQNs derived from the cross product
+ of the adapters and domains assigned to the mediated matrix device.
+ * assign_control_domain:
+ * unassign_control_domain:
+ Write-only attributes for assigning/unassigning an AP control domain
+ to/from the mediated matrix device. To assign/unassign a control domain,
+ the ID of a domain to be assigned/unassigned is written to the respective
+ attribute file.
+ * control_domains:
+ A read-only file for displaying the control domain numbers assigned to the
+ mediated matrix device.
+
+* functions:
+ * create:
+ allocates the ap_matrix_mdev structure used by the vfio_ap driver to:
+ * Store the reference to the KVM structure for the guest using the mdev
+ * Store the AP matrix configuration for the adapters, domains, and control
+ domains assigned via the corresponding sysfs attributes files
+ * remove:
+ deallocates the mediated matrix device's ap_matrix_mdev structure. This will
+ be allowed only if a running guest is not using the mdev.
+
+* callback interfaces
+ * open:
+ The vfio_ap driver uses this callback to register a
+ VFIO_GROUP_NOTIFY_SET_KVM notifier callback function for the mdev matrix
+ device. The open is invoked when QEMU connects the VFIO iommu group
+ for the mdev matrix device to the MDEV bus. Access to the KVM structure used
+ to configure the KVM guest is provided via this callback. The KVM structure,
+ is used to configure the guest's access to the AP matrix defined via the
+ mediated matrix device's sysfs attribute files.
+ * release:
+ unregisters the VFIO_GROUP_NOTIFY_SET_KVM notifier callback function for the
+ mdev matrix device and deconfigures the guest's AP matrix.
+
+Configure the APM, AQM and ADM in the CRYCB:
+-------------------------------------------
+Configuring the AP matrix for a KVM guest will be performed when the
+VFIO_GROUP_NOTIFY_SET_KVM notifier callback is invoked. The notifier
+function is called when QEMU connects to KVM. The CRYCB is configured by:
+* Setting the bits in the APM corresponding to the APIDs assigned to the
+ mediated matrix device via its 'assign_adapter' interface.
+* Setting the bits in the AQM corresponding to the APQIs assigned to the
+ mediated matrix device via its 'assign_domain' interface.
+* Setting the bits in the ADM corresponding to the domain dIDs assigned to the
+ mediated matrix device via its 'assign_control_domains' interface.
+
+The CPU model features for AP
+-----------------------------
+The AP stack relies on the presence of the AP instructions as well as two
+facilities: The AP Facilities Test (APFT) facility; and the AP Query
+Configuration Information (QCI) facility. These features/facilities are made
+available to a KVM guest via the following CPU model features:
+
+1. ap: Indicates whether the AP instructions are installed on the guest. This
+ feature will be enabled by KVM only if the AP instructions are installed
+ on the host.
+
+2. apft: Indicates the APFT facility is available on the guest. This facility
+ can be made available to the guest only if it is available on the host.
+
+3. apft: Indicates the AP QCI facility is available on the guest. This facility
+ can be made available to the guest only if it is available on the host.
+
+Note that if the user chooses to specify a CPU model different than the 'host'
+model to QEMU, the CPU model features and facilities need to be turned on
+explicitly; for example:
+
+ /usr/bin/qemu-system-s390x ... -cpu z13,ap=on,apqci=on,apft=on
+
+A guest can be precluded from using AP features/facilities by turning them off
+explicitly; for example:
+
+ /usr/bin/qemu-system-s390x ... -cpu host,ap=off,apqci=off,apft=off
+
+Example:
+=======
+Let's now provide an example to illustrate how KVM guests may be given
+access to AP facilities. For this example, we will show how to configure
+two guests such that executing the lszcrypt command on the guests would
+look like this:
+
+Guest1
+------
+CARD.DOMAIN TYPE MODE
+------------------------------
+05 CEX5C CCA-Coproc
+05.0004 CEX5C CCA-Coproc
+05.00ab CEX5C CCA-Coproc
+06 CEX5A Accelerator
+06.0004 CEX5A Accelerator
+06.00ab CEX5C CCA-Coproc
+
+Guest2
+------
+CARD.DOMAIN TYPE MODE
+------------------------------
+05 CEX5A Accelerator
+05.0047 CEX5A Accelerator
+05.00ff CEX5A Accelerator
+
+These are the steps:
+
+1. Install the vfio_ap module on the linux host. The dependency chain for the
+ vfio_ap module is:
+ * vfio
+ * mdev
+ * vfio_mdev
+ * KVM
+ * vfio_ap
+
+2. Secure the AP queues to be used by the two guests so that the host can not
+ access them. Only type 10 adapters (i.e., CEX4 and later) are supported
+ for the following reasons: To simplify the implementation; a lack of older
+ systems on which to test; and because the older hardware will go out of
+ service in a relatively short time.
+
+ To secure the AP queues each, each AP Queue device must first be unbound from
+ the cex4queue device driver. The sysfs location of the driver is:
+
+ /sys/bus/ap
+ --- [drivers]
+ ------ [cex4queue]
+ --------- [05.0004]
+ --------- [05.0047]
+ --------- [05.00ab]
+ --------- [05.00ff]
+ --------- [06.0004]
+ --------- [06.00ab]
+ --------- unbind
+
+ To unbind AP queue 05.0004 for example;
+
+ echo 05.0004 > unbind
+
+ The AP queue devices must then be bound to the vfio_ap driver. The sysfs
+ location of the driver is:
+
+ /sys/bus/ap
+ --- [drivers]
+ ------ [cex4queue]
+ ---------- bind
+
+ To bind AP queue 05.0004 to the vfio_ap driver:
+
+ echo 05.0004 > bind
+
+ Take note that the AP queues bound to the vfio_ap driver will be available
+ for guest usage until the vfio_ap module is unloaded, or the host system is
+ shut down.
+
+3. Create the mediated devices needed to configure the AP matrixes for the
+ two guests and to provide an interface to the vfio_ap driver for
+ use by the guests:
+
+ /sys/devices/vfio_ap/matrix/
+ --- [mdev_supported_types]
+ ------ [vfio_ap-passthrough] (passthrough mediated matrix device type)
+ --------- create
+ --------- [devices]
+
+ To create the mediated devices for the two guests:
+
+ uuidgen > create
+ uuidgen > create
+
+ This will create two mediated devices in the [devices] subdirectory named
+ with the UUID written to the create attribute file. We call them $uuid1
+ and $uuid2:
+
+ /sys/devices/vfio_ap/matrix/
+ --- [mdev_supported_types]
+ ------ [vfio_ap-passthrough]
+ --------- [devices]
+ ------------ [$uuid1]
+ --------------- assign_adapter
+ --------------- assign_control_domain
+ --------------- assign_domain
+ --------------- matrix
+ --------------- unassign_adapter
+ --------------- unassign_control_domain
+ --------------- unassign_domain
+
+ ------------ [$uuid2]
+ --------------- assign_adapter
+ --------------- assign_control_domain
+ --------------- assign_domain
+ --------------- matrix
+ --------------- unassign_adapter
+ --------------------- unassign_control_domain
+ --------------------- unassign_domain
+
+4. The administrator now needs to configure the matrixes for mediated
+ devices $uuid1 (for Guest1) and $uuid2 (for Guest2).
+
+ This is how the matrix is configured for Guest1:
+
+ echo 5 > assign_adapter
+ echo 6 > assign_adapter
+ echo 4 > assign_domain
+ echo 0xab > assign_domain
+
+ For this implementation, all usage domains - i.e., domains assigned
+ via the assign_domain attribute file - will also be configured in the ADM
+ field of the KVM guest's CRYCB, so there is no need to assign control
+ domains here unless you want to assign control domains that are not
+ assigned as usage domains.
+
+ If a mistake is made configuring an adapter, domain or control domain,
+ you can use the unassign_xxx files to unassign the adapter, domain or
+ control domain.
+
+ To display the matrix configuration for Guest1:
+
+ cat matrix
+
+ This is how the matrix is configured for Guest2:
+
+ echo 5 > assign_adapter
+ echo 0x47 > assign_domain
+ echo 0xff > assign_domain
+
+ In order to successfully assign an adapter:
+
+ * All APQNs that can be derived from the adapter ID and the IDs of
+ the previously assigned domains must be bound to the vfio_ap device
+ driver. If no domains have yet been assigned, then there must be at least
+ one APQN with the specified APID bound to the vfio_ap driver.
+
+ No APQN that can be derived from the adapter ID and the IDs of the
+ previously assigned domains can be assigned to another mediated matrix
+ device.
+
+ In order to successfully assign a domain:
+
+ * All APQNs that can be derived from the domain ID and the IDs of
+ the previously assigned adapters must be bound to the vfio_ap device
+ driver. If no domains have yet been assigned, then there must be at least
+ one APQN with the specified APQI bound to the vfio_ap driver.
+
+ No APQN that can be derived from the domain ID and the IDs of the
+ previously assigned adapters can be assigned to another mediated matrix
+ device.
+
+5. Start Guest1:
+
+ /usr/bin/qemu-system-s390x ... -cpu xxx,ap=on,apqci=on,apft=on \
+ -device vfio-ap,sysfsdev=/sys/devices/vfio_ap/matrix/$uuid1 ...
+
+7. Start Guest2:
+
+ /usr/bin/qemu-system-s390x ... -cpu xxx,ap=on,apqci=on,apft=on \
+ -device vfio-ap,sysfsdev=/sys/devices/vfio_ap/matrix/$uuid2 ...
+
+When the guest is shut down, the mediated matrix device may be removed.
+
+Using our example again, to remove the mediated matrix device $uuid1:
+
+ /sys/devices/vfio_ap/matrix/
+ --- [mdev_supported_types]
+ ------ [vfio_ap-passthrough]
+ --------- [devices]
+ ------------ [$uuid1]
+ --------------- remove
+
+
+ echo 1 > remove
+
+ This will release all the AP queues configured for the mediated device and
+ remove all of the mdev matrix device's sysfs structures including the mdev
+ device itself. To recreate and reconfigure the mdev matrix device, all of the
+ steps starting with step 3 will have to be performed again. Note that the
+ remove will fail if a guest using the mdev is still running.
+
+ It is not necessary to remove an mdev matrix device, but one may want to
+ remove it if no guest will use it during the lifetime of the linux host. If
+ the mdev matrix device is removed, one may want to unbind the AP queues the
+ guest was using from the vfio_ap device driver and bind them back to the
+ default driver. Alternatively, the AP queues can be configured for another
+ mdev matrix (i.e., guest).
+
+
+Limitations
+===========
+* The KVM/kernel interfaces do not provide a way to prevent unbinding an AP
+ queue that is still assigned to a mediated device. Even if the device
+ 'remove' callback returns an error, the device core detaches the AP
+ queue from the VFIO AP driver. It is therefore incumbent upon the
+ administrator to make sure there is no mediated device to which the
+ APQN - for the AP queue being unbound - is assigned.
+
+* Hot plug/unplug of AP devices is not supported for guests.
+
+* Live guest migration is not supported for guests using AP devices.
\ No newline at end of file
diff --git a/MAINTAINERS b/MAINTAINERS
index f60dd56..beeff24 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -12428,6 +12428,7 @@ S: Supported
F: drivers/s390/crypto/vfio_ap_drv.c
F: drivers/s390/crypto/vfio_ap_private.h
F: drivers/s390/crypto/vfio_ap_ops.c
+F: Documentation/s390/vfio-ap.txt

S390 ZFCP DRIVER
M: Steffen Maier <[email protected]>
--
1.7.1

2018-08-14 08:53:02

[permalink] [raw]

Subject: Re: [PATCH v9 05/22] KVM: s390: vsie: simulate VCPU SIE entry/exit

On Mon, 13 Aug 2018 17:48:02 -0400
Tony Krowiak <[email protected]> wrote:

> From: David Hildenbrand <[email protected]>
>
> VCPU requests and VCPU blocking right now don't take care of the vSIE
> (as it was not necessary until now). But we want to have VCPU requests
> that will also be handled before running the vSIE again.
>
> So let's simulate a SIE entry when entering the vSIE loop and check
> for PROG_ flags. The existing infrastructure (e.g. exit_sie()) will then
> detect that the SIE (in form of the vSIE execution loop) is running and
> properly kick the vSIE CPU, resulting in it leaving the vSIE loop and
> therefore the vSIE interception handler, allowing it to handle VCPU
> requests.
>
> E.g. if we want to modify the crycb of the VCPU and make sure that any
> masks also get applied to the VSIE crycb shadow (which uses masks from the
> VCPU crycb), we will need a way to hinder the vSIE from running and make
> sure to process the updated crycb before reentering the vSIE again.
>
> Signed-off-by: David Hildenbrand <[email protected]>
> Signed-off-by: Tony Krowiak <[email protected]>
> Reviewed-by: Pierre Morel <[email protected]>
> ---
> arch/s390/kvm/kvm-s390.c | 9 ++++++++-
> arch/s390/kvm/kvm-s390.h | 1 +
> arch/s390/kvm/vsie.c | 20 ++++++++++++++++++--
> 3 files changed, 27 insertions(+), 3 deletions(-)

I think that is the RFC version of David's patch, not the v1? Again,
not really relevant for reviewing, but I hope that you test the final
version.

2018-08-14 09:13:04

[permalink] [raw]

Subject: Re: [PATCH v9 04/22] s390/zcrypt: Integrate ap_asm.h into include/asm/ap.h.

On Mon, 13 Aug 2018 17:48:01 -0400
Tony Krowiak <[email protected]> wrote:

> From: Harald Freudenberger <[email protected]>
>
> Move all the inline functions from the ap bus header
> file ap_asm.h into the in-kernel api header file
> arch/s390/include/asm/ap.h so that KVM can make use
> of all the low level AP functions.
>
> Signed-off-by: Harald Freudenberger <[email protected]>
> Signed-off-by: Christian Borntraeger <[email protected]>
> Signed-off-by: Tony Krowiak <[email protected]>
> ---
> arch/s390/include/asm/ap.h | 284 ++++++++++++++++++++++++++++++++++++----
> drivers/s390/crypto/ap_bus.c | 23 +---
> drivers/s390/crypto/ap_bus.h | 1 +
> drivers/s390/crypto/ap_card.c | 1 -
> drivers/s390/crypto/ap_queue.c | 1 -
> 5 files changed, 260 insertions(+), 50 deletions(-)
>
> diff --git a/arch/s390/include/asm/ap.h b/arch/s390/include/asm/ap.h
> index c1bedb4..887494a 100644
> --- a/arch/s390/include/asm/ap.h
> +++ b/arch/s390/include/asm/ap.h
> @@ -47,6 +47,50 @@ struct ap_queue_status {
> };
>
> /**
> + * ap_intructions_available() - Test if AP instructions are available.
> + *
> + * Returns 1 if the AP instructions are installed, otherwise 0.
> + */
> +static inline int ap_instructions_available(void)
> +{
> + register unsigned long reg0 asm ("0") = AP_MKQID(0, 0);
> + register unsigned long reg1 asm ("1") = 0;
> + register unsigned long reg2 asm ("2") = 0;
> +
> + asm volatile(
> + " .long 0xb2af0000\n" /* PQAP(TAPQ) */
> + "0: la %0,1\n"
> + "1:\n"
> + EX_TABLE(0b, 1b)
> + : "+d" (reg1), "+d" (reg2)
> + : "d" (reg0)
> + : "cc");
> + return reg1;
> +}

I think upstream this change (have this function return !0 when the
instructions are installed) will be an addon patch to the one which has
already made its way into master. Not really relevant for the remainder
of this patch series, though.

2018-08-14 10:44:48

[permalink] [raw]

Subject: Re: [PATCH v9 08/22] s390: vfio-ap: base implementation of VFIO AP device driver

On Mon, 13 Aug 2018 17:48:05 -0400
Tony Krowiak <[email protected]> wrote:

> diff --git a/drivers/s390/crypto/vfio_ap_drv.c b/drivers/s390/crypto/vfio_ap_drv.c
> new file mode 100644
> index 0000000..5069580
> --- /dev/null
> +++ b/drivers/s390/crypto/vfio_ap_drv.c
> @@ -0,0 +1,118 @@
> +// SPDX-License-Identifier: GPL-2.0+
> +/*
> + * VFIO based AP device driver
> + *
> + * Copyright IBM Corp. 2018
> + *
> + * Author(s): Tony Krowiak <[email protected]>
> + */
> +
> +#include <linux/module.h>
> +#include <linux/mod_devicetable.h>
> +#include <linux/slab.h>
> +#include <linux/string.h>
> +#include "vfio_ap_private.h"
> +
> +#define VFIO_AP_ROOT_NAME "vfio_ap"
> +#define VFIO_AP_DEV_TYPE_NAME "ap_matrix"
> +#define VFIO_AP_DEV_NAME "matrix"
> +
> +MODULE_AUTHOR("IBM Corporation");
> +MODULE_DESCRIPTION("VFIO AP device driver, Copyright IBM Corp. 2018");
> +MODULE_LICENSE("GPL v2");
> +
> +static struct ap_driver vfio_ap_drv;
> +
> +static struct device_type vfio_ap_dev_type = {
> + .name = VFIO_AP_DEV_TYPE_NAME,
> +};
> +
> +struct ap_matrix_dev matrix_dev;

Please don't add new statically allocated devices, but allocate it
dynamically (see the comment in device_add()).

> +
> +/* Only type 10 adapters (CEX4 and later) are supported
> + * by the AP matrix device driver
> + */
> +static struct ap_device_id ap_queue_ids[] = {
> + { .dev_type = AP_DEVICE_TYPE_CEX4,
> + .match_flags = AP_DEVICE_ID_MATCH_QUEUE_TYPE },
> + { .dev_type = AP_DEVICE_TYPE_CEX5,
> + .match_flags = AP_DEVICE_ID_MATCH_QUEUE_TYPE },
> + { .dev_type = AP_DEVICE_TYPE_CEX6,
> + .match_flags = AP_DEVICE_ID_MATCH_QUEUE_TYPE },
> + { /* end of sibling */ },
> +};
> +
> +MODULE_DEVICE_TABLE(vfio_ap, ap_queue_ids);
> +
> +static int vfio_ap_queue_dev_probe(struct ap_device *apdev)
> +{
> + return 0;
> +}
> +
> +static void vfio_ap_queue_dev_remove(struct ap_device *apdev)
> +{
> + /* Nothing to do yet */
> +}

You need a release callback as well.

> +
> +static int vfio_ap_matrix_dev_init(void)
> +{
> + int ret;
> + struct device *root_device;
> +
> + root_device = root_device_register(VFIO_AP_ROOT_NAME);
> + if (IS_ERR(root_device)) {
> + ret = PTR_ERR(root_device);
> + return ret;
> + }
> +
> + matrix_dev.device.type = &vfio_ap_dev_type;
> + dev_set_name(&matrix_dev.device, "%s", VFIO_AP_DEV_NAME);
> + matrix_dev.device.type = &vfio_ap_dev_type;
> + matrix_dev.device.parent = root_device;
> + matrix_dev.device.driver = &vfio_ap_drv.driver;
> +
> + ret = device_register(&matrix_dev.device);
> + if (ret) {
> + root_device_unregister(root_device);

And this needs a put_device() for the matrix device. (It is getting
ugly with a statically allocated device.)

> + return ret;
> + }
> +
> + return 0;
> +}
> +
> +static void vfio_ap_matrix_dev_destroy(void)
> +{
> + device_unregister(&matrix_dev.device);

This one already does a put_device(). Didn't the driver core complain?

> + root_device_unregister(matrix_dev.device.parent);
> +}

2018-08-14 12:01:05

[permalink] [raw]

Subject: Re: [PATCH v9 09/22] s390: vfio-ap: register matrix device with VFIO mdev framework

On Mon, 13 Aug 2018 17:48:06 -0400
Tony Krowiak <[email protected]> wrote:

> From: Tony Krowiak <[email protected]>
>
> Registers the matrix device created by the VFIO AP device
> driver with the VFIO mediated device framework.
> Registering the matrix device will create the sysfs
> structures needed to create mediated matrix devices
> each of which will be used to configure the AP matrix
> for a guest and connect it to the VFIO AP device driver.
>
> Registering the matrix device with the VFIO mediated device
> framework will create the following sysfs structures:
>
> /sys/devices/vfio_ap/matrix/
> ...... [mdev_supported_types]
> ......... [vfio_ap-passthrough]
> ............ create
>
> To create a mediated device for the AP matrix device, write a UUID
> to the create file:
>
> uuidgen > create
>
> A symbolic link to the mediated device's directory will be created in the
> devices subdirectory named after the generated $uuid:
>
> /sys/devices/vfio_ap/matrix/
> ...... [mdev_supported_types]
> ......... [vfio_ap-passthrough]
> ............ [devices]
> ............... [$uuid]
>
> A symbolic link to the mediated device will also be created
> in the vfio_ap matrix's directory:
>
> /sys/devices/vfio_ap/matrix/[$uuid]
>
> Signed-off-by: Tony Krowiak <[email protected]>
> Reviewed-by: Halil Pasic <[email protected]>
> Tested-by: Michael Mueller <[email protected]>
> Tested-by: Farhan Ali <[email protected]>
> Signed-off-by: Christian Borntraeger <[email protected]>
> ---
> MAINTAINERS | 1 +
> drivers/s390/crypto/Makefile | 2 +-
> drivers/s390/crypto/vfio_ap_drv.c | 23 ++++++
> drivers/s390/crypto/vfio_ap_ops.c | 124 +++++++++++++++++++++++++++++++++
> drivers/s390/crypto/vfio_ap_private.h | 45 ++++++++++++
> include/uapi/linux/vfio.h | 1 +
> 6 files changed, 195 insertions(+), 1 deletions(-)
> create mode 100644 drivers/s390/crypto/vfio_ap_ops.c
>

> diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c
> new file mode 100644
> index 0000000..8018c2d
> --- /dev/null
> +++ b/drivers/s390/crypto/vfio_ap_ops.c
> @@ -0,0 +1,124 @@
> +// SPDX-License-Identifier: GPL-2.0+
> +/*
> + * Adjunct processor matrix VFIO device driver callbacks.
> + *
> + * Copyright IBM Corp. 2018
> + *
> + * Author(s): Tony Krowiak <[email protected]>
> + * Halil Pasic <[email protected]>
> + * Pierre Morel <[email protected]>
> + */
> +#include <linux/string.h>
> +#include <linux/vfio.h>
> +#include <linux/device.h>
> +#include <linux/list.h>
> +#include <linux/ctype.h>
> +
> +#include "vfio_ap_private.h"
> +
> +#define VFIO_AP_MDEV_TYPE_HWVIRT "passthrough"
> +#define VFIO_AP_MDEV_NAME_HWVIRT "VFIO AP Passthrough Device"
> +
> +static void vfio_ap_matrix_init(struct ap_config_info *info,
> + struct ap_matrix *matrix)
> +{
> + matrix->apm_max = info->apxa ? info->Na : 63;
> + matrix->aqm_max = info->apxa ? info->Nd : 15;
> + matrix->adm_max = info->apxa ? info->Nd : 15;
> +}
> +
> +static int vfio_ap_mdev_create(struct kobject *kobj, struct mdev_device *mdev)
> +{
> + struct ap_matrix_mdev *matrix_mdev;
> +
> + matrix_mdev = kzalloc(sizeof(*matrix_mdev), GFP_KERNEL);
> + if (!matrix_mdev)
> + return -ENOMEM;
> +
> + matrix_mdev->name = dev_name(mdev_dev(mdev));
> + vfio_ap_matrix_init(&matrix_dev.info, &matrix_mdev->matrix);
> + mdev_set_drvdata(mdev, matrix_mdev);
> +
> + if (atomic_dec_if_positive(&matrix_dev.available_instances) < 0) {
> + kfree(matrix_mdev);
> + return -EPERM;
> + }

Maybe move this check to the top of the function?

> +
> + mutex_lock(&matrix_dev.lock);
> + list_add(&matrix_mdev->list, &matrix_dev.mdev_list);
> + mutex_unlock(&matrix_dev.lock);
> +
> + return 0;
> +}

2018-08-14 12:48:24

[permalink] [raw]

Subject: Re: [PATCH v9 05/22] KVM: s390: vsie: simulate VCPU SIE entry/exit

On 08/14/2018 04:50 AM, Cornelia Huck wrote:
> On Mon, 13 Aug 2018 17:48:02 -0400
> Tony Krowiak <[email protected]> wrote:
>
>> From: David Hildenbrand <[email protected]>
>>
>> VCPU requests and VCPU blocking right now don't take care of the vSIE
>> (as it was not necessary until now). But we want to have VCPU requests
>> that will also be handled before running the vSIE again.
>>
>> So let's simulate a SIE entry when entering the vSIE loop and check
>> for PROG_ flags. The existing infrastructure (e.g. exit_sie()) will then
>> detect that the SIE (in form of the vSIE execution loop) is running and
>> properly kick the vSIE CPU, resulting in it leaving the vSIE loop and
>> therefore the vSIE interception handler, allowing it to handle VCPU
>> requests.
>>
>> E.g. if we want to modify the crycb of the VCPU and make sure that any
>> masks also get applied to the VSIE crycb shadow (which uses masks from the
>> VCPU crycb), we will need a way to hinder the vSIE from running and make
>> sure to process the updated crycb before reentering the vSIE again.
>>
>> Signed-off-by: David Hildenbrand <[email protected]>
>> Signed-off-by: Tony Krowiak <[email protected]>
>> Reviewed-by: Pierre Morel <[email protected]>
>> ---
>> arch/s390/kvm/kvm-s390.c | 9 ++++++++-
>> arch/s390/kvm/kvm-s390.h | 1 +
>> arch/s390/kvm/vsie.c | 20 ++++++++++++++++++--
>> 3 files changed, 27 insertions(+), 3 deletions(-)
> I think that is the RFC version of David's patch, not the v1? Again,
> not really relevant for reviewing, but I hope that you test the final
> version.

Will do.

>

2018-08-14 23:32:20

[permalink] [raw]

Subject: Re: [PATCH v9 08/22] s390: vfio-ap: base implementation of VFIO AP device driver

On 08/14/2018 06:42 AM, Cornelia Huck wrote:
> On Mon, 13 Aug 2018 17:48:05 -0400
> Tony Krowiak <[email protected]> wrote:
>
>
>> diff --git a/drivers/s390/crypto/vfio_ap_drv.c b/drivers/s390/crypto/vfio_ap_drv.c
>> new file mode 100644
>> index 0000000..5069580
>> --- /dev/null
>> +++ b/drivers/s390/crypto/vfio_ap_drv.c
>> @@ -0,0 +1,118 @@
>> +// SPDX-License-Identifier: GPL-2.0+
>> +/*
>> + * VFIO based AP device driver
>> + *
>> + * Copyright IBM Corp. 2018
>> + *
>> + * Author(s): Tony Krowiak <[email protected]>
>> + */
>> +
>> +#include <linux/module.h>
>> +#include <linux/mod_devicetable.h>
>> +#include <linux/slab.h>
>> +#include <linux/string.h>
>> +#include "vfio_ap_private.h"
>> +
>> +#define VFIO_AP_ROOT_NAME "vfio_ap"
>> +#define VFIO_AP_DEV_TYPE_NAME "ap_matrix"
>> +#define VFIO_AP_DEV_NAME "matrix"
>> +
>> +MODULE_AUTHOR("IBM Corporation");
>> +MODULE_DESCRIPTION("VFIO AP device driver, Copyright IBM Corp. 2018");
>> +MODULE_LICENSE("GPL v2");
>> +
>> +static struct ap_driver vfio_ap_drv;
>> +
>> +static struct device_type vfio_ap_dev_type = {
>> + .name = VFIO_AP_DEV_TYPE_NAME,
>> +};
>> +
>> +struct ap_matrix_dev matrix_dev;
> Please don't add new statically allocated devices, but allocate it
> dynamically (see the comment in device_add()).

Okay, I'll dynamically allocate it.

>
>> +
>> +/* Only type 10 adapters (CEX4 and later) are supported
>> + * by the AP matrix device driver
>> + */
>> +static struct ap_device_id ap_queue_ids[] = {
>> + { .dev_type = AP_DEVICE_TYPE_CEX4,
>> + .match_flags = AP_DEVICE_ID_MATCH_QUEUE_TYPE },
>> + { .dev_type = AP_DEVICE_TYPE_CEX5,
>> + .match_flags = AP_DEVICE_ID_MATCH_QUEUE_TYPE },
>> + { .dev_type = AP_DEVICE_TYPE_CEX6,
>> + .match_flags = AP_DEVICE_ID_MATCH_QUEUE_TYPE },
>> + { /* end of sibling */ },
>> +};
>> +
>> +MODULE_DEVICE_TABLE(vfio_ap, ap_queue_ids);
>> +
>> +static int vfio_ap_queue_dev_probe(struct ap_device *apdev)
>> +{
>> + return 0;
>> +}
>> +
>> +static void vfio_ap_queue_dev_remove(struct ap_device *apdev)
>> +{
>> + /* Nothing to do yet */
>> +}
> You need a release callback as well.

Will do.

>
>> +
>> +static int vfio_ap_matrix_dev_init(void)
>> +{
>> + int ret;
>> + struct device *root_device;
>> +
>> + root_device = root_device_register(VFIO_AP_ROOT_NAME);
>> + if (IS_ERR(root_device)) {
>> + ret = PTR_ERR(root_device);
>> + return ret;
>> + }
>> +
>> + matrix_dev.device.type = &vfio_ap_dev_type;
>> + dev_set_name(&matrix_dev.device, "%s", VFIO_AP_DEV_NAME);
>> + matrix_dev.device.type = &vfio_ap_dev_type;
>> + matrix_dev.device.parent = root_device;
>> + matrix_dev.device.driver = &vfio_ap_drv.driver;
>> +
>> + ret = device_register(&matrix_dev.device);
>> + if (ret) {
>> + root_device_unregister(root_device);
> And this needs a put_device() for the matrix device. (It is getting
> ugly with a statically allocated device.)

Will do.

>
>> + return ret;
>> + }
>> +
>> + return 0;
>> +}
>> +
>> +static void vfio_ap_matrix_dev_destroy(void)
>> +{
>> + device_unregister(&matrix_dev.device);
> This one already does a put_device(). Didn't the driver core complain?

The driver core did not complain.

>
>> + root_device_unregister(matrix_dev.device.parent);
>> +}

2018-08-15 09:53:58

[permalink] [raw]

Subject: Re: [PATCH v9 10/22] s390: vfio-ap: sysfs interfaces to configure adapters

On Mon, 13 Aug 2018 17:48:07 -0400
Tony Krowiak <[email protected]> wrote:

> +/**
> + * assign_adapter_store
> + *
> + * @dev: the matrix device
> + * @attr: a mediated matrix device attribute
> + * @buf: a buffer containing the adapter ID (APID) to be assigned
> + * @count: the number of bytes in @buf
> + *
> + * Parses the APID from @buf and assigns it to the mediated matrix device.
> + *
> + * Returns the number of bytes processed if the APID is valid; otherwise returns
> + * an error.
> + */
> +static ssize_t assign_adapter_store(struct device *dev,
> + struct device_attribute *attr,
> + const char *buf, size_t count)
> +{
> + int ret = 0;

You don't need to initialize this to 0, as kstrtoul will set it in any
case.

> + unsigned long apid;
> + struct mdev_device *mdev = mdev_from_dev(dev);
> + struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
> + unsigned long max_apid = matrix_mdev->matrix.apm_max;
> +
> + ret = kstrtoul(buf, 0, &apid);
> + if (ret)
> + return ret;
> + if (apid > max_apid)
> + return -EINVAL;
> +
> + /* Set the bit in the AP mask (APM) corresponding to the AP adapter
> + * number (APID). The bits in the mask, from most significant to least
> + * significant bit, correspond to APIDs 0-255.
> + */
> + mutex_lock(&matrix_dev.lock);
> +
> + ret = vfio_ap_mdev_verify_queues_reserved_for_apid(matrix_mdev, apid);

That function name really is a mouthful :) I don't have any better
suggestions, though.

> + if (ret)
> + goto done;
> +
> + set_bit_inv(apid, matrix_mdev->matrix.apm);
> +
> + ret = vfio_ap_mdev_verify_no_sharing(matrix_mdev);
> + if (ret)
> + goto share_err;
> +
> + ret = count;
> + goto done;
> +
> +share_err:
> + clear_bit_inv(apid, matrix_mdev->matrix.apm);
> +done:
> + mutex_unlock(&matrix_dev.lock);
> +
> + return ret;
> +}
> +static DEVICE_ATTR_WO(assign_adapter);
> +
> +/**
> + * unassign_adapter_store
> + *
> + * @dev: the matrix device
> + * @attr: a mediated matrix device attribute
> + * @buf: a buffer containing the adapter ID (APID) to be assigned
> + * @count: the number of bytes in @buf
> + *
> + * Parses the APID from @buf and unassigns it from the mediated matrix device.
> + * The APID must be a valid value

A valid value, but not necessarily assigned, right?

> + *
> + * Returns the number of bytes processed if the APID is valid; otherwise returns
> + * an error.
> + */
> +static ssize_t unassign_adapter_store(struct device *dev,
> + struct device_attribute *attr,
> + const char *buf, size_t count)
> +{
> + int ret;
> + unsigned long apid;
> + struct mdev_device *mdev = mdev_from_dev(dev);
> + struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
> +
> + ret = kstrtoul(buf, 0, &apid);
> + if (ret)
> + return ret;
> +
> + if (apid > matrix_mdev->matrix.apm_max)
> + return -EINVAL;
> +
> + mutex_lock(&matrix_dev.lock);
> + clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm);
> + mutex_unlock(&matrix_dev.lock);
> +
> + return count;
> +}
> +DEVICE_ATTR_WO(unassign_adapter);

In general, looks good to me.

2018-08-15 12:06:25

[permalink] [raw]

Subject: Re: [PATCH v9 11/22] s390: vfio-ap: sysfs interfaces to configure domains

On Mon, 13 Aug 2018 17:48:08 -0400
Tony Krowiak <[email protected]> wrote:

> From: Tony Krowiak <[email protected]>
>
> Introduces two new sysfs attributes for the VFIO mediated
> matrix device for assigning AP domains to and unassigning
> AP domains from a mediated matrix device. The IDs of the
> AP domains assigned to the mediated matrix device will be
> stored in an AP queue mask (AQM).
>
> The bits in the AQM, from most significant to least
> significant bit, correspond to AP queue index (APQI) 0 to
> 255 (note that an APQI is synonymous with with a domain ID).
> On some systems, the maximum allowable domain number may be
> less than 255 - depending upon the host's AP configuration -
> and assignment may be rejected if the input domain ID exceeds
> the limit.
>
> When a domain is assigned, the bit corresponding to the APQI
> will be set in the AQM. Likewise, when a domain is unassigned,
> the bit corresponding to the APQI will be cleared from the AQM.
>
> In order to successfully assign a domain, the APQNs derived from
> the domain ID being assigned and the adapter numbers of all
> adapters previously assigned:
>
> 1. Must be bound to the vfio_ap device driver.
>
> 2. Must not be assigned to any other mediated matrix device.
>
> If there are no adapters assigned to the mdev, then there must
> be an AP queue bound to the vfio_ap device driver with an
> APQN containing the domain ID (i.e., APQI), otherwise all
> adapters subsequently assigned will fail because there will be no
> AP queues bound with an APQN containing the APQI.
>
> Assigning or un-assigning an AP domain will also be rejected if
> a guest using the mediated matrix device is running.
>
> The relevant sysfs structures are:
>
> /sys/devices/vfio_ap/matrix/
> ...... [mdev_supported_types]
> ......... [vfio_ap-passthrough]
> ............ [devices]
> ...............[$uuid]
> .................. assign_domain
> .................. unassign_domain
>
> To assign a domain to the $uuid mediated matrix device,
> write the domain's ID to the assign_domain file. To
> unassign a domain, write the domain's ID to the
> unassign_domain file. The ID is specified using
> conventional semantics: If it begins with 0x, the number
> will be parsed as a hexadecimal (case insensitive) number;
> if it begins with 0, it will be parsed as an octal number;
> otherwise, it will be parsed as a decimal number.
>
> For example, to assign domain 173 (0xad) to the mediated matrix
> device $uuid:
>
> echo 173 > assign_domain
>
> or
>
> echo 0255 > assign_domain
>
> or
>
> echo 0xad > assign_domain
>
> To unassign domain 173 (0xad):
>
> echo 173 > unassign_domain
>
> or
>
> echo 0255 > unassign_domain
>
> or
>
> echo 0xad > unassign_domain
>
> Signed-off-by: Tony Krowiak <[email protected]>
> Reviewed-by: Halil Pasic <[email protected]>
> Tested-by: Michael Mueller <[email protected]>
> Tested-by: Farhan Ali <[email protected]>
> Tested-by: Pierre Morel <[email protected]>
> Signed-off-by: Christian Borntraeger <[email protected]>
> ---
> drivers/s390/crypto/vfio_ap_ops.c | 88 ++++++++++++++++++++++++++++++++++++-
> 1 files changed, 87 insertions(+), 1 deletions(-)

Looks sane to me.

2018-08-15 13:11:51

[permalink] [raw]

Subject: Re: [PATCH v9 14/22] KVM: s390: interfaces to clear CRYCB masks

On Mon, 13 Aug 2018 17:48:11 -0400
Tony Krowiak <[email protected]> wrote:

> From: Tony Krowiak <[email protected]>
>
> Introduces two new KVM interface to clear the APM, AQM and ADM masks in
> the guest's CRYCB. The VCPUs are taken out of SIE to ensure the VCPUs do
> not get out of sync.
>
> Signed-off-by: Tony Krowiak <[email protected]>
> Acked-by: Halil Pasic <[email protected]>
> Tested-by: Michael Mueller <[email protected]>
> Tested-by: Farhan Ali <[email protected]>
> Tested-by: Pierre Morel <[email protected]>
> Signed-off-by: Christian Borntraeger <[email protected]>
> ---
> arch/s390/include/asm/kvm_host.h | 3 +++
> arch/s390/kvm/kvm-s390.c | 15 +++++++++++++++
> 2 files changed, 18 insertions(+), 0 deletions(-)
>
> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
> index 4a739d4..07e58d8 100644
> --- a/arch/s390/include/asm/kvm_host.h
> +++ b/arch/s390/include/asm/kvm_host.h
> @@ -258,6 +258,7 @@ struct kvm_s390_sie_block {
> __u64 tecmc; /* 0x00e8 */
> __u8 reservedf0[12]; /* 0x00f0 */
> #define CRYCB_FORMAT_MASK 0x00000003
> +#define CRYCB_FORMAT0 0x00000000

This should probably go into a different patch (define is not used
here)?

> #define CRYCB_FORMAT1 0x00000001
> #define CRYCB_FORMAT2 0x00000003
> __u32 crycbd; /* 0x00fc */
> @@ -861,6 +862,8 @@ void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
> void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
> struct kvm_async_pf *work);
>
> +void kvm_arch_crypto_clear_masks(struct kvm *kvm);
> +
> extern int sie64a(struct kvm_s390_sie_block *, u64 *);
> extern char sie_exit;
>
> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
> index f7de123..8d8a65a 100644
> --- a/arch/s390/kvm/kvm-s390.c
> +++ b/arch/s390/kvm/kvm-s390.c
> @@ -1919,6 +1919,21 @@ static void kvm_s390_set_crycb_format(struct kvm *kvm)
> kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
> }
>
> +void kvm_arch_crypto_clear_masks(struct kvm *kvm)
> +{
> + mutex_lock(&kvm->lock);
> + kvm_s390_vcpu_block_all(kvm);
> +
> + memset(&kvm->arch.crypto.crycb->apcb0, 0,
> + sizeof(kvm->arch.crypto.crycb->apcb0));
> + memset(&kvm->arch.crypto.crycb->apcb1, 0,
> + sizeof(kvm->arch.crypto.crycb->apcb1));
> +
> + kvm_s390_vcpu_unblock_all(kvm);
> + mutex_unlock(&kvm->lock);
> +}
> +EXPORT_SYMBOL(kvm_arch_crypto_clear_masks);

A quick grep shows that kvm-related exports tend to be
EXPORT_SYMBOL_GPL. Should that also be done here?

> +
> static u64 kvm_s390_get_initial_cpuid(void)
> {
> struct cpuid cpuid;

2018-08-15 16:09:35

[permalink] [raw]

Subject: Re: [PATCH v9 15/22] s390: vfio-ap: implement mediated device open callback

On Mon, 13 Aug 2018 17:48:12 -0400
Tony Krowiak <[email protected]> wrote:

> From: Tony Krowiak <[email protected]>
>
> Implements the open callback on the mediated matrix device.
> The function registers a group notifier to receive notification
> of the VFIO_GROUP_NOTIFY_SET_KVM event. When notified,
> the vfio_ap device driver will get access to the guest's
> kvm structure. The open callback must ensure that only one
> mediated device shall be opened per guest.
>
> Signed-off-by: Tony Krowiak <[email protected]>
> Acked-by: Halil Pasic <[email protected]>
> Tested-by: Michael Mueller <[email protected]>
> Tested-by: Farhan Ali <[email protected]>
> Tested-by: Pierre Morel <[email protected]>
> Acked-by: Pierre Morel <[email protected]>
> Signed-off-by: Christian Borntraeger <[email protected]>
> ---
> drivers/s390/crypto/vfio_ap_ops.c | 174 ++++++++++++++++++++++++++++++++-
> drivers/s390/crypto/vfio_ap_private.h | 2 +
> 2 files changed, 175 insertions(+), 1 deletions(-)

> @@ -602,7 +633,6 @@ static ssize_t matrix_show(struct device *dev, struct device_attribute *attr,
> }
> DEVICE_ATTR_RO(matrix);
>
> -

Nit: whitespace change

> static struct attribute *vfio_ap_mdev_attrs[] = {
> &dev_attr_assign_adapter.attr,
> &dev_attr_unassign_adapter.attr,

(...)

> +/**
> + * vfio_ap_mdev_open_once
> + *
> + * @matrix_mdev: a mediated matrix device
> + *
> + * Return 0 if no other mediated matrix device has been opened for the
> + * KVM guest assigned to @matrix_mdev; otherwise, returns an error.
> + */
> +static int vfio_ap_mdev_open_once(struct ap_matrix_mdev *matrix_mdev,
> + struct kvm *kvm)

You're passing kvm in here, but do not use it.

> +{
> + struct ap_matrix_mdev *m;
> +
> + mutex_lock(&matrix_dev.lock);
> +
> + list_for_each_entry(m, &matrix_dev.mdev_list, list) {
> + if ((m != matrix_mdev) && (m->kvm == matrix_mdev->kvm)) {

If you used it here instead of matrix_mdev->kvm...

> + mutex_unlock(&matrix_dev.lock);
> + return -EPERM;
> + }
> + }
> +
> + mutex_unlock(&matrix_dev.lock);
> +
> + return 0;
> +}
> +
> +static int vfio_ap_mdev_group_notifier(struct notifier_block *nb,
> + unsigned long action, void *data)
> +{
> + int ret;
> + struct ap_matrix_mdev *matrix_mdev;
> +
> + if (action != VFIO_GROUP_NOTIFY_SET_KVM)
> + return NOTIFY_OK;
> +
> + matrix_mdev = container_of(nb, struct ap_matrix_mdev, group_notifier);
> +
> + matrix_mdev->kvm = data;
> + if (data == NULL)
> + return NOTIFY_OK;
> +
> + ret = vfio_ap_mdev_open_once(matrix_mdev, data);

...you could move this up to before overwriting matrix_mdev->kvm and
bailing out when the check failed, which makes more sense to me.

> + if (ret)
> + return ret;
> +
> + ret = kvm_ap_validate_crypto_setup(matrix_mdev->kvm);
> + if (ret)
> + return ret;

It probably makes more sense to return NOTIFY_DONE in the error case
(NOTIFY_BAD does not sound like a good idea as it would stop processing
the notifier chain).

> +
> + vfio_ap_mdev_copy_masks(matrix_mdev);
> +
> + return NOTIFY_OK;
> +}

Otherwise, looks sane.

2018-08-15 16:25:25

[permalink] [raw]

Subject: Re: [PATCH v9 17/22] s390: vfio-ap: zeroize the AP queues.

On Mon, 13 Aug 2018 17:48:14 -0400
Tony Krowiak <[email protected]> wrote:

Nit: please drop the leading period in the subject.

> From: Tony Krowiak <[email protected]>
>
> Let's call PAPQ(ZAPQ) to zeroize a queue:
>
> * For each queue configured for a mediated matrix device
> when it is released.
>
> Zeroizing a queue resets the queue, clears all pending
> messages for the queue entries and disables adapter interruptions
> associated with the queue.
>
> Signed-off-by: Tony Krowiak <[email protected]>
> Reviewed-by: Halil Pasic <[email protected]>
> Tested-by: Michael Mueller <[email protected]>
> Tested-by: Farhan Ali <[email protected]>
> Signed-off-by: Christian Borntraeger <[email protected]>
> ---
> drivers/s390/crypto/vfio_ap_ops.c | 25 +++++++++++++++++++++++++
> drivers/s390/crypto/vfio_ap_private.h | 25 +++++++++++++++++++++++++
> 2 files changed, 50 insertions(+), 0 deletions(-)
>

> diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h
> index 3e8534b..34f982a 100644
> --- a/drivers/s390/crypto/vfio_ap_private.h
> +++ b/drivers/s390/crypto/vfio_ap_private.h
> @@ -74,4 +74,29 @@ struct ap_matrix_mdev {
> extern int vfio_ap_mdev_register(void);
> extern void vfio_ap_mdev_unregister(void);
>
> +static inline int vfio_ap_reset_queue(unsigned int apid, unsigned int apqi,
> + unsigned int retry)
> +{
> + struct ap_queue_status status;
> +
> + do {
> + status = ap_zapq(AP_MKQID(apid, apqi));
> + switch (status.response_code) {
> + case AP_RESPONSE_NORMAL:
> + return 0;
> + case AP_RESPONSE_RESET_IN_PROGRESS:
> + case AP_RESPONSE_BUSY:
> + msleep(20);
> + break;
> + default:
> + pr_warn("%s: error zeroizing %02x.%04x: response code %d\n",
> + VFIO_AP_MODULE_NAME, apid, apqi,
> + status.response_code);

How can we end up here? Does this mean that we just don't know what to
do with this response, or is this something that should never happen?
(How much sense does it make to print an error?)

> + return -EIO;
> + }
> + } while (retry--);
> +
> + return -EBUSY;
> +}
> +
> #endif /* _VFIO_AP_PRIVATE_H_ */

2018-08-15 16:39:48

[permalink] [raw]

Subject: Re: [PATCH v9 18/22] s390: vfio-ap: implement VFIO_DEVICE_RESET ioctl

On Mon, 13 Aug 2018 17:48:15 -0400
Tony Krowiak <[email protected]> wrote:

> From: Tony Krowiak <[email protected]>
>
> Implements the VFIO_DEVICE_RESET ioctl. This ioctl zeroizes
> all of the AP queues assigned to the guest.
>
> Signed-off-by: Tony Krowiak <[email protected]>
> Reviewed-by: Halil Pasic <[email protected]>
> Tested-by: Michael Mueller <[email protected]>
> Tested-by: Farhan Ali <[email protected]>
> Reviewed-by: Pierre Morel <[email protected]>
> Tested-by: Pierre Morel <[email protected]>
> Signed-off-by: Christian Borntraeger <[email protected]>
> ---
> drivers/s390/crypto/vfio_ap_ops.c | 5 ++++-
> 1 files changed, 4 insertions(+), 1 deletions(-)
>
> diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c
> index cc90323..d4a065b 100644
> --- a/drivers/s390/crypto/vfio_ap_ops.c
> +++ b/drivers/s390/crypto/vfio_ap_ops.c
> @@ -832,7 +832,7 @@ static int vfio_ap_mdev_get_device_info(unsigned long arg)
> if (info.argsz < minsz)
> return -EINVAL;
>
> - info.flags = VFIO_DEVICE_FLAGS_AP;
> + info.flags = VFIO_DEVICE_FLAGS_AP | VFIO_DEVICE_FLAGS_RESET;
> info.num_regions = 0;
> info.num_irqs = 0;
>
> @@ -848,6 +848,9 @@ static ssize_t vfio_ap_mdev_ioctl(struct mdev_device *mdev,
> case VFIO_DEVICE_GET_INFO:
> ret = vfio_ap_mdev_get_device_info(arg);
> break;
> + case VFIO_DEVICE_RESET:
> + ret = vfio_ap_mdev_reset_queues(mdev, true);

If I see it correctly, you call this function only ever with force ==
true (here and in the previous patch). Is that what you intended?

> + break;
> default:
> ret = -EOPNOTSUPP;
> break;

2018-08-15 16:53:29

[permalink] [raw]

Subject: Re: [PATCH v9 09/22] s390: vfio-ap: register matrix device with VFIO mdev framework

On 08/14/2018 07:19 AM, Cornelia Huck wrote:
> On Mon, 13 Aug 2018 17:48:06 -0400
> Tony Krowiak <[email protected]> wrote:
>
>> From: Tony Krowiak <[email protected]>
>>
>> Registers the matrix device created by the VFIO AP device
>> driver with the VFIO mediated device framework.
>> Registering the matrix device will create the sysfs
>> structures needed to create mediated matrix devices
>> each of which will be used to configure the AP matrix
>> for a guest and connect it to the VFIO AP device driver.
>>
>> Registering the matrix device with the VFIO mediated device
>> framework will create the following sysfs structures:
>>
>> /sys/devices/vfio_ap/matrix/
>> ...... [mdev_supported_types]
>> ......... [vfio_ap-passthrough]
>> ............ create
>>
>> To create a mediated device for the AP matrix device, write a UUID
>> to the create file:
>>
>> uuidgen > create
>>
>> A symbolic link to the mediated device's directory will be created in the
>> devices subdirectory named after the generated $uuid:
>>
>> /sys/devices/vfio_ap/matrix/
>> ...... [mdev_supported_types]
>> ......... [vfio_ap-passthrough]
>> ............ [devices]
>> ............... [$uuid]
>>
>> A symbolic link to the mediated device will also be created
>> in the vfio_ap matrix's directory:
>>
>> /sys/devices/vfio_ap/matrix/[$uuid]
>>
>> Signed-off-by: Tony Krowiak <[email protected]>
>> Reviewed-by: Halil Pasic <[email protected]>
>> Tested-by: Michael Mueller <[email protected]>
>> Tested-by: Farhan Ali <[email protected]>
>> Signed-off-by: Christian Borntraeger <[email protected]>
>> ---
>> MAINTAINERS | 1 +
>> drivers/s390/crypto/Makefile | 2 +-
>> drivers/s390/crypto/vfio_ap_drv.c | 23 ++++++
>> drivers/s390/crypto/vfio_ap_ops.c | 124 +++++++++++++++++++++++++++++++++
>> drivers/s390/crypto/vfio_ap_private.h | 45 ++++++++++++
>> include/uapi/linux/vfio.h | 1 +
>> 6 files changed, 195 insertions(+), 1 deletions(-)
>> create mode 100644 drivers/s390/crypto/vfio_ap_ops.c
>>
>
>> diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c
>> new file mode 100644
>> index 0000000..8018c2d
>> --- /dev/null
>> +++ b/drivers/s390/crypto/vfio_ap_ops.c
>> @@ -0,0 +1,124 @@
>> +// SPDX-License-Identifier: GPL-2.0+
>> +/*
>> + * Adjunct processor matrix VFIO device driver callbacks.
>> + *
>> + * Copyright IBM Corp. 2018
>> + *
>> + * Author(s): Tony Krowiak <[email protected]>
>> + * Halil Pasic <[email protected]>
>> + * Pierre Morel <[email protected]>
>> + */
>> +#include <linux/string.h>
>> +#include <linux/vfio.h>
>> +#include <linux/device.h>
>> +#include <linux/list.h>
>> +#include <linux/ctype.h>
>> +
>> +#include "vfio_ap_private.h"
>> +
>> +#define VFIO_AP_MDEV_TYPE_HWVIRT "passthrough"
>> +#define VFIO_AP_MDEV_NAME_HWVIRT "VFIO AP Passthrough Device"
>> +
>> +static void vfio_ap_matrix_init(struct ap_config_info *info,
>> + struct ap_matrix *matrix)
>> +{
>> + matrix->apm_max = info->apxa ? info->Na : 63;
>> + matrix->aqm_max = info->apxa ? info->Nd : 15;
>> + matrix->adm_max = info->apxa ? info->Nd : 15;
>> +}
>> +
>> +static int vfio_ap_mdev_create(struct kobject *kobj, struct mdev_device *mdev)
>> +{
>> + struct ap_matrix_mdev *matrix_mdev;
>> +
>> + matrix_mdev = kzalloc(sizeof(*matrix_mdev), GFP_KERNEL);
>> + if (!matrix_mdev)
>> + return -ENOMEM;
>> +
>> + matrix_mdev->name = dev_name(mdev_dev(mdev));
>> + vfio_ap_matrix_init(&matrix_dev.info, &matrix_mdev->matrix);
>> + mdev_set_drvdata(mdev, matrix_mdev);
>> +
>> + if (atomic_dec_if_positive(&matrix_dev.available_instances) < 0) {
>> + kfree(matrix_mdev);
>> + return -EPERM;
>> + }
> Maybe move this check to the top of the function?

I am going to remove this check. Since we have added a mutex to lock the
matrix_dev, I see no need for the available_instances to be an atomic_t
type because we can update it while holding the matrix_dev lock when the
other updates are made to the matrix_dev.

>
>> +
>> + mutex_lock(&matrix_dev.lock);
>> + list_add(&matrix_mdev->list, &matrix_dev.mdev_list);
>> + mutex_unlock(&matrix_dev.lock);
>> +
>> + return 0;
>> +}

2018-08-15 17:00:54

[permalink] [raw]

Subject: Re: [PATCH v9 10/22] s390: vfio-ap: sysfs interfaces to configure adapters

On 08/15/2018 05:52 AM, Cornelia Huck wrote:
> On Mon, 13 Aug 2018 17:48:07 -0400
> Tony Krowiak <[email protected]> wrote:
>
>> +/**
>> + * assign_adapter_store
>> + *
>> + * @dev: the matrix device
>> + * @attr: a mediated matrix device attribute
>> + * @buf: a buffer containing the adapter ID (APID) to be assigned
>> + * @count: the number of bytes in @buf
>> + *
>> + * Parses the APID from @buf and assigns it to the mediated matrix device.
>> + *
>> + * Returns the number of bytes processed if the APID is valid; otherwise returns
>> + * an error.
>> + */
>> +static ssize_t assign_adapter_store(struct device *dev,
>> + struct device_attribute *attr,
>> + const char *buf, size_t count)
>> +{
>> + int ret = 0;
> You don't need to initialize this to 0, as kstrtoul will set it in any
> case.

Right you are! Will change it.

>
>> + unsigned long apid;
>> + struct mdev_device *mdev = mdev_from_dev(dev);
>> + struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
>> + unsigned long max_apid = matrix_mdev->matrix.apm_max;
>> +
>> + ret = kstrtoul(buf, 0, &apid);
>> + if (ret)
>> + return ret;
>> + if (apid > max_apid)
>> + return -EINVAL;
>> +
>> + /* Set the bit in the AP mask (APM) corresponding to the AP adapter
>> + * number (APID). The bits in the mask, from most significant to least
>> + * significant bit, correspond to APIDs 0-255.
>> + */
>> + mutex_lock(&matrix_dev.lock);
>> +
>> + ret = vfio_ap_mdev_verify_queues_reserved_for_apid(matrix_mdev, apid);
> That function name really is a mouthful :) I don't have any better
> suggestions, though.

It is, but I think it describes exactly what the function does.

>
>> + if (ret)
>> + goto done;
>> +
>> + set_bit_inv(apid, matrix_mdev->matrix.apm);
>> +
>> + ret = vfio_ap_mdev_verify_no_sharing(matrix_mdev);
>> + if (ret)
>> + goto share_err;
>> +
>> + ret = count;
>> + goto done;
>> +
>> +share_err:
>> + clear_bit_inv(apid, matrix_mdev->matrix.apm);
>> +done:
>> + mutex_unlock(&matrix_dev.lock);
>> +
>> + return ret;
>> +}
>> +static DEVICE_ATTR_WO(assign_adapter);
>> +
>> +/**
>> + * unassign_adapter_store
>> + *
>> + * @dev: the matrix device
>> + * @attr: a mediated matrix device attribute
>> + * @buf: a buffer containing the adapter ID (APID) to be assigned
>> + * @count: the number of bytes in @buf
>> + *
>> + * Parses the APID from @buf and unassigns it from the mediated matrix device.
>> + * The APID must be a valid value
> A valid value, but not necessarily assigned, right?

You are correct, if the APID is not assigned, then the corresponding bit
will be
cleared regardless. In a previous version, the functions failed if the
APID is
not assigned, but a colleague removed that check. I guess it makes sense
given
it really does not hurt anything to ask to unassign an APID that isn't
assigned
to begin with. Would you prefer I update the comment, or do you feel the
user
should be made aware of an attempt to unassign an APID that is not assigned?

>
>> + *
>> + * Returns the number of bytes processed if the APID is valid; otherwise returns
>> + * an error.
>> + */
>> +static ssize_t unassign_adapter_store(struct device *dev,
>> + struct device_attribute *attr,
>> + const char *buf, size_t count)
>> +{
>> + int ret;
>> + unsigned long apid;
>> + struct mdev_device *mdev = mdev_from_dev(dev);
>> + struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
>> +
>> + ret = kstrtoul(buf, 0, &apid);
>> + if (ret)
>> + return ret;
>> +
>> + if (apid > matrix_mdev->matrix.apm_max)
>> + return -EINVAL;
>> +
>> + mutex_lock(&matrix_dev.lock);
>> + clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm);
>> + mutex_unlock(&matrix_dev.lock);
>> +
>> + return count;
>> +}
>> +DEVICE_ATTR_WO(unassign_adapter);
> In general, looks good to me.

That is good news indeed.

>

2018-08-15 17:02:12

[permalink] [raw]

Subject: Re: [PATCH v9 11/22] s390: vfio-ap: sysfs interfaces to configure domains

On 08/15/2018 08:05 AM, Cornelia Huck wrote:
> On Mon, 13 Aug 2018 17:48:08 -0400
> Tony Krowiak <[email protected]> wrote:
>
>> From: Tony Krowiak <[email protected]>
>>
>> Introduces two new sysfs attributes for the VFIO mediated
>> matrix device for assigning AP domains to and unassigning
>> AP domains from a mediated matrix device. The IDs of the
>> AP domains assigned to the mediated matrix device will be
>> stored in an AP queue mask (AQM).
>>
>> The bits in the AQM, from most significant to least
>> significant bit, correspond to AP queue index (APQI) 0 to
>> 255 (note that an APQI is synonymous with with a domain ID).
>> On some systems, the maximum allowable domain number may be
>> less than 255 - depending upon the host's AP configuration -
>> and assignment may be rejected if the input domain ID exceeds
>> the limit.
>>
>> When a domain is assigned, the bit corresponding to the APQI
>> will be set in the AQM. Likewise, when a domain is unassigned,
>> the bit corresponding to the APQI will be cleared from the AQM.
>>
>> In order to successfully assign a domain, the APQNs derived from
>> the domain ID being assigned and the adapter numbers of all
>> adapters previously assigned:
>>
>> 1. Must be bound to the vfio_ap device driver.
>>
>> 2. Must not be assigned to any other mediated matrix device.
>>
>> If there are no adapters assigned to the mdev, then there must
>> be an AP queue bound to the vfio_ap device driver with an
>> APQN containing the domain ID (i.e., APQI), otherwise all
>> adapters subsequently assigned will fail because there will be no
>> AP queues bound with an APQN containing the APQI.
>>
>> Assigning or un-assigning an AP domain will also be rejected if
>> a guest using the mediated matrix device is running.
>>
>> The relevant sysfs structures are:
>>
>> /sys/devices/vfio_ap/matrix/
>> ...... [mdev_supported_types]
>> ......... [vfio_ap-passthrough]
>> ............ [devices]
>> ...............[$uuid]
>> .................. assign_domain
>> .................. unassign_domain
>>
>> To assign a domain to the $uuid mediated matrix device,
>> write the domain's ID to the assign_domain file. To
>> unassign a domain, write the domain's ID to the
>> unassign_domain file. The ID is specified using
>> conventional semantics: If it begins with 0x, the number
>> will be parsed as a hexadecimal (case insensitive) number;
>> if it begins with 0, it will be parsed as an octal number;
>> otherwise, it will be parsed as a decimal number.
>>
>> For example, to assign domain 173 (0xad) to the mediated matrix
>> device $uuid:
>>
>> echo 173 > assign_domain
>>
>> or
>>
>> echo 0255 > assign_domain
>>
>> or
>>
>> echo 0xad > assign_domain
>>
>> To unassign domain 173 (0xad):
>>
>> echo 173 > unassign_domain
>>
>> or
>>
>> echo 0255 > unassign_domain
>>
>> or
>>
>> echo 0xad > unassign_domain
>>
>> Signed-off-by: Tony Krowiak <[email protected]>
>> Reviewed-by: Halil Pasic <[email protected]>
>> Tested-by: Michael Mueller <[email protected]>
>> Tested-by: Farhan Ali <[email protected]>
>> Tested-by: Pierre Morel <[email protected]>
>> Signed-off-by: Christian Borntraeger <[email protected]>
>> ---
>> drivers/s390/crypto/vfio_ap_ops.c | 88 ++++++++++++++++++++++++++++++++++++-
>> 1 files changed, 87 insertions(+), 1 deletions(-)
> Looks sane to me.

Sanity rules!

>

2018-08-15 17:57:14

[permalink] [raw]

Subject: Re: [PATCH v9 14/22] KVM: s390: interfaces to clear CRYCB masks

On 08/15/2018 09:10 AM, Cornelia Huck wrote:
> On Mon, 13 Aug 2018 17:48:11 -0400
> Tony Krowiak <[email protected]> wrote:
>
>> From: Tony Krowiak <[email protected]>
>>
>> Introduces two new KVM interface to clear the APM, AQM and ADM masks in
>> the guest's CRYCB. The VCPUs are taken out of SIE to ensure the VCPUs do
>> not get out of sync.
>>
>> Signed-off-by: Tony Krowiak <[email protected]>
>> Acked-by: Halil Pasic <[email protected]>
>> Tested-by: Michael Mueller <[email protected]>
>> Tested-by: Farhan Ali <[email protected]>
>> Tested-by: Pierre Morel <[email protected]>
>> Signed-off-by: Christian Borntraeger <[email protected]>
>> ---
>> arch/s390/include/asm/kvm_host.h | 3 +++
>> arch/s390/kvm/kvm-s390.c | 15 +++++++++++++++
>> 2 files changed, 18 insertions(+), 0 deletions(-)
>>
>> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
>> index 4a739d4..07e58d8 100644
>> --- a/arch/s390/include/asm/kvm_host.h
>> +++ b/arch/s390/include/asm/kvm_host.h
>> @@ -258,6 +258,7 @@ struct kvm_s390_sie_block {
>> __u64 tecmc; /* 0x00e8 */
>> __u8 reservedf0[12]; /* 0x00f0 */
>> #define CRYCB_FORMAT_MASK 0x00000003
>> +#define CRYCB_FORMAT0 0x00000000
> This should probably go into a different patch (define is not used
> here)?

You are spot on. It needs to go in patch 15 where it is first used.

>
>> #define CRYCB_FORMAT1 0x00000001
>> #define CRYCB_FORMAT2 0x00000003
>> __u32 crycbd; /* 0x00fc */
>> @@ -861,6 +862,8 @@ void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
>> void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
>> struct kvm_async_pf *work);
>>
>> +void kvm_arch_crypto_clear_masks(struct kvm *kvm);
>> +
>> extern int sie64a(struct kvm_s390_sie_block *, u64 *);
>> extern char sie_exit;
>>
>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
>> index f7de123..8d8a65a 100644
>> --- a/arch/s390/kvm/kvm-s390.c
>> +++ b/arch/s390/kvm/kvm-s390.c
>> @@ -1919,6 +1919,21 @@ static void kvm_s390_set_crycb_format(struct kvm *kvm)
>> kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
>> }
>>
>> +void kvm_arch_crypto_clear_masks(struct kvm *kvm)
>> +{
>> + mutex_lock(&kvm->lock);
>> + kvm_s390_vcpu_block_all(kvm);
>> +
>> + memset(&kvm->arch.crypto.crycb->apcb0, 0,
>> + sizeof(kvm->arch.crypto.crycb->apcb0));
>> + memset(&kvm->arch.crypto.crycb->apcb1, 0,
>> + sizeof(kvm->arch.crypto.crycb->apcb1));
>> +
>> + kvm_s390_vcpu_unblock_all(kvm);
>> + mutex_unlock(&kvm->lock);
>> +}
>> +EXPORT_SYMBOL(kvm_arch_crypto_clear_masks);
> A quick grep shows that kvm-related exports tend to be
> EXPORT_SYMBOL_GPL. Should that also be done here?

I did some internet searching to find out why the EXPORT_SYMBOL_GPL
macro is used
rather than EXPORT_SYMBOL. I found the link below which explains why.
According to
that article, if the EXPORT_SYMBOL_GPL macro is used, modules that do
not carry a
GPL-compatible license can not use the function. Apparently Linus
Trovalds and the
lawyers he consulted believe it codifies the intention in code itself
makes the code
more flexible and a lot less likely to be misunderstood. It is further
pointed out
that "circumventing the GPL-only export requires an explicit action,
making it clear
that the copyright infringement was a deliberate act.

Since our modules are GPL-licensed, it would probably behoove us to use the
EXPORT_SYMBOL_GPL macro to export this function.

https://lwn.net/Articles/154602/

>
>> +
>> static u64 kvm_s390_get_initial_cpuid(void)
>> {
>> struct cpuid cpuid;

2018-08-15 18:23:23

[permalink] [raw]

Subject: Re: [PATCH v9 15/22] s390: vfio-ap: implement mediated device open callback

On 08/15/2018 12:08 PM, Cornelia Huck wrote:
> On Mon, 13 Aug 2018 17:48:12 -0400
> Tony Krowiak <[email protected]> wrote:
>
>> From: Tony Krowiak <[email protected]>
>>
>> Implements the open callback on the mediated matrix device.
>> The function registers a group notifier to receive notification
>> of the VFIO_GROUP_NOTIFY_SET_KVM event. When notified,
>> the vfio_ap device driver will get access to the guest's
>> kvm structure. The open callback must ensure that only one
>> mediated device shall be opened per guest.
>>
>> Signed-off-by: Tony Krowiak <[email protected]>
>> Acked-by: Halil Pasic <[email protected]>
>> Tested-by: Michael Mueller <[email protected]>
>> Tested-by: Farhan Ali <[email protected]>
>> Tested-by: Pierre Morel <[email protected]>
>> Acked-by: Pierre Morel <[email protected]>
>> Signed-off-by: Christian Borntraeger <[email protected]>
>> ---
>> drivers/s390/crypto/vfio_ap_ops.c | 174 ++++++++++++++++++++++++++++++++-
>> drivers/s390/crypto/vfio_ap_private.h | 2 +
>> 2 files changed, 175 insertions(+), 1 deletions(-)
>> @@ -602,7 +633,6 @@ static ssize_t matrix_show(struct device *dev, struct device_attribute *attr,
>> }
>> DEVICE_ATTR_RO(matrix);
>>
>> -
> Nit: whitespace change

A nit, but somebody else will point it out too, so I will fix it.

>
>> static struct attribute *vfio_ap_mdev_attrs[] = {
>> &dev_attr_assign_adapter.attr,
>> &dev_attr_unassign_adapter.attr,
> (...)
>
>> +/**
>> + * vfio_ap_mdev_open_once
>> + *
>> + * @matrix_mdev: a mediated matrix device
>> + *
>> + * Return 0 if no other mediated matrix device has been opened for the
>> + * KVM guest assigned to @matrix_mdev; otherwise, returns an error.
>> + */
>> +static int vfio_ap_mdev_open_once(struct ap_matrix_mdev *matrix_mdev,
>> + struct kvm *kvm)
> You're passing kvm in here, but do not use it.

I'll fix it.

>
>> +{
>> + struct ap_matrix_mdev *m;
>> +
>> + mutex_lock(&matrix_dev.lock);
>> +
>> + list_for_each_entry(m, &matrix_dev.mdev_list, list) {
>> + if ((m != matrix_mdev) && (m->kvm == matrix_mdev->kvm)) {
> If you used it here instead of matrix_mdev->kvm...

I believe that was the case in a previous patch, but for some reason the
code
was changed. I'll rework this so it uses the kvm param instead.

>
>> + mutex_unlock(&matrix_dev.lock);
>> + return -EPERM;
>> + }
>> + }
>> +
>> + mutex_unlock(&matrix_dev.lock);
>> +
>> + return 0;
>> +}
>> +
>> +static int vfio_ap_mdev_group_notifier(struct notifier_block *nb,
>> + unsigned long action, void *data)
>> +{
>> + int ret;
>> + struct ap_matrix_mdev *matrix_mdev;
>> +
>> + if (action != VFIO_GROUP_NOTIFY_SET_KVM)
>> + return NOTIFY_OK;
>> +
>> + matrix_mdev = container_of(nb, struct ap_matrix_mdev, group_notifier);
>> +
>> + matrix_mdev->kvm = data;
>> + if (data == NULL)
>> + return NOTIFY_OK;
>> +
>> + ret = vfio_ap_mdev_open_once(matrix_mdev, data);
> ...you could move this up to before overwriting matrix_mdev->kvm and
> bailing out when the check failed, which makes more sense to me.

That makes more sense to me too. Don't set matrix_mdev->kvm until we
know it is okay to do so.

>
>> + if (ret)
>> + return ret;
>> +
>> + ret = kvm_ap_validate_crypto_setup(matrix_mdev->kvm);
>> + if (ret)
>> + return ret;
> It probably makes more sense to return NOTIFY_DONE in the error case
> (NOTIFY_BAD does not sound like a good idea as it would stop processing
> the notifier chain).

Will do.

>
>> +
>> + vfio_ap_mdev_copy_masks(matrix_mdev);
>> +
>> + return NOTIFY_OK;
>> +}
> Otherwise, looks sane.

Good!!!

>

2018-08-15 20:38:03

[permalink] [raw]

Subject: Re: [PATCH v9 17/22] s390: vfio-ap: zeroize the AP queues.

On 08/15/2018 12:24 PM, Cornelia Huck wrote:
> On Mon, 13 Aug 2018 17:48:14 -0400
> Tony Krowiak <[email protected]> wrote:
>
> Nit: please drop the leading period in the subject.

I assume you mean the ending period?

>
>> From: Tony Krowiak <[email protected]>
>>
>> Let's call PAPQ(ZAPQ) to zeroize a queue:
>>
>> * For each queue configured for a mediated matrix device
>> when it is released.
>>
>> Zeroizing a queue resets the queue, clears all pending
>> messages for the queue entries and disables adapter interruptions
>> associated with the queue.
>>
>> Signed-off-by: Tony Krowiak <[email protected]>
>> Reviewed-by: Halil Pasic <[email protected]>
>> Tested-by: Michael Mueller <[email protected]>
>> Tested-by: Farhan Ali <[email protected]>
>> Signed-off-by: Christian Borntraeger <[email protected]>
>> ---
>> drivers/s390/crypto/vfio_ap_ops.c | 25 +++++++++++++++++++++++++
>> drivers/s390/crypto/vfio_ap_private.h | 25 +++++++++++++++++++++++++
>> 2 files changed, 50 insertions(+), 0 deletions(-)
>>
>> diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h
>> index 3e8534b..34f982a 100644
>> --- a/drivers/s390/crypto/vfio_ap_private.h
>> +++ b/drivers/s390/crypto/vfio_ap_private.h
>> @@ -74,4 +74,29 @@ struct ap_matrix_mdev {
>> extern int vfio_ap_mdev_register(void);
>> extern void vfio_ap_mdev_unregister(void);
>>
>> +static inline int vfio_ap_reset_queue(unsigned int apid, unsigned int apqi,
>> + unsigned int retry)
>> +{
>> + struct ap_queue_status status;
>> +
>> + do {
>> + status = ap_zapq(AP_MKQID(apid, apqi));
>> + switch (status.response_code) {
>> + case AP_RESPONSE_NORMAL:
>> + return 0;
>> + case AP_RESPONSE_RESET_IN_PROGRESS:
>> + case AP_RESPONSE_BUSY:
>> + msleep(20);
>> + break;
>> + default:
>> + pr_warn("%s: error zeroizing %02x.%04x: response code %d\n",
>> + VFIO_AP_MODULE_NAME, apid, apqi,
>> + status.response_code);
> How can we end up here? Does this mean that we just don't know what to
> do with this response, or is this something that should never happen?
> (How much sense does it make to print an error?)

There are additional response codes that could be returned; for example,
in the case of a catastrophic failure such as: The function can not be
performed because the AP was somehow deconfigured or the functiona
cannot be performed due to a machine check failure that caused the AP
path to be removed. It shouldn't happen, but all are possibilities.
I can get rid of the message and just return -EIO if you prefer.

>
>> + return -EIO;
>> + }
>> + } while (retry--);
>> +
>> + return -EBUSY;
>> +}
>> +
>> #endif /* _VFIO_AP_PRIVATE_H_ */

2018-08-15 21:07:03

[permalink] [raw]

Subject: Re: [PATCH v9 18/22] s390: vfio-ap: implement VFIO_DEVICE_RESET ioctl

On 08/15/2018 12:38 PM, Cornelia Huck wrote:
> On Mon, 13 Aug 2018 17:48:15 -0400
> Tony Krowiak <[email protected]> wrote:
>
>> From: Tony Krowiak <[email protected]>
>>
>> Implements the VFIO_DEVICE_RESET ioctl. This ioctl zeroizes
>> all of the AP queues assigned to the guest.
>>
>> Signed-off-by: Tony Krowiak <[email protected]>
>> Reviewed-by: Halil Pasic <[email protected]>
>> Tested-by: Michael Mueller <[email protected]>
>> Tested-by: Farhan Ali <[email protected]>
>> Reviewed-by: Pierre Morel <[email protected]>
>> Tested-by: Pierre Morel <[email protected]>
>> Signed-off-by: Christian Borntraeger <[email protected]>
>> ---
>> drivers/s390/crypto/vfio_ap_ops.c | 5 ++++-
>> 1 files changed, 4 insertions(+), 1 deletions(-)
>>
>> diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c
>> index cc90323..d4a065b 100644
>> --- a/drivers/s390/crypto/vfio_ap_ops.c
>> +++ b/drivers/s390/crypto/vfio_ap_ops.c
>> @@ -832,7 +832,7 @@ static int vfio_ap_mdev_get_device_info(unsigned long arg)
>> if (info.argsz < minsz)
>> return -EINVAL;
>>
>> - info.flags = VFIO_DEVICE_FLAGS_AP;
>> + info.flags = VFIO_DEVICE_FLAGS_AP | VFIO_DEVICE_FLAGS_RESET;
>> info.num_regions = 0;
>> info.num_irqs = 0;
>>
>> @@ -848,6 +848,9 @@ static ssize_t vfio_ap_mdev_ioctl(struct mdev_device *mdev,
>> case VFIO_DEVICE_GET_INFO:
>> ret = vfio_ap_mdev_get_device_info(arg);
>> break;
>> + case VFIO_DEVICE_RESET:
>> + ret = vfio_ap_mdev_reset_queues(mdev, true);
> If I see it correctly, you call this function only ever with force ==
> true (here and in the previous patch). Is that what you intended?

That does seem to be the case now; however, I think at one time there were
additional calls to this function. For some reason of which I am not aware,
those were removed, so there is probably no need for it now.

>
>> + break;
>> default:
>> ret = -EOPNOTSUPP;
>> break;

2018-08-16 13:04:48

[permalink] [raw]

Subject: Re: [PATCH v9 10/22] s390: vfio-ap: sysfs interfaces to configure adapters

On Wed, 15 Aug 2018 12:59:35 -0400
Tony Krowiak <[email protected]> wrote:

> On 08/15/2018 05:52 AM, Cornelia Huck wrote:
> > On Mon, 13 Aug 2018 17:48:07 -0400
> > Tony Krowiak <[email protected]> wrote:

> >> +/**
> >> + * unassign_adapter_store
> >> + *
> >> + * @dev: the matrix device
> >> + * @attr: a mediated matrix device attribute
> >> + * @buf: a buffer containing the adapter ID (APID) to be assigned
> >> + * @count: the number of bytes in @buf
> >> + *
> >> + * Parses the APID from @buf and unassigns it from the mediated matrix device.
> >> + * The APID must be a valid value
> > A valid value, but not necessarily assigned, right?
>
> You are correct, if the APID is not assigned, then the corresponding bit
> will be
> cleared regardless. In a previous version, the functions failed if the
> APID is
> not assigned, but a colleague removed that check. I guess it makes sense
> given
> it really does not hurt anything to ask to unassign an APID that isn't
> assigned
> to begin with. Would you prefer I update the comment, or do you feel the
> user
> should be made aware of an attempt to unassign an APID that is not assigned?

I think the code is fine; updating the comment would be good.

2018-08-16 18:45:34

[permalink] [raw]

Subject: Re: [PATCH v9 09/22] s390: vfio-ap: register matrix device with VFIO mdev framework

On 08/14/2018 07:19 AM, Cornelia Huck wrote:
> On Mon, 13 Aug 2018 17:48:06 -0400
> Tony Krowiak <[email protected]> wrote:
>
>> From: Tony Krowiak <[email protected]>
>>
>> Registers the matrix device created by the VFIO AP device
>> driver with the VFIO mediated device framework.
>> Registering the matrix device will create the sysfs
>> structures needed to create mediated matrix devices
>> each of which will be used to configure the AP matrix
>> for a guest and connect it to the VFIO AP device driver.
>>
>> Registering the matrix device with the VFIO mediated device
>> framework will create the following sysfs structures:
>>
>> /sys/devices/vfio_ap/matrix/
>> ...... [mdev_supported_types]
>> ......... [vfio_ap-passthrough]
>> ............ create
>>
>> To create a mediated device for the AP matrix device, write a UUID
>> to the create file:
>>
>> uuidgen > create
>>
>> A symbolic link to the mediated device's directory will be created in the
>> devices subdirectory named after the generated $uuid:
>>
>> /sys/devices/vfio_ap/matrix/
>> ...... [mdev_supported_types]
>> ......... [vfio_ap-passthrough]
>> ............ [devices]
>> ............... [$uuid]
>>
>> A symbolic link to the mediated device will also be created
>> in the vfio_ap matrix's directory:
>>
>> /sys/devices/vfio_ap/matrix/[$uuid]
>>
>> Signed-off-by: Tony Krowiak <[email protected]>
>> Reviewed-by: Halil Pasic <[email protected]>
>> Tested-by: Michael Mueller <[email protected]>
>> Tested-by: Farhan Ali <[email protected]>
>> Signed-off-by: Christian Borntraeger <[email protected]>
>> ---
>> MAINTAINERS | 1 +
>> drivers/s390/crypto/Makefile | 2 +-
>> drivers/s390/crypto/vfio_ap_drv.c | 23 ++++++
>> drivers/s390/crypto/vfio_ap_ops.c | 124 +++++++++++++++++++++++++++++++++
>> drivers/s390/crypto/vfio_ap_private.h | 45 ++++++++++++
>> include/uapi/linux/vfio.h | 1 +
>> 6 files changed, 195 insertions(+), 1 deletions(-)
>> create mode 100644 drivers/s390/crypto/vfio_ap_ops.c
>>
>
>> diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c
>> new file mode 100644
>> index 0000000..8018c2d
>> --- /dev/null
>> +++ b/drivers/s390/crypto/vfio_ap_ops.c
>> @@ -0,0 +1,124 @@
>> +// SPDX-License-Identifier: GPL-2.0+
>> +/*
>> + * Adjunct processor matrix VFIO device driver callbacks.
>> + *
>> + * Copyright IBM Corp. 2018
>> + *
>> + * Author(s): Tony Krowiak <[email protected]>
>> + * Halil Pasic <[email protected]>
>> + * Pierre Morel <[email protected]>
>> + */
>> +#include <linux/string.h>
>> +#include <linux/vfio.h>
>> +#include <linux/device.h>
>> +#include <linux/list.h>
>> +#include <linux/ctype.h>
>> +
>> +#include "vfio_ap_private.h"
>> +
>> +#define VFIO_AP_MDEV_TYPE_HWVIRT "passthrough"
>> +#define VFIO_AP_MDEV_NAME_HWVIRT "VFIO AP Passthrough Device"
>> +
>> +static void vfio_ap_matrix_init(struct ap_config_info *info,
>> + struct ap_matrix *matrix)
>> +{
>> + matrix->apm_max = info->apxa ? info->Na : 63;
>> + matrix->aqm_max = info->apxa ? info->Nd : 15;
>> + matrix->adm_max = info->apxa ? info->Nd : 15;
>> +}
>> +
>> +static int vfio_ap_mdev_create(struct kobject *kobj, struct mdev_device *mdev)
>> +{
>> + struct ap_matrix_mdev *matrix_mdev;
>> +
>> + matrix_mdev = kzalloc(sizeof(*matrix_mdev), GFP_KERNEL);
>> + if (!matrix_mdev)
>> + return -ENOMEM;
>> +
>> + matrix_mdev->name = dev_name(mdev_dev(mdev));
>> + vfio_ap_matrix_init(&matrix_dev.info, &matrix_mdev->matrix);
>> + mdev_set_drvdata(mdev, matrix_mdev);
>> +
>> + if (atomic_dec_if_positive(&matrix_dev.available_instances) < 0) {
>> + kfree(matrix_mdev);
>> + return -EPERM;
>> + }
> Maybe move this check to the top of the function?

Please ignore my previous response to your comment. I can't move the call to
atomic_dec_if_positive() to the top of the function because it
decrements the
available_instances and if the kzalloc() of matrix_mdev fails, then the
value
would have to then be incremented to remain valid. What I can do is this:

1. Check the value of available_instances using atomic_read() at the top of
the function and if it is zero, return an error.

2. Replace the call to atomic_dec_if_positive() with a call to atomic_dec()
to decrement the available_instances.

I agree that it makes sense to return before attempting to allocate the
matrix_mdev if available_instances is zero.

>
>> +
>> + mutex_lock(&matrix_dev.lock);
>> + list_add(&matrix_mdev->list, &matrix_dev.mdev_list);
>> + mutex_unlock(&matrix_dev.lock);
>> +
>> + return 0;
>> +}

2018-08-17 08:46:13

[permalink] [raw]

Subject: Re: [PATCH v9 09/22] s390: vfio-ap: register matrix device with VFIO mdev framework

On Thu, 16 Aug 2018 12:24:16 -0400
Tony Krowiak <[email protected]> wrote:

> On 08/14/2018 07:19 AM, Cornelia Huck wrote:
> > On Mon, 13 Aug 2018 17:48:06 -0400
> > Tony Krowiak <[email protected]> wrote:

> >> +static int vfio_ap_mdev_create(struct kobject *kobj, struct mdev_device *mdev)
> >> +{
> >> + struct ap_matrix_mdev *matrix_mdev;
> >> +
> >> + matrix_mdev = kzalloc(sizeof(*matrix_mdev), GFP_KERNEL);
> >> + if (!matrix_mdev)
> >> + return -ENOMEM;
> >> +
> >> + matrix_mdev->name = dev_name(mdev_dev(mdev));
> >> + vfio_ap_matrix_init(&matrix_dev.info, &matrix_mdev->matrix);
> >> + mdev_set_drvdata(mdev, matrix_mdev);
> >> +
> >> + if (atomic_dec_if_positive(&matrix_dev.available_instances) < 0) {
> >> + kfree(matrix_mdev);
> >> + return -EPERM;
> >> + }
> > Maybe move this check to the top of the function?
>
> Please ignore my previous response to your comment. I can't move the call to
> atomic_dec_if_positive() to the top of the function because it
> decrements the
> available_instances and if the kzalloc() of matrix_mdev fails, then the
> value
> would have to then be incremented to remain valid. What I can do is this:
>
> 1. Check the value of available_instances using atomic_read() at the top of
> the function and if it is zero, return an error.
>
> 2. Replace the call to atomic_dec_if_positive() with a call to atomic_dec()
> to decrement the available_instances.
>
> I agree that it makes sense to return before attempting to allocate the
> matrix_mdev if available_instances is zero.

Wouldn't that be racy, though?

I don't think re-incrementing the counter is too bad, and it's
certainly better than going through allocation/freeing of structures.

>
> >
> >> +
> >> + mutex_lock(&matrix_dev.lock);
> >> + list_add(&matrix_mdev->list, &matrix_dev.mdev_list);
> >> + mutex_unlock(&matrix_dev.lock);
> >> +
> >> + return 0;
> >> +}
>
>

2018-08-17 09:36:59

[permalink] [raw]

Subject: Re: [PATCH v9 17/22] s390: vfio-ap: zeroize the AP queues.

On Wed, 15 Aug 2018 16:36:32 -0400
Tony Krowiak <[email protected]> wrote:

> On 08/15/2018 12:24 PM, Cornelia Huck wrote:
> > On Mon, 13 Aug 2018 17:48:14 -0400
> > Tony Krowiak <[email protected]> wrote:
> >
> > Nit: please drop the leading period in the subject.
>
> I assume you mean the ending period?

Err, of course.

>
> >
> >> From: Tony Krowiak <[email protected]>
> >>
> >> Let's call PAPQ(ZAPQ) to zeroize a queue:
> >>
> >> * For each queue configured for a mediated matrix device
> >> when it is released.
> >>
> >> Zeroizing a queue resets the queue, clears all pending
> >> messages for the queue entries and disables adapter interruptions
> >> associated with the queue.
> >>
> >> Signed-off-by: Tony Krowiak <[email protected]>
> >> Reviewed-by: Halil Pasic <[email protected]>
> >> Tested-by: Michael Mueller <[email protected]>
> >> Tested-by: Farhan Ali <[email protected]>
> >> Signed-off-by: Christian Borntraeger <[email protected]>
> >> ---
> >> drivers/s390/crypto/vfio_ap_ops.c | 25 +++++++++++++++++++++++++
> >> drivers/s390/crypto/vfio_ap_private.h | 25 +++++++++++++++++++++++++
> >> 2 files changed, 50 insertions(+), 0 deletions(-)
> >>
> >> diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h
> >> index 3e8534b..34f982a 100644
> >> --- a/drivers/s390/crypto/vfio_ap_private.h
> >> +++ b/drivers/s390/crypto/vfio_ap_private.h
> >> @@ -74,4 +74,29 @@ struct ap_matrix_mdev {
> >> extern int vfio_ap_mdev_register(void);
> >> extern void vfio_ap_mdev_unregister(void);
> >>
> >> +static inline int vfio_ap_reset_queue(unsigned int apid, unsigned int apqi,
> >> + unsigned int retry)
> >> +{
> >> + struct ap_queue_status status;
> >> +
> >> + do {
> >> + status = ap_zapq(AP_MKQID(apid, apqi));
> >> + switch (status.response_code) {
> >> + case AP_RESPONSE_NORMAL:
> >> + return 0;
> >> + case AP_RESPONSE_RESET_IN_PROGRESS:
> >> + case AP_RESPONSE_BUSY:
> >> + msleep(20);
> >> + break;
> >> + default:
> >> + pr_warn("%s: error zeroizing %02x.%04x: response code %d\n",
> >> + VFIO_AP_MODULE_NAME, apid, apqi,
> >> + status.response_code);
> > How can we end up here? Does this mean that we just don't know what to
> > do with this response, or is this something that should never happen?
> > (How much sense does it make to print an error?)
>
> There are additional response codes that could be returned; for example,
> in the case of a catastrophic failure such as: The function can not be
> performed because the AP was somehow deconfigured or the functiona
> cannot be performed due to a machine check failure that caused the AP
> path to be removed. It shouldn't happen, but all are possibilities.
> I can get rid of the message and just return -EIO if you prefer.

These sound like "ugh, we're broken anyway". Not sure if an additional
message would help here much; I'd expect other code to just handle the
failure (especially things like machine checks). I would not oppose
removing the message :)

Maybe add a comment /* things are really broken, give up */ instead?

>
> >
> >> + return -EIO;
> >> + }
> >> + } while (retry--);
> >> +
> >> + return -EBUSY;
> >> +}
> >> +
> >> #endif /* _VFIO_AP_PRIVATE_H_ */
>
>

2018-08-17 09:40:55

[permalink] [raw]

Subject: Re: [PATCH v9 18/22] s390: vfio-ap: implement VFIO_DEVICE_RESET ioctl

On Wed, 15 Aug 2018 17:05:48 -0400
Tony Krowiak <[email protected]> wrote:

> On 08/15/2018 12:38 PM, Cornelia Huck wrote:
> > On Mon, 13 Aug 2018 17:48:15 -0400
> > Tony Krowiak <[email protected]> wrote:

> >> + case VFIO_DEVICE_RESET:
> >> + ret = vfio_ap_mdev_reset_queues(mdev, true);
> > If I see it correctly, you call this function only ever with force ==
> > true (here and in the previous patch). Is that what you intended?
>
> That does seem to be the case now; however, I think at one time there were
> additional calls to this function. For some reason of which I am not aware,
> those were removed, so there is probably no need for it now.

If you don't see a need for it anymore, I'd just remove the parameter.
Even makes vfio_ap_mdev_reset_queues() a bit nicer :)

2018-08-17 13:20:52

[permalink] [raw]

Subject: Re: [PATCH v9 04/22] s390/zcrypt: Integrate ap_asm.h into include/asm/ap.h.

On 08/14/2018 04:43 AM, Cornelia Huck wrote:
> On Mon, 13 Aug 2018 17:48:01 -0400
> Tony Krowiak <[email protected]> wrote:
>
>> From: Harald Freudenberger <[email protected]>
>>
>> Move all the inline functions from the ap bus header
>> file ap_asm.h into the in-kernel api header file
>> arch/s390/include/asm/ap.h so that KVM can make use
>> of all the low level AP functions.
>>
>> Signed-off-by: Harald Freudenberger <[email protected]>
>> Signed-off-by: Christian Borntraeger <[email protected]>
>> Signed-off-by: Tony Krowiak <[email protected]>
>> ---
>> arch/s390/include/asm/ap.h | 284 ++++++++++++++++++++++++++++++++++++----
>> drivers/s390/crypto/ap_bus.c | 23 +---
>> drivers/s390/crypto/ap_bus.h | 1 +
>> drivers/s390/crypto/ap_card.c | 1 -
>> drivers/s390/crypto/ap_queue.c | 1 -
>> 5 files changed, 260 insertions(+), 50 deletions(-)
>>
>> diff --git a/arch/s390/include/asm/ap.h b/arch/s390/include/asm/ap.h
>> index c1bedb4..887494a 100644
>> --- a/arch/s390/include/asm/ap.h
>> +++ b/arch/s390/include/asm/ap.h
>> @@ -47,6 +47,50 @@ struct ap_queue_status {
>> };
>>
>> /**
>> + * ap_intructions_available() - Test if AP instructions are available.
>> + *
>> + * Returns 1 if the AP instructions are installed, otherwise 0.
>> + */
>> +static inline int ap_instructions_available(void)
>> +{
>> + register unsigned long reg0 asm ("0") = AP_MKQID(0, 0);
>> + register unsigned long reg1 asm ("1") = 0;
>> + register unsigned long reg2 asm ("2") = 0;
>> +
>> + asm volatile(
>> + " .long 0xb2af0000\n" /* PQAP(TAPQ) */
>> + "0: la %0,1\n"
>> + "1:\n"
>> + EX_TABLE(0b, 1b)
>> + : "+d" (reg1), "+d" (reg2)
>> + : "d" (reg0)
>> + : "cc");
>> + return reg1;
>> +}
> I think upstream this change (have this function return !0 when the
> instructions are installed) will be an addon patch to the one which has
> already made its way into master. Not really relevant for the remainder
> of this patch series, though.

I noticed that Harald's patches showed up in our master branch yesterday
without this change.

>

2018-08-17 13:25:29

[permalink] [raw]

Subject: Re: [PATCH v9 10/22] s390: vfio-ap: sysfs interfaces to configure adapters

On 08/16/2018 03:30 AM, Cornelia Huck wrote:
> On Wed, 15 Aug 2018 12:59:35 -0400
> Tony Krowiak <[email protected]> wrote:
>
>> On 08/15/2018 05:52 AM, Cornelia Huck wrote:
>>> On Mon, 13 Aug 2018 17:48:07 -0400
>>> Tony Krowiak <[email protected]> wrote:
>>>> +/**
>>>> + * unassign_adapter_store
>>>> + *
>>>> + * @dev: the matrix device
>>>> + * @attr: a mediated matrix device attribute
>>>> + * @buf: a buffer containing the adapter ID (APID) to be assigned
>>>> + * @count: the number of bytes in @buf
>>>> + *
>>>> + * Parses the APID from @buf and unassigns it from the mediated matrix device.
>>>> + * The APID must be a valid value
>>> A valid value, but not necessarily assigned, right?
>> You are correct, if the APID is not assigned, then the corresponding bit
>> will be
>> cleared regardless. In a previous version, the functions failed if the
>> APID is
>> not assigned, but a colleague removed that check. I guess it makes sense
>> given
>> it really does not hurt anything to ask to unassign an APID that isn't
>> assigned
>> to begin with. Would you prefer I update the comment, or do you feel the
>> user
>> should be made aware of an attempt to unassign an APID that is not assigned?
> I think the code is fine; updating the comment would be good.

Will do.

>

2018-08-17 13:28:34

[permalink] [raw]

Subject: Re: [PATCH v9 04/22] s390/zcrypt: Integrate ap_asm.h into include/asm/ap.h.

On Fri, 17 Aug 2018 09:18:51 -0400
Tony Krowiak <[email protected]> wrote:

> On 08/14/2018 04:43 AM, Cornelia Huck wrote:
> > On Mon, 13 Aug 2018 17:48:01 -0400
> > Tony Krowiak <[email protected]> wrote:
> >
> >> From: Harald Freudenberger <[email protected]>
> >>
> >> Move all the inline functions from the ap bus header
> >> file ap_asm.h into the in-kernel api header file
> >> arch/s390/include/asm/ap.h so that KVM can make use
> >> of all the low level AP functions.
> >>
> >> Signed-off-by: Harald Freudenberger <[email protected]>
> >> Signed-off-by: Christian Borntraeger <[email protected]>
> >> Signed-off-by: Tony Krowiak <[email protected]>
> >> ---
> >> arch/s390/include/asm/ap.h | 284 ++++++++++++++++++++++++++++++++++++----
> >> drivers/s390/crypto/ap_bus.c | 23 +---
> >> drivers/s390/crypto/ap_bus.h | 1 +
> >> drivers/s390/crypto/ap_card.c | 1 -
> >> drivers/s390/crypto/ap_queue.c | 1 -
> >> 5 files changed, 260 insertions(+), 50 deletions(-)
> >>
> >> diff --git a/arch/s390/include/asm/ap.h b/arch/s390/include/asm/ap.h
> >> index c1bedb4..887494a 100644
> >> --- a/arch/s390/include/asm/ap.h
> >> +++ b/arch/s390/include/asm/ap.h
> >> @@ -47,6 +47,50 @@ struct ap_queue_status {
> >> };
> >>
> >> /**
> >> + * ap_intructions_available() - Test if AP instructions are available.
> >> + *
> >> + * Returns 1 if the AP instructions are installed, otherwise 0.
> >> + */
> >> +static inline int ap_instructions_available(void)
> >> +{
> >> + register unsigned long reg0 asm ("0") = AP_MKQID(0, 0);
> >> + register unsigned long reg1 asm ("1") = 0;
> >> + register unsigned long reg2 asm ("2") = 0;
> >> +
> >> + asm volatile(
> >> + " .long 0xb2af0000\n" /* PQAP(TAPQ) */
> >> + "0: la %0,1\n"
> >> + "1:\n"
> >> + EX_TABLE(0b, 1b)
> >> + : "+d" (reg1), "+d" (reg2)
> >> + : "d" (reg0)
> >> + : "cc");
> >> + return reg1;
> >> +}
> > I think upstream this change (have this function return !0 when the
> > instructions are installed) will be an addon patch to the one which has
> > already made its way into master. Not really relevant for the remainder
> > of this patch series, though.
>
> I noticed that Harald's patches showed up in our master branch yesterday
> without this change.

Yep, but the s390/features branch has the patch changing the return
code. Maybe just wait for the next s390 pull request before you
rebase :)

2018-08-17 19:03:48

[permalink] [raw]

Subject: Re: [PATCH v9 09/22] s390: vfio-ap: register matrix device with VFIO mdev framework

On 08/17/2018 04:43 AM, Cornelia Huck wrote:
> On Thu, 16 Aug 2018 12:24:16 -0400
> Tony Krowiak <[email protected]> wrote:
>
>> On 08/14/2018 07:19 AM, Cornelia Huck wrote:
>>> On Mon, 13 Aug 2018 17:48:06 -0400
>>> Tony Krowiak <[email protected]> wrote:
>>>> +static int vfio_ap_mdev_create(struct kobject *kobj, struct mdev_device *mdev)
>>>> +{
>>>> + struct ap_matrix_mdev *matrix_mdev;
>>>> +
>>>> + matrix_mdev = kzalloc(sizeof(*matrix_mdev), GFP_KERNEL);
>>>> + if (!matrix_mdev)
>>>> + return -ENOMEM;
>>>> +
>>>> + matrix_mdev->name = dev_name(mdev_dev(mdev));
>>>> + vfio_ap_matrix_init(&matrix_dev.info, &matrix_mdev->matrix);
>>>> + mdev_set_drvdata(mdev, matrix_mdev);
>>>> +
>>>> + if (atomic_dec_if_positive(&matrix_dev.available_instances) < 0) {
>>>> + kfree(matrix_mdev);
>>>> + return -EPERM;
>>>> + }
>>> Maybe move this check to the top of the function?
>> Please ignore my previous response to your comment. I can't move the call to
>> atomic_dec_if_positive() to the top of the function because it
>> decrements the
>> available_instances and if the kzalloc() of matrix_mdev fails, then the
>> value
>> would have to then be incremented to remain valid. What I can do is this:
>>
>> 1. Check the value of available_instances using atomic_read() at the top of
>> the function and if it is zero, return an error.
>>
>> 2. Replace the call to atomic_dec_if_positive() with a call to atomic_dec()
>> to decrement the available_instances.
>>
>> I agree that it makes sense to return before attempting to allocate the
>> matrix_mdev if available_instances is zero.
> Wouldn't that be racy, though?
>
> I don't think re-incrementing the counter is too bad, and it's
> certainly better than going through allocation/freeing of structures.

I'll make it happen.

>
>>>
>>>> +
>>>> + mutex_lock(&matrix_dev.lock);
>>>> + list_add(&matrix_mdev->list, &matrix_dev.mdev_list);
>>>> + mutex_unlock(&matrix_dev.lock);
>>>> +
>>>> + return 0;
>>>> +}
>>

2018-08-17 19:05:38

[permalink] [raw]

Subject: Re: [PATCH v9 18/22] s390: vfio-ap: implement VFIO_DEVICE_RESET ioctl

On 08/17/2018 05:38 AM, Cornelia Huck wrote:
> On Wed, 15 Aug 2018 17:05:48 -0400
> Tony Krowiak <[email protected]> wrote:
>
>> On 08/15/2018 12:38 PM, Cornelia Huck wrote:
>>> On Mon, 13 Aug 2018 17:48:15 -0400
>>> Tony Krowiak <[email protected]> wrote:
>>>> + case VFIO_DEVICE_RESET:
>>>> + ret = vfio_ap_mdev_reset_queues(mdev, true);
>>> If I see it correctly, you call this function only ever with force ==
>>> true (here and in the previous patch). Is that what you intended?
>> That does seem to be the case now; however, I think at one time there were
>> additional calls to this function. For some reason of which I am not aware,
>> those were removed, so there is probably no need for it now.
> If you don't see a need for it anymore, I'd just remove the parameter.
> Even makes vfio_ap_mdev_reset_queues() a bit nicer :)

I guess I wasn't clear, I intended to remove it.

>

2018-08-17 19:10:49

[permalink] [raw]

Subject: Re: [PATCH v9 04/22] s390/zcrypt: Integrate ap_asm.h into include/asm/ap.h.

On 08/17/2018 09:27 AM, Cornelia Huck wrote:
> On Fri, 17 Aug 2018 09:18:51 -0400
> Tony Krowiak <[email protected]> wrote:
>
>> On 08/14/2018 04:43 AM, Cornelia Huck wrote:
>>> On Mon, 13 Aug 2018 17:48:01 -0400
>>> Tony Krowiak <[email protected]> wrote:
>>>
>>>> From: Harald Freudenberger <[email protected]>
>>>>
>>>> Move all the inline functions from the ap bus header
>>>> file ap_asm.h into the in-kernel api header file
>>>> arch/s390/include/asm/ap.h so that KVM can make use
>>>> of all the low level AP functions.
>>>>
>>>> Signed-off-by: Harald Freudenberger <[email protected]>
>>>> Signed-off-by: Christian Borntraeger <[email protected]>
>>>> Signed-off-by: Tony Krowiak <[email protected]>
>>>> ---
>>>> arch/s390/include/asm/ap.h | 284 ++++++++++++++++++++++++++++++++++++----
>>>> drivers/s390/crypto/ap_bus.c | 23 +---
>>>> drivers/s390/crypto/ap_bus.h | 1 +
>>>> drivers/s390/crypto/ap_card.c | 1 -
>>>> drivers/s390/crypto/ap_queue.c | 1 -
>>>> 5 files changed, 260 insertions(+), 50 deletions(-)
>>>>
>>>> diff --git a/arch/s390/include/asm/ap.h b/arch/s390/include/asm/ap.h
>>>> index c1bedb4..887494a 100644
>>>> --- a/arch/s390/include/asm/ap.h
>>>> +++ b/arch/s390/include/asm/ap.h
>>>> @@ -47,6 +47,50 @@ struct ap_queue_status {
>>>> };
>>>>
>>>> /**
>>>> + * ap_intructions_available() - Test if AP instructions are available.
>>>> + *
>>>> + * Returns 1 if the AP instructions are installed, otherwise 0.
>>>> + */
>>>> +static inline int ap_instructions_available(void)
>>>> +{
>>>> + register unsigned long reg0 asm ("0") = AP_MKQID(0, 0);
>>>> + register unsigned long reg1 asm ("1") = 0;
>>>> + register unsigned long reg2 asm ("2") = 0;
>>>> +
>>>> + asm volatile(
>>>> + " .long 0xb2af0000\n" /* PQAP(TAPQ) */
>>>> + "0: la %0,1\n"
>>>> + "1:\n"
>>>> + EX_TABLE(0b, 1b)
>>>> + : "+d" (reg1), "+d" (reg2)
>>>> + : "d" (reg0)
>>>> + : "cc");
>>>> + return reg1;
>>>> +}
>>> I think upstream this change (have this function return !0 when the
>>> instructions are installed) will be an addon patch to the one which has
>>> already made its way into master. Not really relevant for the remainder
>>> of this patch series, though.
>> I noticed that Harald's patches showed up in our master branch yesterday
>> without this change.
> Yep, but the s390/features branch has the patch changing the return
> code. Maybe just wait for the next s390 pull request before you
> rebase :)

Too late. I didn't notice until after the rebase, so I created a patch
on top of master to fix it. I will remove that fixup patch when
the latest appears in our master branch.

>

2018-08-20 14:26:56

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On Mon, 13 Aug 2018 17:48:09 -0400
Tony Krowiak <[email protected]> wrote:

> From: Tony Krowiak <[email protected]>
>
> Provides the sysfs interfaces for:
>
> 1. Assigning AP control domains to the mediated matrix device
>
> 2. Unassigning AP control domains from a mediated matrix device
>
> 3. Displaying the control domains assigned to a mediated matrix
> device
>
> The IDs of the AP control domains assigned to the mediated matrix
> device are stored in an AP domain mask (ADM). The bits in the ADM,
> from most significant to least significant bit, correspond to
> AP domain numbers 0 to 255. On some systems, the maximum allowable
> domain number may be less than 255 - depending upon the host's
> AP configuration - and assignment may be rejected if the input
> domain ID exceeds the limit.

Please remind me of the relationship between control domains and usage
domains... IIRC, usage domains allow both requests and configuration,
while control domains allow only configuration, and are by convention a
superset of usage domains.

Is there a hard requirement somewhere in there, or can the admin
cheerfully use different masks for usage domains and control domains
without the SIE choking on it?

2018-08-20 14:27:10

[permalink] [raw]

Subject: Re: [PATCH v9 16/22] s390: vfio-ap: implement VFIO_DEVICE_GET_INFO ioctl

On Mon, 13 Aug 2018 17:48:13 -0400
Tony Krowiak <[email protected]> wrote:

> From: Tony Krowiak <[email protected]>
>
> Introduces ioctl access to the VFIO AP Matrix device driver
> by implementing the VFIO_DEVICE_GET_INFO ioctl. This ioctl
> provides the VFIO AP Matrix device driver information to the
> guest machine.
>
> Signed-off-by: Tony Krowiak <[email protected]>
> Reviewed-by: Pierre Morel <[email protected]>
> Acked-by: Halil Pasic <[email protected]>
> Tested-by: Michael Mueller <[email protected]>
> Tested-by: Farhan Ali <[email protected]>
> Tested-by: Pierre Morel <[email protected]>
> Signed-off-by: Christian Borntraeger <[email protected]>
> ---
> drivers/s390/crypto/vfio_ap_ops.c | 38 +++++++++++++++++++++++++++++++++++++
> include/uapi/linux/vfio.h | 1 +
> 2 files changed, 39 insertions(+), 0 deletions(-)

Looks sane to me.

2018-08-20 14:28:37

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

On Mon, 13 Aug 2018 17:48:18 -0400
Tony Krowiak <[email protected]> wrote:

> From: Tony Krowiak <[email protected]>
>
> Introduces a new CPU model feature and two CPU model
> facilities to support AP virtualization for KVM guests.
>
> CPU model feature:
>
> The KVM_S390_VM_CPU_FEAT_AP feature indicates that
> AP instructions are available on the guest. This
> feature will be enabled by the kernel only if the AP
> instructions are installed on the linux host. This feature
> must be specifically turned on for the KVM guest from
> userspace to use the VFIO AP device driver for guest
> access to AP devices.
>
> CPU model facilities:
>
> 1. AP Query Configuration Information (QCI) facility is installed.
>
> This is indicated by setting facilities bit 12 for
> the guest. The kernel will not enable this facility
> for the guest if it is not set on the host.
>
> If this facility is not set for the KVM guest, then only
> APQNs with an APQI less than 16 will be used by a Linux
> guest regardless of the matrix configuration for the virtual
> machine. This is a limitation of the Linux AP bus.
>
> 2. AP Facilities Test facility (APFT) is installed.
>
> This is indicated by setting facilities bit 15 for
> the guest. The kernel will not enable this facility for
> the guest if it is not set on the host.
>
> If this facility is not set for the KVM guest, then no
> AP devices will be available to the guest regardless of
> the guest's matrix configuration for the virtual
> machine. This is a limitation of the Linux AP bus.
>
> Signed-off-by: Tony Krowiak <[email protected]>
> Reviewed-by: Christian Borntraeger <[email protected]>
> Reviewed-by: Halil Pasic <[email protected]>
> Tested-by: Michael Mueller <[email protected]>
> Tested-by: Farhan Ali <[email protected]>
> Signed-off-by: Christian Borntraeger <[email protected]>
> ---
> arch/s390/kvm/kvm-s390.c | 5 +++++
> arch/s390/tools/gen_facilities.c | 2 ++
> 2 files changed, 7 insertions(+), 0 deletions(-)

Looks sane to me.

2018-08-20 15:15:02

[permalink] [raw]

Subject: Re: [PATCH v9 13/22] s390: vfio-ap: sysfs interface to view matrix mdev matrix

On Mon, 13 Aug 2018 17:48:10 -0400
Tony Krowiak <[email protected]> wrote:

> From: Tony Krowiak <[email protected]>
>
> Provides a sysfs interface to view the AP matrix configured for the
> mediated matrix device.
>
> The relevant sysfs structures are:
>
> /sys/devices/vfio_ap/matrix/
> ...... [mdev_supported_types]
> ......... [vfio_ap-passthrough]
> ............ [devices]
> ...............[$uuid]
> .................. matrix
>
> To view the matrix configured for the mediated matrix device,
> print the matrix file:
>
> cat matrix
>
> Below are examples of the output from the above command:
>
> Example 1: Adapters and domains assigned
> Assignments:
> Adapters 5 and 6
> Domains 4 and 71 (0x47)
>
> Output
> 05.0004
> 05.0047
> 06.0004
> 06.0047
>
> Examples 2: Only adapters assigned
> Assignments:
> Adapters 5 and 6
>
> Output:
> 05.
> 06.
>
> Examples 3: Only domains assigned
> Assignments:
> Domains 4 and 71 (0x47)
>
> Output:
> .0004
> .0047

I find this output to be a bit confusing; but OTOH, I'm probably not
the person to parse it :) Still, some comments.

From previous discussions, ISTR that this is mainly supposed to be a
debugging/administration aid. Of course, this generates some questions:
- Should this be in sysfs (sysfs attributes are supposed to follow the
"one value per file" rule, at least for the most part), or would
debugfs be a better fit?
- Should userspace code be able to introspect the current
configuration? If yes, it might be better to have some
not-so-nice-but-easily-parsable output, possibly one attribute for
the assigned adapters and one for the assigned domains, and a tool
which distills that into a nice "matrix" with labeled rows and
columns for human consumption.

That said, I don't really have major objections to that interface.

>
> Signed-off-by: Tony Krowiak <[email protected]>
> Reviewed-by: Halil Pasic <[email protected]>
> Tested-by: Michael Mueller <[email protected]>
> Tested-by: Farhan Ali <[email protected]>
> Tested-by: Pierre Morel <[email protected]>
> Signed-off-by: Christian Borntraeger <[email protected]>
> ---
> drivers/s390/crypto/vfio_ap_ops.c | 52 +++++++++++++++++++++++++++++++++++++
> 1 files changed, 52 insertions(+), 0 deletions(-)

2018-08-20 16:06:57

[permalink] [raw]

Subject: Re: [PATCH v9 22/22] s390: doc: detailed specifications for AP virtualization

On Mon, 13 Aug 2018 17:48:19 -0400
Tony Krowiak <[email protected]> wrote:

> From: Tony Krowiak <[email protected]>
>
> This patch provides documentation describing the AP architecture and
> design concepts behind the virtualization of AP devices. It also
> includes an example of how to configure AP devices for exclusive
> use of KVM guests.
>
> Signed-off-by: Tony Krowiak <[email protected]>
> Reviewed-by: Halil Pasic <[email protected]>
> Signed-off-by: Christian Borntraeger <[email protected]>
> ---
> Documentation/s390/vfio-ap.txt | 615 ++++++++++++++++++++++++++++++++++++++++
> MAINTAINERS | 1 +
> 2 files changed, 616 insertions(+), 0 deletions(-)
> create mode 100644 Documentation/s390/vfio-ap.txt
>

> +AP Architectural Overview:
> +=========================
> +To facilitate the comprehension of the design, let's start with some
> +definitions:
> +
> +* AP adapter
> +
> + An AP adapter is an IBM Z adapter card that can perform cryptographic
> + functions. There can be from 0 to 256 adapters assigned to an LPAR. Adapters
> + assigned to the LPAR in which a linux host is running will be available to
> + the linux host. Each adapter is identified by a number from 0 to 255. When
> + installed, an AP adapter is accessed by AP instructions executed by any CPU.
> +
> + The AP adapter cards are assigned to a given LPAR via the system's Activation
> + Profile which can be edited via the HMC. When the system is IPL'd, the AP bus

There's lots of s390 jargon in here... but one hopes that someone
trying to understand AP is already familiar with the basics...

> + module is loaded and detects the AP adapter cards assigned to the LPAR. The AP
> + bus creates a sysfs device for each adapter as they are detected. For example,
> + if AP adapters 4 and 10 (0x0a) are assigned to the LPAR, the AP bus will
> + create the following sysfs entries:
> +
> + /sys/devices/ap/card04
> + /sys/devices/ap/card0a
> +
> + Symbolic links to these devices will also be created in the AP bus devices
> + sub-directory:
> +
> + /sys/bus/ap/devices/[card04]
> + /sys/bus/ap/devices/[card04]
> +
> +* AP domain
> +
> + An adapter is partitioned into domains. Each domain can be thought of as
> + a set of hardware registers for processing AP instructions. An adapter can
> + hold up to 256 domains. Each domain is identified by a number from 0 to 255.
> + Domains can be further classified into two types:
> +
> + * Usage domains are domains that can be accessed directly to process AP
> + commands.
> +
> + * Control domains are domains that are accessed indirectly by AP
> + commands sent to a usage domain to control or change the domain; for
> + example, to set a secure private key for the domain.
> +
> + The AP usage and control domains are assigned to a given LPAR via the system's
> + Activation Profile which can be edited via the HMC. When the system is IPL'd,
> + the AP bus module is loaded and detects the AP usage and control domains
> + assigned to the LPAR. The domain number of each usage domain will be coupled
> + with the adapter number of each AP adapter assigned to the LPAR to identify
> + the AP queues (see AP Queue section below). The domain number of each control
> + domain will be represented in a bitmask and stored in a sysfs file
> + /sys/bus/ap/ap_control_domain_mask created by the bus. The bits in the mask,
> + from most to least significant bit, correspond to domains 0-255.
> +
> + A domain may be assigned to a system as both a usage and control domain, or
> + as a control domain only. Consequently, all domains assigned as both a usage
> + and control domain can both process AP commands as well as be changed by an AP
> + command sent to any usage domain assigned to the same system. Domains assigned
> + only as control domains can not process AP commands but can be changed by AP
> + commands sent to any usage domain assigned to the system.

I'm struggling a bit with this paragraph. Does that mean that you can
use control domains as the target of an instruction changing
configuration on the system? (Or on the VM, if they are listed in the
relevant control block?)

> +
> +* AP Queue
> +
> + An AP queue is the means by which an AP command-request message is sent to a
> + usage domain inside a specific adapter. An AP queue is identified by a tuple
> + comprised of an AP adapter ID (APID) and an AP queue index (APQI). The
> + APQI corresponds to a given usage domain number within the adapter. This tuple
> + forms an AP Queue Number (APQN) uniquely identifying an AP queue. AP
> + instructions include a field containing the APQN to identify the AP queue to
> + which the AP command-request message is to be sent for processing.
> +
> + The AP bus will create a sysfs device for each APQN that can be derived from
> + the cross product of the AP adapter and usage domain numbers detected when the
> + AP bus module is loaded. For example, if adapters 4 and 10 (0x0a) and usage
> + domains 6 and 71 (0x47) are assigned to the LPAR, the AP bus will create the
> + following sysfs entries:
> +
> + /sys/devices/ap/card04/04.0006
> + /sys/devices/ap/card04/04.0047
> + /sys/devices/ap/card0a/0a.0006
> + /sys/devices/ap/card0a/0a.0047
> +
> + The following symbolic links to these devices will be created in the AP bus
> + devices subdirectory:
> +
> + /sys/bus/ap/devices/[04.0006]
> + /sys/bus/ap/devices/[04.0047]
> + /sys/bus/ap/devices/[0a.0006]
> + /sys/bus/ap/devices/[0a.0047]
> +
> +* AP Instructions:
> +
> + There are three AP instructions:
> +
> + * NQAP: to enqueue an AP command-request message to a queue
> + * DQAP: to dequeue an AP command-reply message from a queue
> + * PQAP: to administer the queues

So, NQAP/DQAP need usage domains, while PQAP needs a control domain? Or
is it that all of them need usage domains, but PQAP can target a control
domain as well?

[I don't want to dive deeply into the AP architecture here, just far
enough to really understand the design implications.]

> +
> +AP and SIE:
> +==========
> +Let's now take a look at how AP instructions executed on a guest are interpreted
> +by the hardware.
> +
> +A satellite control block called the Crypto Control Block (CRYCB) is attached to
> +our main hardware virtualization control block. The CRYCB contains three fields
> +to identify the adapters, usage domains and control domains assigned to the KVM
> +guest:
> +
> +* The AP Mask (APM) field is a bit mask that identifies the AP adapters assigned
> + to the KVM guest. Each bit in the mask, from most significant to least
> + significant bit, corresponds to an APID from 0-255. If a bit is set, the
> + corresponding adapter is valid for use by the KVM guest.
> +
> +* The AP Queue Mask (AQM) field is a bit mask identifying the AP usage domains
> + assigned to the KVM guest. Each bit in the mask, from most significant to
> + least significant bit, corresponds to an AP queue index (APQI) from 0-255. If
> + a bit is set, the corresponding queue is valid for use by the KVM guest.
> +
> +* The AP Domain Mask field is a bit mask that identifies the AP control domains
> + assigned to the KVM guest. The ADM bit mask controls which domains can be
> + changed by an AP command-request message sent to a usage domain from the
> + guest. Each bit in the mask, from least significant to most significant bit,
> + corresponds to a domain from 0-255. If a bit is set, the corresponding domain
> + can be modified by an AP command-request message sent to a usage domain
> + configured for the KVM guest.

OK, that seems to imply that you modify a control domain by sending a
request to (any) usage domain? I do not doubt that, but the whole
architecture is really confusing :)

> +
> +If you recall from the description of an AP Queue, AP instructions include
> +an APQN to identify the AP adapter and AP queue to which an AP command-request
> +message is to be sent (NQAP and PQAP instructions), or from which a
> +command-reply message is to be received (DQAP instruction). The validity of an
> +APQN is defined by the matrix calculated from the APM and AQM; it is the
> +cross product of all assigned adapter numbers (APM) with all assigned queue
> +indexes (AQM). For example, if adapters 1 and 2 and usage domains 5 and 6 are
> +assigned to a guest, the APQNs (1,5), (1,6), (2,5) and (2,6) will be valid for
> +the guest.

How does the control domain mask interact with that? Can you send a
command to an APQN valid for the guest to modify any control domain
specified in the mask? Does the SIE complain if you specify a control
domain that the host does not have access to (I'd guess so)?

> +
> +The APQNs can provide secure key functionality - i.e., a private key is stored
> +on the adapter card for each of its domains - so each APQN must be assigned to
> +at most one guest or to the linux host.
> +
> + Example 1: Valid configuration:
> + ------------------------------
> + Guest1: adapters 1,2 domains 5,6
> + Guest2: adapter 1,2 domain 7
> +
> + This is valid because both guests have a unique set of APQNs: Guest1 has
> + APQNs (1,5), (1,6), (2,5) and (2,6); Guest2 has APQNs (1,7) and (2,7).
> +
> + Example 2: Invalid configuration:
> + Guest1: adapters 1,2 domains 5,6
> + Guest2: adapter 1 domains 6,7
> +
> + This is an invalid configuration because both guests have access to
> + APQN (1,6).

So, the adapters or the domains can overlap , but the cross product
mustn't? If I had

Guest1: adapters 1,2 domains 5,6
Guest2: adapters 3,4 domains 5,6

would that be fine?

Is there any rule about shared control domains?

(...)

> +Limitations
> +===========
> +* The KVM/kernel interfaces do not provide a way to prevent unbinding an AP
> + queue that is still assigned to a mediated device. Even if the device
> + 'remove' callback returns an error, the device core detaches the AP
> + queue from the VFIO AP driver. It is therefore incumbent upon the
> + administrator to make sure there is no mediated device to which the
> + APQN - for the AP queue being unbound - is assigned.
> +
> +* Hot plug/unplug of AP devices is not supported for guests.

Not sure what that sentence means. Adding/removing devices by the
hypervisor is not supported? Or some guest actions, respectively
injecting notifications that would trigger some actions on the real
hardware?

Do you want to add (some of) this in the future?

> +
> +* Live guest migration is not supported for guests using AP devices.

Migration and vfio is an interesting area in general :) Would be great
if vfio-ap could benefit from any generic efforts in that area, but
that probably requires that someone with access to documentation and
hardware keeps an eye on developments.

> \ No newline at end of file

Please add one :)

2018-08-20 16:43:34

[permalink] [raw]

Subject: Re: [PATCH v9 07/22] KVM: s390: refactor crypto initialization

On 13.08.2018 23:48, Tony Krowiak wrote:
> From: Tony Krowiak <[email protected]>
>
> This patch refactors the code that initializes and sets up the
> crypto configuration for a guest. The following changes are
> implemented via this patch:
>
> 1. Prior to the introduction of AP device virtualization, it
> was not necessary to provide guest access to the CRYCB
> unless the MSA extension 3 (MSAX3) facility was installed
> on the host system. With the introduction of AP device
> virtualization, the CRYCB must be made accessible to the
> guest if the AP instructions are installed on the host
> and are to be provided to the guest.
>
> 2. Introduces a flag indicating AP instructions executed on
> the guest shall be interpreted by the firmware. It is
> initialized to indicate AP instructions are to be
> to be interpreted and is used to set the SIE bit for
> each vcpu during vcpu setup.
>
> Signed-off-by: Tony Krowiak <[email protected]>
> Reviewed-by: Halil Pasic <[email protected]>
> Acked-by: Christian Borntraeger <[email protected]>
> Acked-by: Janosch Frank <[email protected]>
> Tested-by: Michael Mueller <[email protected]>
> Tested-by: Farhan Ali <[email protected]>
> Signed-off-by: Christian Borntraeger <[email protected]>
> ---
> arch/s390/include/asm/kvm_host.h | 2 +
> arch/s390/include/uapi/asm/kvm.h | 1 +
> arch/s390/kvm/kvm-s390.c | 81 +++++++++++++++++++-------------------
> 3 files changed, 44 insertions(+), 40 deletions(-)
>
> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
> index af39561..4a739d4 100644
> --- a/arch/s390/include/asm/kvm_host.h
> +++ b/arch/s390/include/asm/kvm_host.h
> @@ -187,6 +187,7 @@ struct kvm_s390_sie_block {
> #define ECA_AIV 0x00200000
> #define ECA_VX 0x00020000
> #define ECA_PROTEXCI 0x00002000
> +#define ECA_APIE 0x00000008
> #define ECA_SII 0x00000001
> __u32 eca; /* 0x004c */
> #define ICPT_INST 0x04
> @@ -256,6 +257,7 @@ struct kvm_s390_sie_block {
> __u8 reservede4[4]; /* 0x00e4 */
> __u64 tecmc; /* 0x00e8 */
> __u8 reservedf0[12]; /* 0x00f0 */
> +#define CRYCB_FORMAT_MASK 0x00000003
> #define CRYCB_FORMAT1 0x00000001
> #define CRYCB_FORMAT2 0x00000003
> __u32 crycbd; /* 0x00fc */
> diff --git a/arch/s390/include/uapi/asm/kvm.h b/arch/s390/include/uapi/asm/kvm.h
> index 4cdaa55..a580dec 100644
> --- a/arch/s390/include/uapi/asm/kvm.h
> +++ b/arch/s390/include/uapi/asm/kvm.h
> @@ -130,6 +130,7 @@ struct kvm_s390_vm_cpu_machine {
> #define KVM_S390_VM_CPU_FEAT_PFMFI 11
> #define KVM_S390_VM_CPU_FEAT_SIGPIF 12
> #define KVM_S390_VM_CPU_FEAT_KSS 13
> +#define KVM_S390_VM_CPU_FEAT_AP 14
> struct kvm_s390_vm_cpu_feat {
> __u64 feat[16];
> };
> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
> index 19f4f44..f7de123 100644
> --- a/arch/s390/kvm/kvm-s390.c
> +++ b/arch/s390/kvm/kvm-s390.c
> @@ -40,6 +40,7 @@
> #include <asm/sclp.h>
> #include <asm/cpacf.h>
> #include <asm/timex.h>
> +#include <asm/ap.h>
> #include "kvm-s390.h"
> #include "gaccess.h"
>
> @@ -1881,49 +1882,37 @@ long kvm_arch_vm_ioctl(struct file *filp,
> return r;
> }
>
> -static int kvm_s390_query_ap_config(u8 *config)
> -{
> - u32 fcn_code = 0x04000000UL;
> - u32 cc = 0;
> -
> - memset(config, 0, 128);
> - asm volatile(
> - "lgr 0,%1\n"
> - "lgr 2,%2\n"
> - ".long 0xb2af0000\n" /* PQAP(QCI) */
> - "0: ipm %0\n"
> - "srl %0,28\n"
> - "1:\n"
> - EX_TABLE(0b, 1b)
> - : "+r" (cc)
> - : "r" (fcn_code), "r" (config)
> - : "cc", "0", "2", "memory"
> - );
> -
> - return cc;
> -}
> -
> static int kvm_s390_apxa_installed(void)
> {
> - u8 config[128];
> - int cc;
> + struct ap_config_info info;
>
> - if (test_facility(12)) {
> - cc = kvm_s390_query_ap_config(config);
> -
> - if (cc)
> - pr_err("PQAP(QCI) failed with cc=%d", cc);
> - else
> - return config[0] & 0x40;
> + if (ap_instructions_available()) {
> + if (ap_qci(&info) == 0)
> + return info.apxa;
> }
>
> return 0;
> }
>
> +/*
> + * The format of the crypto control block (CRYCB) is specified in the 3 low
> + * order bits of the CRYCB designation (CRYCBD) field as follows:
> + * Format 0: Neither the message security assist extension 3 (MSAX3) nor the
> + * AP extended addressing (APXA) facility are installed.
> + * Format 1: The APXA facility is not installed but the MSAX3 facility is.
> + * Format 2: Both the APXA and MSAX3 facilities are installed
> + */
> static void kvm_s390_set_crycb_format(struct kvm *kvm)
> {
> kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
>
> + /* Clear the CRYCB format bits - i.e., set format 0 by default */
> + kvm->arch.crypto.crycbd &= ~(CRYCB_FORMAT_MASK);
> +
> + /* Check whether MSAX3 is installed */
> + if (!test_kvm_facility(kvm, 76))
> + return;
> +
> if (kvm_s390_apxa_installed())
> kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
> else
> @@ -1941,12 +1930,12 @@ static u64 kvm_s390_get_initial_cpuid(void)
>
> static void kvm_s390_crypto_init(struct kvm *kvm)
> {
> - if (!test_kvm_facility(kvm, 76))
> - return;
> -
> kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
> kvm_s390_set_crycb_format(kvm);
>
> + if (!test_kvm_facility(kvm, 76))
> + return;
> +
> /* Enable AES/DEA protected key functions by default */
> kvm->arch.crypto.aes_kw = 1;
> kvm->arch.crypto.dea_kw = 1;
> @@ -2474,17 +2463,29 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
>
> static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
> {
> - if (!test_kvm_facility(vcpu->kvm, 76))
> + /*
> + * If neither the AP instructions nor the MSAX3 facility are installed
> + * on the host, then there is no need for a CRYCB in SIE because they
> + * will not be installed on the guest either.
> + */
> + if (!test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_AP) &&
> + !test_facility(76))

I think this can just stay "test_kvm_facility(vcpu->kvm, 76)", and the
comment should be changed to

"If neither the AP instructions nor the MSAX3 facility are configured
for the guest, there is nothing to set up."

Or am I missing something important here?

After the CPU has been created, both feature can no longer change. The
only thing that might change is kvm->arch.crypto.aes_kw/dea_kw but only
with test_kvm_facility(vcpu->kvm, 76).

> return;
>
> + vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
> vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
>
> - if (vcpu->kvm->arch.crypto.aes_kw)
> - vcpu->arch.sie_block->ecb3 |= ECB3_AES;
> - if (vcpu->kvm->arch.crypto.dea_kw)
> - vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
> + vcpu->arch.sie_block->eca &= ~ECA_APIE;

As this feature can never flip, clearing the flag is not necessary.

> + if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_AP))
> + vcpu->arch.sie_block->eca |= ECA_APIE;
>
> - vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
> + /* If MSAX3 is installed on the guest, set up protected key support */
> + if (test_kvm_facility(vcpu->kvm, 76)) {
> + if (vcpu->kvm->arch.crypto.aes_kw)
> + vcpu->arch.sie_block->ecb3 |= ECB3_AES;
> + if (vcpu->kvm->arch.crypto.dea_kw)
> + vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
> + }

As the feature can never change, and aes_kw/dea_kw are only set to 1 in
case we have test_kvm_facility(vcpu->kvm, 76), this change is not needed.

I think this function can be pretty much left alone. Just add the
KVM_S390_VM_CPU_FEAT_AP handling.

> }
>
> void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
>

--

Thanks,

David / dhildenb

2018-08-20 16:45:07

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On 08/20/2018 04:23 PM, Cornelia Huck wrote:
> On Mon, 13 Aug 2018 17:48:09 -0400
> Tony Krowiak <[email protected]> wrote:
>
>> From: Tony Krowiak <[email protected]>
>>
>> Provides the sysfs interfaces for:
>>
>> 1. Assigning AP control domains to the mediated matrix device
>>
>> 2. Unassigning AP control domains from a mediated matrix device
>>
>> 3. Displaying the control domains assigned to a mediated matrix
>> device
>>
>> The IDs of the AP control domains assigned to the mediated matrix
>> device are stored in an AP domain mask (ADM). The bits in the ADM,
>> from most significant to least significant bit, correspond to
>> AP domain numbers 0 to 255. On some systems, the maximum allowable
>> domain number may be less than 255 - depending upon the host's
>> AP configuration - and assignment may be rejected if the input
>> domain ID exceeds the limit.
>
> Please remind me of the relationship between control domains and usage
> domains... IIRC, usage domains allow both requests and configuration,
> while control domains allow only configuration, and are by convention a
> superset of usage domains.
>

The whole terminology with control and usage domains is IMHO a bit
confusing. With the HMC one can assign a domain either as a 'Control'
or as a 'Control and Usage' domain.

Regarding the masks in the CRYCB, the AQM controls 'using' the domain
(e.g. if AQM bit is zero NQAP will refuse to enqueue on that queue)
while ADM tells if the guest is allowed to 'change' the given domain.
Whether a command-request is of type 'using' or 'changing' is a property
of the command request.

You can think of 'using' a domain like signing stuff with a key residing
within the domain, and of 'changing' a domain like issuing a command to
generate a new key for the given domain.

> Is there a hard requirement somewhere in there, or can the admin
> cheerfully use different masks for usage domains and control domains
> without the SIE choking on it?
>

It is a convention. AFAIR it ain't architecture. SIE won't choke on it
I've tried it out. I was arguing along the lines that the kernel should
not enforce this convention -- tooling can still do that if we want this
enforced.

Regards,
Halil

2018-08-20 17:43:08

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On 08/20/2018 10:23 AM, Cornelia Huck wrote:
> On Mon, 13 Aug 2018 17:48:09 -0400
> Tony Krowiak <[email protected]> wrote:
>
>> From: Tony Krowiak <[email protected]>
>>
>> Provides the sysfs interfaces for:
>>
>> 1. Assigning AP control domains to the mediated matrix device
>>
>> 2. Unassigning AP control domains from a mediated matrix device
>>
>> 3. Displaying the control domains assigned to a mediated matrix
>> device
>>
>> The IDs of the AP control domains assigned to the mediated matrix
>> device are stored in an AP domain mask (ADM). The bits in the ADM,
>> from most significant to least significant bit, correspond to
>> AP domain numbers 0 to 255. On some systems, the maximum allowable
>> domain number may be less than 255 - depending upon the host's
>> AP configuration - and assignment may be rejected if the input
>> domain ID exceeds the limit.
> Please remind me of the relationship between control domains and usage
> domains... IIRC, usage domains allow both requests and configuration,
> while control domains allow only configuration, and are by convention a
> superset of usage domains.

A usage domain is a domain to which an AP command-request message can be
submitted for processing. A control domain is a domain that can
be changed by an AP command request message submitted to a usage domain.
AP command request messages to configure a domain will contain the domain
number of the domain to be modified. The AP firmware will check the
control domain mask (ADM) and will allow the request to proceed only if
the corresponding bit in the ADM is set.

>
> Is there a hard requirement somewhere in there, or can the admin
> cheerfully use different masks for usage domains and control domains
> without the SIE choking on it?

There is no hard requirement that control domains must be a superset of
the usage domains, it is merely an architectural convention. AFAIK,
SIE doesn't enforce this and will not break if the convention is not
enforced externally. Having said that, you should note that the AQM
and ADM masks configured for the mediated matrix device will be logically
OR'd together to create the ADM stored in the CRYCB referenced from the
guest's SIE state description. In other words, we are enforcing the
convention in our software.

>

2018-08-20 20:17:54

[permalink] [raw]

Subject: Re: [PATCH v9 22/22] s390: doc: detailed specifications for AP virtualization

On 08/20/2018 12:03 PM, Cornelia Huck wrote:
> On Mon, 13 Aug 2018 17:48:19 -0400
> Tony Krowiak <[email protected]> wrote:
>
>> From: Tony Krowiak <[email protected]>
>>
>> This patch provides documentation describing the AP architecture and
>> design concepts behind the virtualization of AP devices. It also
>> includes an example of how to configure AP devices for exclusive
>> use of KVM guests.
>>
>> Signed-off-by: Tony Krowiak <[email protected]>
>> Reviewed-by: Halil Pasic <[email protected]>
>> Signed-off-by: Christian Borntraeger <[email protected]>
>> ---
>> Documentation/s390/vfio-ap.txt | 615 ++++++++++++++++++++++++++++++++++++++++
>> MAINTAINERS | 1 +
>> 2 files changed, 616 insertions(+), 0 deletions(-)
>> create mode 100644 Documentation/s390/vfio-ap.txt
>>
>> +AP Architectural Overview:
>> +=========================
>> +To facilitate the comprehension of the design, let's start with some
>> +definitions:
>> +
>> +* AP adapter
>> +
>> + An AP adapter is an IBM Z adapter card that can perform cryptographic
>> + functions. There can be from 0 to 256 adapters assigned to an LPAR. Adapters
>> + assigned to the LPAR in which a linux host is running will be available to
>> + the linux host. Each adapter is identified by a number from 0 to 255. When
>> + installed, an AP adapter is accessed by AP instructions executed by any CPU.
>> +
>> + The AP adapter cards are assigned to a given LPAR via the system's Activation
>> + Profile which can be edited via the HMC. When the system is IPL'd, the AP bus
> There's lots of s390 jargon in here... but one hopes that someone
> trying to understand AP is already familiar with the basics...

I'm not quite sure how one can describe s390-specific devices that can
be installed
only on an s390 system without using s390 jargon. I would think that one
who is
administering a linux host or guest running on an s390 system would have
some
basic knowledge of s390. If you have any suggestions, I'd be happy to
entertain them.

>
>> + module is loaded and detects the AP adapter cards assigned to the LPAR. The AP
>> + bus creates a sysfs device for each adapter as they are detected. For example,
>> + if AP adapters 4 and 10 (0x0a) are assigned to the LPAR, the AP bus will
>> + create the following sysfs entries:
>> +
>> + /sys/devices/ap/card04
>> + /sys/devices/ap/card0a
>> +
>> + Symbolic links to these devices will also be created in the AP bus devices
>> + sub-directory:
>> +
>> + /sys/bus/ap/devices/[card04]
>> + /sys/bus/ap/devices/[card04]
>> +
>> +* AP domain
>> +
>> + An adapter is partitioned into domains. Each domain can be thought of as
>> + a set of hardware registers for processing AP instructions. An adapter can
>> + hold up to 256 domains. Each domain is identified by a number from 0 to 255.
>> + Domains can be further classified into two types:
>> +
>> + * Usage domains are domains that can be accessed directly to process AP
>> + commands.
>> +
>> + * Control domains are domains that are accessed indirectly by AP
>> + commands sent to a usage domain to control or change the domain; for
>> + example, to set a secure private key for the domain.
>> +
>> + The AP usage and control domains are assigned to a given LPAR via the system's
>> + Activation Profile which can be edited via the HMC. When the system is IPL'd,
>> + the AP bus module is loaded and detects the AP usage and control domains
>> + assigned to the LPAR. The domain number of each usage domain will be coupled
>> + with the adapter number of each AP adapter assigned to the LPAR to identify
>> + the AP queues (see AP Queue section below). The domain number of each control
>> + domain will be represented in a bitmask and stored in a sysfs file
>> + /sys/bus/ap/ap_control_domain_mask created by the bus. The bits in the mask,
>> + from most to least significant bit, correspond to domains 0-255.
>> +
>> + A domain may be assigned to a system as both a usage and control domain, or
>> + as a control domain only. Consequently, all domains assigned as both a usage
>> + and control domain can both process AP commands as well as be changed by an AP
>> + command sent to any usage domain assigned to the same system. Domains assigned
>> + only as control domains can not process AP commands but can be changed by AP
>> + commands sent to any usage domain assigned to the system.
> I'm struggling a bit with this paragraph. Does that mean that you can
> use control domains as the target of an instruction changing
> configuration on the system? (Or on the VM, if they are listed in the
> relevant control block?)

Only usage domains can be the target of an AP command request message.
If an AP message
sent to a usage domain is a request to change a domain, the number of
the domain to
be changed will be contained in the command request message. That domain
number
must be configured as a control domain or the AP command will fail.

The fact you are struggling with understanding the last paragraph leads
me to believe
it should probably be rewritten, or eliminated. Allow me to reconsider
this section.

>
>> +
>> +* AP Queue
>> +
>> + An AP queue is the means by which an AP command-request message is sent to a
>> + usage domain inside a specific adapter. An AP queue is identified by a tuple
>> + comprised of an AP adapter ID (APID) and an AP queue index (APQI). The
>> + APQI corresponds to a given usage domain number within the adapter. This tuple
>> + forms an AP Queue Number (APQN) uniquely identifying an AP queue. AP
>> + instructions include a field containing the APQN to identify the AP queue to
>> + which the AP command-request message is to be sent for processing.
>> +
>> + The AP bus will create a sysfs device for each APQN that can be derived from
>> + the cross product of the AP adapter and usage domain numbers detected when the
>> + AP bus module is loaded. For example, if adapters 4 and 10 (0x0a) and usage
>> + domains 6 and 71 (0x47) are assigned to the LPAR, the AP bus will create the
>> + following sysfs entries:
>> +
>> + /sys/devices/ap/card04/04.0006
>> + /sys/devices/ap/card04/04.0047
>> + /sys/devices/ap/card0a/0a.0006
>> + /sys/devices/ap/card0a/0a.0047
>> +
>> + The following symbolic links to these devices will be created in the AP bus
>> + devices subdirectory:
>> +
>> + /sys/bus/ap/devices/[04.0006]
>> + /sys/bus/ap/devices/[04.0047]
>> + /sys/bus/ap/devices/[0a.0006]
>> + /sys/bus/ap/devices/[0a.0047]
>> +
>> +* AP Instructions:
>> +
>> + There are three AP instructions:
>> +
>> + * NQAP: to enqueue an AP command-request message to a queue
>> + * DQAP: to dequeue an AP command-reply message from a queue
>> + * PQAP: to administer the queues
> So, NQAP/DQAP need usage domains, while PQAP needs a control domain? Or
> is it that all of them need usage domains, but PQAP can target a control
> domain as well?

All AP instructions - the lone exception being the PQAP(QCI) subfunction -
identify the usage domain that is the target of the instruction. I think
using the term 'control domain' is the source of much confusion. It makes
it seem as if there are two types of domains that serve different purposes.
That is simply not true. A domain is a partition within an AP adapter that
can process AP command request messages. All AP commands are sent to a
domain. Configuring a domain as a usage domain means it can be used to
process AP commands; in other words, it can be the target of an AP
instruction. Configuring a domain as a control domain means it can be
changed by an AP command. AP commands that change a domain are sent to
a usage domain, but the domain to be changed is specified in the payload
of the AP command message. The domain thus specified must be
identified via the AP configuration as a control domain, or the AP command
will be rejected.

>
> [I don't want to dive deeply into the AP architecture here, just far
> enough to really understand the design implications.]

Are you suggesting some of the above should be removed? If so, what?

>
>> +
>> +AP and SIE:
>> +==========
>> +Let's now take a look at how AP instructions executed on a guest are interpreted
>> +by the hardware.
>> +
>> +A satellite control block called the Crypto Control Block (CRYCB) is attached to
>> +our main hardware virtualization control block. The CRYCB contains three fields
>> +to identify the adapters, usage domains and control domains assigned to the KVM
>> +guest:
>> +
>> +* The AP Mask (APM) field is a bit mask that identifies the AP adapters assigned
>> + to the KVM guest. Each bit in the mask, from most significant to least
>> + significant bit, corresponds to an APID from 0-255. If a bit is set, the
>> + corresponding adapter is valid for use by the KVM guest.
>> +
>> +* The AP Queue Mask (AQM) field is a bit mask identifying the AP usage domains
>> + assigned to the KVM guest. Each bit in the mask, from most significant to
>> + least significant bit, corresponds to an AP queue index (APQI) from 0-255. If
>> + a bit is set, the corresponding queue is valid for use by the KVM guest.
>> +
>> +* The AP Domain Mask field is a bit mask that identifies the AP control domains
>> + assigned to the KVM guest. The ADM bit mask controls which domains can be
>> + changed by an AP command-request message sent to a usage domain from the
>> + guest. Each bit in the mask, from least significant to most significant bit,
>> + corresponds to a domain from 0-255. If a bit is set, the corresponding domain
>> + can be modified by an AP command-request message sent to a usage domain
>> + configured for the KVM guest.
> OK, that seems to imply that you modify a control domain by sending a
> request to (any) usage domain?

That is a true statement. I reality, you are just modifying a domain.
The control
domain designation identifies a domain that can be controlled as opposed
to used.
Maybe if you think of these bitmasks as access control masks it would
clarify
things. The AQM specifies domains to which AP commands can be sent and the
ADM specifies domains that can be changed by an AP command.

> I do not doubt that, but the whole
> architecture is really confusing :)

I couldn't agree more. It took me a while to wrap my head around it.

>
>> +
>> +If you recall from the description of an AP Queue, AP instructions include
>> +an APQN to identify the AP adapter and AP queue to which an AP command-request
>> +message is to be sent (NQAP and PQAP instructions), or from which a
>> +command-reply message is to be received (DQAP instruction). The validity of an
>> +APQN is defined by the matrix calculated from the APM and AQM; it is the
>> +cross product of all assigned adapter numbers (APM) with all assigned queue
>> +indexes (AQM). For example, if adapters 1 and 2 and usage domains 5 and 6 are
>> +assigned to a guest, the APQNs (1,5), (1,6), (2,5) and (2,6) will be valid for
>> +the guest.
> How does the control domain mask interact with that?

The control domain mask does not interact with the other two masks. It
merely specifies which domains can be modified by an AP command. In fact,
the ADM can have bits set that are not included in the AQM; in other words,
a guest can be used to control domains that it can not use.

> Can you send a
> command to an APQN valid for the guest to modify any control domain
> specified in the mask?

Yes.

> Does the SIE complain if you specify a control
> domain that the host does not have access to (I'd guess so)?

The SIE does not complain if you specify a domain to which the host - or a
lower level guest - does not have access. The firmware performs a logical
AND of the guest's and hosts's - or lower level guest's - APMs, AQMs and
ADMs
to create effective masks EAPM, EAQM and EADM. Only devices corresponding to
the bits set in the EAPM, EAQM and EADM will be accessible by the guest.

>
>> +
>> +The APQNs can provide secure key functionality - i.e., a private key is stored
>> +on the adapter card for each of its domains - so each APQN must be assigned to
>> +at most one guest or to the linux host.
>> +
>> + Example 1: Valid configuration:
>> + ------------------------------
>> + Guest1: adapters 1,2 domains 5,6
>> + Guest2: adapter 1,2 domain 7
>> +
>> + This is valid because both guests have a unique set of APQNs: Guest1 has
>> + APQNs (1,5), (1,6), (2,5) and (2,6); Guest2 has APQNs (1,7) and (2,7).
>> +
>> + Example 2: Invalid configuration:
>> + Guest1: adapters 1,2 domains 5,6
>> + Guest2: adapter 1 domains 6,7
>> +
>> + This is an invalid configuration because both guests have access to
>> + APQN (1,6).
> So, the adapters or the domains can overlap , but the cross product
> mustn't? If I had
>
> Guest1: adapters 1,2 domains 5,6
> Guest2: adapters 3,4 domains 5,6
>
> would that be fine?

Yes, that would be fine because Guest1 would have access to APQNs
(1,5), (1,6), (2,5) and (2,6) while Guest2 would have access to
(3,5), (3,6), (4,5) AND (4,6), but neither would have access to
the same APQN.

>
> Is there any rule about shared control domains?

AFAIK there isn't, but I will consult with Reinhard about that.

>
> (...)
>
>> +Limitations
>> +===========
>> +* The KVM/kernel interfaces do not provide a way to prevent unbinding an AP
>> + queue that is still assigned to a mediated device. Even if the device
>> + 'remove' callback returns an error, the device core detaches the AP
>> + queue from the VFIO AP driver. It is therefore incumbent upon the
>> + administrator to make sure there is no mediated device to which the
>> + APQN - for the AP queue being unbound - is assigned.
>> +
>> +* Hot plug/unplug of AP devices is not supported for guests.
> Not sure what that sentence means. Adding/removing devices by the
> hypervisor is not supported? Or some guest actions, respectively
> injecting notifications that would trigger some actions on the real
> hardware?

No means is provided to modify a guest's AP matrix - i.e., APM, AQM
and ADM - while a guest is running. Once a guest is running, its AP
configuration can not be changed dynamically.

>
> Do you want to add (some of) this in the future?

Yes, we plan to introduce dynamic configurations in future releases.

>
>> +
>> +* Live guest migration is not supported for guests using AP devices.
> Migration and vfio is an interesting area in general :) Would be great
> if vfio-ap could benefit from any generic efforts in that area, but
> that probably requires that someone with access to documentation and
> hardware keeps an eye on developments.

I have briefly looked at some of the articles talking about live migration
of passthrough devices, but nothing seemed applicable to AP architecture.
From my limited perspective, it would seem that architectural changes
would have to be implemented to fully support live migration of in-process
AP queues.

>
>> \ No newline at end of file
> Please add one :)

Will do.

>

2018-08-20 20:35:31

[permalink] [raw]

Subject: Re: [PATCH v9 07/22] KVM: s390: refactor crypto initialization

On 08/20/2018 12:41 PM, David Hildenbrand wrote:
> On 13.08.2018 23:48, Tony Krowiak wrote:
>> From: Tony Krowiak <[email protected]>
>>
>> This patch refactors the code that initializes and sets up the
>> crypto configuration for a guest. The following changes are
>> implemented via this patch:
>>
>> 1. Prior to the introduction of AP device virtualization, it
>> was not necessary to provide guest access to the CRYCB
>> unless the MSA extension 3 (MSAX3) facility was installed
>> on the host system. With the introduction of AP device
>> virtualization, the CRYCB must be made accessible to the
>> guest if the AP instructions are installed on the host
>> and are to be provided to the guest.
>>
>> 2. Introduces a flag indicating AP instructions executed on
>> the guest shall be interpreted by the firmware. It is
>> initialized to indicate AP instructions are to be
>> to be interpreted and is used to set the SIE bit for
>> each vcpu during vcpu setup.
>>
>> Signed-off-by: Tony Krowiak <[email protected]>
>> Reviewed-by: Halil Pasic <[email protected]>
>> Acked-by: Christian Borntraeger <[email protected]>
>> Acked-by: Janosch Frank <[email protected]>
>> Tested-by: Michael Mueller <[email protected]>
>> Tested-by: Farhan Ali <[email protected]>
>> Signed-off-by: Christian Borntraeger <[email protected]>
>> ---
>> arch/s390/include/asm/kvm_host.h | 2 +
>> arch/s390/include/uapi/asm/kvm.h | 1 +
>> arch/s390/kvm/kvm-s390.c | 81 +++++++++++++++++++-------------------
>> 3 files changed, 44 insertions(+), 40 deletions(-)
>>
>> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
>> index af39561..4a739d4 100644
>> --- a/arch/s390/include/asm/kvm_host.h
>> +++ b/arch/s390/include/asm/kvm_host.h
>> @@ -187,6 +187,7 @@ struct kvm_s390_sie_block {
>> #define ECA_AIV 0x00200000
>> #define ECA_VX 0x00020000
>> #define ECA_PROTEXCI 0x00002000
>> +#define ECA_APIE 0x00000008
>> #define ECA_SII 0x00000001
>> __u32 eca; /* 0x004c */
>> #define ICPT_INST 0x04
>> @@ -256,6 +257,7 @@ struct kvm_s390_sie_block {
>> __u8 reservede4[4]; /* 0x00e4 */
>> __u64 tecmc; /* 0x00e8 */
>> __u8 reservedf0[12]; /* 0x00f0 */
>> +#define CRYCB_FORMAT_MASK 0x00000003
>> #define CRYCB_FORMAT1 0x00000001
>> #define CRYCB_FORMAT2 0x00000003
>> __u32 crycbd; /* 0x00fc */
>> diff --git a/arch/s390/include/uapi/asm/kvm.h b/arch/s390/include/uapi/asm/kvm.h
>> index 4cdaa55..a580dec 100644
>> --- a/arch/s390/include/uapi/asm/kvm.h
>> +++ b/arch/s390/include/uapi/asm/kvm.h
>> @@ -130,6 +130,7 @@ struct kvm_s390_vm_cpu_machine {
>> #define KVM_S390_VM_CPU_FEAT_PFMFI 11
>> #define KVM_S390_VM_CPU_FEAT_SIGPIF 12
>> #define KVM_S390_VM_CPU_FEAT_KSS 13
>> +#define KVM_S390_VM_CPU_FEAT_AP 14
>> struct kvm_s390_vm_cpu_feat {
>> __u64 feat[16];
>> };
>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
>> index 19f4f44..f7de123 100644
>> --- a/arch/s390/kvm/kvm-s390.c
>> +++ b/arch/s390/kvm/kvm-s390.c
>> @@ -40,6 +40,7 @@
>> #include <asm/sclp.h>
>> #include <asm/cpacf.h>
>> #include <asm/timex.h>
>> +#include <asm/ap.h>
>> #include "kvm-s390.h"
>> #include "gaccess.h"
>>
>> @@ -1881,49 +1882,37 @@ long kvm_arch_vm_ioctl(struct file *filp,
>> return r;
>> }
>>
>> -static int kvm_s390_query_ap_config(u8 *config)
>> -{
>> - u32 fcn_code = 0x04000000UL;
>> - u32 cc = 0;
>> -
>> - memset(config, 0, 128);
>> - asm volatile(
>> - "lgr 0,%1\n"
>> - "lgr 2,%2\n"
>> - ".long 0xb2af0000\n" /* PQAP(QCI) */
>> - "0: ipm %0\n"
>> - "srl %0,28\n"
>> - "1:\n"
>> - EX_TABLE(0b, 1b)
>> - : "+r" (cc)
>> - : "r" (fcn_code), "r" (config)
>> - : "cc", "0", "2", "memory"
>> - );
>> -
>> - return cc;
>> -}
>> -
>> static int kvm_s390_apxa_installed(void)
>> {
>> - u8 config[128];
>> - int cc;
>> + struct ap_config_info info;
>>
>> - if (test_facility(12)) {
>> - cc = kvm_s390_query_ap_config(config);
>> -
>> - if (cc)
>> - pr_err("PQAP(QCI) failed with cc=%d", cc);
>> - else
>> - return config[0] & 0x40;
>> + if (ap_instructions_available()) {
>> + if (ap_qci(&info) == 0)
>> + return info.apxa;
>> }
>>
>> return 0;
>> }
>>
>> +/*
>> + * The format of the crypto control block (CRYCB) is specified in the 3 low
>> + * order bits of the CRYCB designation (CRYCBD) field as follows:
>> + * Format 0: Neither the message security assist extension 3 (MSAX3) nor the
>> + * AP extended addressing (APXA) facility are installed.
>> + * Format 1: The APXA facility is not installed but the MSAX3 facility is.
>> + * Format 2: Both the APXA and MSAX3 facilities are installed
>> + */
>> static void kvm_s390_set_crycb_format(struct kvm *kvm)
>> {
>> kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
>>
>> + /* Clear the CRYCB format bits - i.e., set format 0 by default */
>> + kvm->arch.crypto.crycbd &= ~(CRYCB_FORMAT_MASK);
>> +
>> + /* Check whether MSAX3 is installed */
>> + if (!test_kvm_facility(kvm, 76))
>> + return;
>> +
>> if (kvm_s390_apxa_installed())
>> kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
>> else
>> @@ -1941,12 +1930,12 @@ static u64 kvm_s390_get_initial_cpuid(void)
>>
>> static void kvm_s390_crypto_init(struct kvm *kvm)
>> {
>> - if (!test_kvm_facility(kvm, 76))
>> - return;
>> -
>> kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
>> kvm_s390_set_crycb_format(kvm);
>>
>> + if (!test_kvm_facility(kvm, 76))
>> + return;
>> +
>> /* Enable AES/DEA protected key functions by default */
>> kvm->arch.crypto.aes_kw = 1;
>> kvm->arch.crypto.dea_kw = 1;
>> @@ -2474,17 +2463,29 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
>>
>> static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
>> {
>> - if (!test_kvm_facility(vcpu->kvm, 76))
>> + /*
>> + * If neither the AP instructions nor the MSAX3 facility are installed
>> + * on the host, then there is no need for a CRYCB in SIE because they
>> + * will not be installed on the guest either.
>> + */
>> + if (!test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_AP) &&
>> + !test_facility(76))
> I think this can just stay "test_kvm_facility(vcpu->kvm, 76)", and the
> comment should be changed to
>
> "If neither the AP instructions nor the MSAX3 facility are configured
> for the guest, there is nothing to set up."
>
> Or am I missing something important here?

No, you're not missing anything, it can be test_kvm_facility() and I'm
more than happy to change the comment.

>
> After the CPU has been created, both feature can no longer change. The
> only thing that might change is kvm->arch.crypto.aes_kw/dea_kw but only
> with test_kvm_facility(vcpu->kvm, 76).
>
>> return;
>>
>> + vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
>> vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
>>
>> - if (vcpu->kvm->arch.crypto.aes_kw)
>> - vcpu->arch.sie_block->ecb3 |= ECB3_AES;
>> - if (vcpu->kvm->arch.crypto.dea_kw)
>> - vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
>> + vcpu->arch.sie_block->eca &= ~ECA_APIE;
> As this feature can never flip, clearing the flag is not necessary.

Okay.

>
>> + if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_AP))
>> + vcpu->arch.sie_block->eca |= ECA_APIE;
>>
>> - vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
>> + /* If MSAX3 is installed on the guest, set up protected key support */
>> + if (test_kvm_facility(vcpu->kvm, 76)) {
>> + if (vcpu->kvm->arch.crypto.aes_kw)
>> + vcpu->arch.sie_block->ecb3 |= ECB3_AES;
>> + if (vcpu->kvm->arch.crypto.dea_kw)
>> + vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
>> + }
> As the feature can never change, and aes_kw/dea_kw are only set to 1 in
> case we have test_kvm_facility(vcpu->kvm, 76), this change is not needed.
>
> I think this function can be pretty much left alone. Just add the
> KVM_S390_VM_CPU_FEAT_AP handling.

I disagree, what about the case where the KVM_S390_VM_CPU_FEAT_AP is
configured for the guest but the MSAX3 facility (76) is not?

>
>> }
>>
>> void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
>>
>

2018-08-20 20:43:31

[permalink] [raw]

Subject: Re: [PATCH v9 07/22] KVM: s390: refactor crypto initialization

>>
>>> + if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_AP))
>>> + vcpu->arch.sie_block->eca |= ECA_APIE;
>>>
>>> - vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
>>> + /* If MSAX3 is installed on the guest, set up protected key support */
>>> + if (test_kvm_facility(vcpu->kvm, 76)) {
>>> + if (vcpu->kvm->arch.crypto.aes_kw)
>>> + vcpu->arch.sie_block->ecb3 |= ECB3_AES;
>>> + if (vcpu->kvm->arch.crypto.dea_kw)
>>> + vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
>>> + }
>> As the feature can never change, and aes_kw/dea_kw are only set to 1 in
>> case we have test_kvm_facility(vcpu->kvm, 76), this change is not needed.
>>
>> I think this function can be pretty much left alone. Just add the
>> KVM_S390_VM_CPU_FEAT_AP handling.
>
> I disagree, what about the case where the KVM_S390_VM_CPU_FEAT_AP is
> configured for the guest but the MSAX3 facility (76) is not?

Then aes_kw/dea_kw can never be set.

kvm_s390_vm_set_crypto() and kvm_s390_crypto_init() correctly test for
facility 76.

Or am I missing a case?

>
>>
>>> }
>>>
>>> void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
>>>
>>
>

--

Thanks,

David / dhildenb

2018-08-21 08:05:19

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

On 13.08.2018 23:48, Tony Krowiak wrote:
> From: Tony Krowiak <[email protected]>
>
> Introduces a new CPU model feature and two CPU model
> facilities to support AP virtualization for KVM guests.
>
> CPU model feature:
>
> The KVM_S390_VM_CPU_FEAT_AP feature indicates that
> AP instructions are available on the guest. This
> feature will be enabled by the kernel only if the AP
> instructions are installed on the linux host. This feature
> must be specifically turned on for the KVM guest from
> userspace to use the VFIO AP device driver for guest
> access to AP devices.
>
> CPU model facilities:
>
> 1. AP Query Configuration Information (QCI) facility is installed.
>
> This is indicated by setting facilities bit 12 for
> the guest. The kernel will not enable this facility
> for the guest if it is not set on the host.
>
> If this facility is not set for the KVM guest, then only
> APQNs with an APQI less than 16 will be used by a Linux
> guest regardless of the matrix configuration for the virtual
> machine. This is a limitation of the Linux AP bus.
>
> 2. AP Facilities Test facility (APFT) is installed.
>
> This is indicated by setting facilities bit 15 for
> the guest. The kernel will not enable this facility for
> the guest if it is not set on the host.
>
> If this facility is not set for the KVM guest, then no
> AP devices will be available to the guest regardless of
> the guest's matrix configuration for the virtual
> machine. This is a limitation of the Linux AP bus.
>
> Signed-off-by: Tony Krowiak <[email protected]>
> Reviewed-by: Christian Borntraeger <[email protected]>
> Reviewed-by: Halil Pasic <[email protected]>
> Tested-by: Michael Mueller <[email protected]>
> Tested-by: Farhan Ali <[email protected]>
> Signed-off-by: Christian Borntraeger <[email protected]>
> ---
> arch/s390/kvm/kvm-s390.c | 5 +++++
> arch/s390/tools/gen_facilities.c | 2 ++
> 2 files changed, 7 insertions(+), 0 deletions(-)
>
> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
> index 1e8cb67..d5e04d2 100644
> --- a/arch/s390/kvm/kvm-s390.c
> +++ b/arch/s390/kvm/kvm-s390.c
> @@ -367,6 +367,11 @@ static void kvm_s390_cpu_feat_init(void)
>
> if (MACHINE_HAS_ESOP)
> allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
> +
> + /* Check if AP instructions installed on host */
> + if (ap_instructions_available())
> + allow_cpu_feat(KVM_S390_VM_CPU_FEAT_AP);
> +
> /*
> * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
> * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
> diff --git a/arch/s390/tools/gen_facilities.c b/arch/s390/tools/gen_facilities.c
> index 90a8c9e..a52290b 100644
> --- a/arch/s390/tools/gen_facilities.c
> +++ b/arch/s390/tools/gen_facilities.c
> @@ -106,6 +106,8 @@ struct facility_def {
>
> .name = "FACILITIES_KVM_CPUMODEL",
> .bits = (int[]){
> + 12, /* AP Query Configuration Information */
> + 15, /* AP Facilities Test */
> -1 /* END */
> }
> },
>

Reviewed-by: David Hildenbrand <[email protected]>

--

Thanks,

David / dhildenb

2018-08-21 08:32:05

[permalink] [raw]

Subject: Re: [PATCH v9 20/22] KVM: s390: Handling of Cypto control block in VSIE

On 13.08.2018 23:48, Tony Krowiak wrote:
> From: Pierre Morel <[email protected]>
>
> Shadowing the crypto control block now supports APCB shadowing.
>
> AP instruction interpretation for guest 3 through ECA.28 is shadowed when
> guest 2 ECA.28 is set.
>
> CRYCB is shadowed for APCB and wrapping keys.
>
> CRYCB format 0 is now supported for both guests 2 and 3.
>
> Shadow CRYCB always uses the guest 2 CRYCB format and it
> follows that:
>
> * Guest 3 CRYCB format 0 is supported with guest 2 CRYCB format 0,1 or 2
> * Guest 3 CRYCB format 1 is supported with guest 2 CRYCB format 1 or 2
> * Guest 3 CRYCB format 2 is supported with guest 2 CRYCB format 2
>
> Signed-off-by: Pierre Morel <[email protected]>
> Signed-off-by: Tony Krowiak <[email protected]>
> Acked-by: Halil Pasic <[email protected]>
> Tested-by: Michael Mueller <[email protected]>
> Tested-by: Farhan Ali <[email protected]>
> Signed-off-by: Christian Borntraeger <[email protected]>
> ---
> arch/s390/kvm/vsie.c | 222 ++++++++++++++++++++++++++++++++++++++++++++------
> 1 files changed, 198 insertions(+), 24 deletions(-)
>
> diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c
> index aa30b48..e15240e 100644
> --- a/arch/s390/kvm/vsie.c
> +++ b/arch/s390/kvm/vsie.c
> @@ -136,17 +136,8 @@ static int prepare_cpuflags(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
> return 0;
> }
>
> -/*
> - * Create a shadow copy of the crycb block and setup key wrapping, if
> - * requested for guest 3 and enabled for guest 2.
> - *
> - * We only accept format-1 (no AP in g2), but convert it into format-2
> - * There is nothing to do for format-0.
> - *
> - * Returns: - 0 if shadowed or nothing to do
> - * - > 0 if control has to be given to guest 2
> - */
> -static int shadow_crycb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
> +/* Copy keys into shadow crycb, is only called if MSA3 is available. */
> +static int copy_key_masks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
> {
> struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
> struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
> @@ -155,30 +146,17 @@ static int shadow_crycb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
> unsigned long *b1, *b2;
> u8 ecb3_flags;
>
> - scb_s->crycbd = 0;
> - if (!(crycbd_o & vcpu->arch.sie_block->crycbd & CRYCB_FORMAT1))
> - return 0;
> - /* format-1 is supported with message-security-assist extension 3 */
> - if (!test_kvm_facility(vcpu->kvm, 76))
> - return 0;
> /* we may only allow it if enabled for guest 2 */
> ecb3_flags = scb_o->ecb3 & vcpu->arch.sie_block->ecb3 &
> (ECB3_AES | ECB3_DEA);
> if (!ecb3_flags)
> return 0;
>
> - if ((crycb_addr & PAGE_MASK) != ((crycb_addr + 128) & PAGE_MASK))
> - return set_validity_icpt(scb_s, 0x003CU);
> - else if (!crycb_addr)
> - return set_validity_icpt(scb_s, 0x0039U);
> -
> /* copy only the wrapping keys */
> if (read_guest_real(vcpu, crycb_addr + 72, &vsie_page->crycb, 56))
> return set_validity_icpt(scb_s, 0x0035U);
>
> scb_s->ecb3 |= ecb3_flags;
> - scb_s->crycbd = ((__u32)(__u64) &vsie_page->crycb) | CRYCB_FORMAT1 |
> - CRYCB_FORMAT2;
>
> /* xor both blocks in one run */
> b1 = (unsigned long *) vsie_page->crycb.dea_wrapping_key_mask;
> @@ -189,6 +167,202 @@ static int shadow_crycb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
> return 0;
> }
>
> +/* Copy masks into apcb when g2 and g3 use format 1 */
> +static int copy_apcb1(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
> +{
> + struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
> + struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
> + const uint32_t crycbd_o = READ_ONCE(scb_o->crycbd);
> + const u32 crycb_o = crycbd_o & 0x7ffffff8U;

Can you pass crycb_o instead please to all applicable users from the
initial handler? Otherwise the READ_ONCE() is of no use any more.

> + struct kvm_s390_crypto_cb *crycb_h = &vcpu->kvm->arch.sie_page2->crycb;
> + struct kvm_s390_crypto_cb *crycb_s = &vsie_page->crycb;
> + unsigned long *apcb_s = (unsigned long *) &crycb_s->apcb1;
> + unsigned long *apcb_h = (unsigned long *) &crycb_h->apcb1;
> + int i;
> + u32 src;
> +
> + src = crycb_o + offsetof(struct kvm_s390_crypto_cb, apcb1);
> + if (read_guest_real(vcpu, src, apcb_s, sizeof(struct kvm_s390_apcb1)))

can you instead return -EFAULT and handle 0035U for all cases in the caller?

So you might be able to not pass vsie_page in here.

Same applies to the other handlers below. (and also above)

> + return set_validity_icpt(scb_s, 0x0035U);
> +
> + for (i = 0; i < sizeof(struct kvm_s390_apcb1); i += sizeof(*apcb_s))
> + *apcb_s &= *apcb_h;
> +
> + return 0;
> +}
> +
> +/*
> + * Copy masks into apcb when g2 use format 1 and g3 use format 0
> + * In this case the shadow APCB uses format 1
> + */
> +static int copy_apcb01(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
> +{
> + struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
> + struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
> + const uint32_t crycbd_o = READ_ONCE(scb_o->crycbd);
> + const u32 crycb_o = crycbd_o & 0x7ffffff8U;
> + struct kvm_s390_apcb1 *apcb_h = &vcpu->kvm->arch.sie_page2->crycb.apcb1;
> + struct kvm_s390_apcb1 *apcb_s = &vsie_page->crycb.apcb1;
> + u32 src;
> +
> + memset(apcb_s, 0, sizeof(*apcb_s));
> +
> + src = crycb_o + offsetof(struct kvm_s390_crypto_cb, apcb0.apm[0]);
> + if (read_guest_real(vcpu, src, &apcb_s->apm[0], sizeof(__u64)))
> + return set_validity_icpt(scb_s, 0x0035U);
> +
> + src = crycb_o + offsetof(struct kvm_s390_crypto_cb, apcb0.aqm[0]);
> + if (read_guest_real(vcpu, src, &apcb_s->aqm[0], sizeof(__u64)))
> + return set_validity_icpt(scb_s, 0x0035U);
> +
> + src = crycb_o + offsetof(struct kvm_s390_crypto_cb, apcb0.adm[0]);
> + if (read_guest_real(vcpu, src, &apcb_s->adm[0], sizeof(__u64)))
> + return set_validity_icpt(scb_s, 0x0035U);
> +
> + apcb_s->apm[0] &= apcb_h->apm[0];
> + apcb_s->aqm[0] &= apcb_h->aqm[0];
> + apcb_s->adm[0] &= apcb_h->adm[0];
> +
> + return 0;
> +}
> +
> +/* Copy masks into apcb when g2 and g3 use format 0 */
> +static int copy_apcb0(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
> +{
> + struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
> + struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
> + const uint32_t crycbd_o = READ_ONCE(scb_o->crycbd);
> + const u32 crycb_o = crycbd_o & 0x7ffffff8U;
> + struct kvm_s390_apcb0 *apcb_h = &vcpu->kvm->arch.sie_page2->crycb.apcb0;
> + struct kvm_s390_apcb0 *apcb_s = &vsie_page->crycb.apcb0;
> + u32 src;
> +
> + src = crycb_o + offsetof(struct kvm_s390_crypto_cb, apcb0.apm[0]);
> + if (read_guest_real(vcpu, src, &apcb_s->apm[0], sizeof(__u64)))
> + return set_validity_icpt(scb_s, 0x0035U);
> +
> + src = crycb_o + offsetof(struct kvm_s390_crypto_cb, apcb0.aqm[0]);
> + if (read_guest_real(vcpu, src, &apcb_s->aqm[0], sizeof(__u64)))
> + return set_validity_icpt(scb_s, 0x0035U);
> +
> + src = crycb_o + offsetof(struct kvm_s390_crypto_cb, apcb0.adm[0]);
> + if (read_guest_real(vcpu, src, &apcb_s->adm[0], sizeof(__u64)))
> + return set_validity_icpt(scb_s, 0x0035U);
> +
> + apcb_s->apm[0] &= apcb_h->apm[0];
> + apcb_s->aqm[0] &= apcb_h->aqm[0];
> + apcb_s->adm[0] &= apcb_h->adm[0];
> +
> + return 0;
> +}
> +
> +/* Shadowing APCB depends on G2 and G3 CRYCB format */
> +static int copy_apcb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page,
> + int g2_fmt, int g3_fmt)
> +{
> + struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
> + int ret = 0;
> +
> + switch (g2_fmt) {
> + case CRYCB_FORMAT0:
> + switch (g3_fmt) {
> + case CRYCB_FORMAT0:
> + ret = copy_apcb0(vcpu, vsie_page);
> + break;
> + default:
> + return set_validity_icpt(scb_s, 0x0020U);

return -EINVAL and handle it in the caller (like I suggested with -EFAULT).

Try to not pass vsie_page down here. (same applies to copy_key_masks())

> + }
> + break;
> + case CRYCB_FORMAT1:
> + switch (g3_fmt) {
> + case CRYCB_FORMAT1:
> + case CRYCB_FORMAT0: /* Fall through to copy APCB */
> + ret = copy_apcb0(vcpu, vsie_page);
> + break;
> + default:
> + return set_validity_icpt(scb_s, 0x0020U);
> + }
> + break;
> + case CRYCB_FORMAT2:
> + switch (g3_fmt) {
> + case CRYCB_FORMAT0:
> + case CRYCB_FORMAT1:
> + ret = copy_apcb01(vcpu, vsie_page);
> + break;
> + case CRYCB_FORMAT2:
> + ret = copy_apcb1(vcpu, vsie_page);
> + break;
> + }
> + break;
> + default:
> + /*
> + * Guest 2 format is valid or we can not get to here.
> + */
> + break;
> + }
> +
> + return ret;
> +}
> +
> +/*
> + * Create a shadow copy of the crycb block.
> + * - Setup key wrapping, if requested for guest 3 and enabled for guest 2.
> + * - Shadow APCB if requested by guest 3 and enabled for guest 2 through
> + * ECA_APIE.
> + *
> + * We only accept format-1 (no AP in g2), but convert it into format-2
> + * There is nothing to do for format-0.
> + *
> + * Returns: - 0 if shadowed or nothing to do
> + * - > 0 if control has to be given to guest 2
> + * - < 0 if something went wrong on copy
> + */
> +static int shadow_crycb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
> +{
> + struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
> + struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
> + const uint32_t crycbd_o = READ_ONCE(scb_o->crycbd);
> + const u32 crycb_addr = crycbd_o & 0x7ffffff8U;
> + int g2_fmt = vcpu->arch.sie_block->crycbd & CRYCB_FORMAT_MASK;
> + int g3_fmt = crycbd_o & CRYCB_FORMAT_MASK;

you can make these two const, too.

> + int g2_apie, g2_msa3, g3_apie, g3_msa3;

you can initialize these all directly and make them const.

> + int size, ret;
> +
> + /* crycb should not cross a page boundary */
> + size = (g3_fmt == CRYCB_FORMAT2) ? 0x100 : 0x80;
> + if ((crycb_addr & PAGE_MASK) != ((crycb_addr + size) & PAGE_MASK))
> + return set_validity_icpt(scb_s, 0x003CU);

You are changing the order of checks compared to existing code. E.g. if
we will not be using the crycbo (as !g3_apie and !g3_msa3), you are
still injecting a validity - existing code won't do that.

Also, you would allow to accept a CRYCB_FORMAT2 here (and perform a
check) although the emulated machine might not even be aware of that
(e.g. !g2_apie and ! g2_msa3)

I don't like such subtle changes. If existing code is wrong, please fix
it first (and explain why it is the right thing to do) or stick to
existing handling as close as possible. Not having access to the
documentation makes me rely on the existing code (and assume it is correct).

> +
> + g2_apie = vcpu->arch.sie_block->eca & ECA_APIE;
> + g3_apie = scb_o->eca & g2_apie;
> +
> + g2_msa3 = test_kvm_facility(vcpu->kvm, 76);
> + g3_msa3 = (g3_fmt != CRYCB_FORMAT0) & g2_msa3;

We should also consider (ECB3_AES | ECB3_DEA) here. because if both are
not set, there is logically no g3_msa3. (existing code does that)

> +
> + scb_s->crycbd = 0;
> + /* If no AP instructions and no keys we just set crycbd to 0 */
> + if (!(g3_apie || g3_msa3))
> + return 0;
> +
> + if (!crycb_addr)
> + return set_validity_icpt(scb_s, 0x0039U);
> +
> + if (g3_apie) {
> + ret = copy_apcb(vcpu, vsie_page, g2_fmt, g3_fmt);
> + if (ret)
> + return ret;
> + scb_s->eca |= g3_apie;
> + }
> +
> + if (g3_msa3)
> + ret = copy_key_masks(vcpu, vsie_page);
> +
> + if (!ret)
> + scb_s->crycbd = ((__u32)(__u64) &vsie_page->crycb) | g2_fmt;
> +
> + return ret;
> +}
> +
> /* shadow (round up/down) the ibc to avoid validity icpt */
> static void prepare_ibc(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
> {
>

--

Thanks,

David / dhildenb

2018-08-21 10:07:55

by Harald Freudenberger

[permalink] [raw]

Subject: Re: [PATCH v9 22/22] s390: doc: detailed specifications for AP virtualization

On 20.08.2018 18:03, Cornelia Huck wrote:
> On Mon, 13 Aug 2018 17:48:19 -0400
> Tony Krowiak <[email protected]> wrote:
>
>> From: Tony Krowiak <[email protected]>
>>
>> This patch provides documentation describing the AP architecture and
>> design concepts behind the virtualization of AP devices. It also
>> includes an example of how to configure AP devices for exclusive
>> use of KVM guests.
>>
>> Signed-off-by: Tony Krowiak <[email protected]>
>> Reviewed-by: Halil Pasic <[email protected]>
>> Signed-off-by: Christian Borntraeger <[email protected]>
>> ---
>> Documentation/s390/vfio-ap.txt | 615 ++++++++++++++++++++++++++++++++++++++++
>> MAINTAINERS | 1 +
>> 2 files changed, 616 insertions(+), 0 deletions(-)
>> create mode 100644 Documentation/s390/vfio-ap.txt
>>
>> +AP Architectural Overview:
>> +=========================
>> +To facilitate the comprehension of the design, let's start with some
>> +definitions:
>> +
>> +* AP adapter
>> +
>> + An AP adapter is an IBM Z adapter card that can perform cryptographic
>> + functions. There can be from 0 to 256 adapters assigned to an LPAR. Adapters
>> + assigned to the LPAR in which a linux host is running will be available to
>> + the linux host. Each adapter is identified by a number from 0 to 255. When
>> + installed, an AP adapter is accessed by AP instructions executed by any CPU.
>> +
>> + The AP adapter cards are assigned to a given LPAR via the system's Activation
>> + Profile which can be edited via the HMC. When the system is IPL'd, the AP bus
> There's lots of s390 jargon in here... but one hopes that someone
> trying to understand AP is already familiar with the basics...
>
>> + module is loaded and detects the AP adapter cards assigned to the LPAR. The AP
>> + bus creates a sysfs device for each adapter as they are detected. For example,
>> + if AP adapters 4 and 10 (0x0a) are assigned to the LPAR, the AP bus will
>> + create the following sysfs entries:
>> +
>> + /sys/devices/ap/card04
>> + /sys/devices/ap/card0a
>> +
>> + Symbolic links to these devices will also be created in the AP bus devices
>> + sub-directory:
>> +
>> + /sys/bus/ap/devices/[card04]
>> + /sys/bus/ap/devices/[card04]
>> +
>> +* AP domain
>> +
>> + An adapter is partitioned into domains. Each domain can be thought of as
>> + a set of hardware registers for processing AP instructions. An adapter can
>> + hold up to 256 domains. Each domain is identified by a number from 0 to 255.
>> + Domains can be further classified into two types:
>> +
>> + * Usage domains are domains that can be accessed directly to process AP
>> + commands.
>> +
>> + * Control domains are domains that are accessed indirectly by AP
>> + commands sent to a usage domain to control or change the domain; for
>> + example, to set a secure private key for the domain.
>> +
>> + The AP usage and control domains are assigned to a given LPAR via the system's
>> + Activation Profile which can be edited via the HMC. When the system is IPL'd,
>> + the AP bus module is loaded and detects the AP usage and control domains
>> + assigned to the LPAR. The domain number of each usage domain will be coupled
>> + with the adapter number of each AP adapter assigned to the LPAR to identify
>> + the AP queues (see AP Queue section below). The domain number of each control
>> + domain will be represented in a bitmask and stored in a sysfs file
>> + /sys/bus/ap/ap_control_domain_mask created by the bus. The bits in the mask,
>> + from most to least significant bit, correspond to domains 0-255.
>> +
>> + A domain may be assigned to a system as both a usage and control domain, or
>> + as a control domain only. Consequently, all domains assigned as both a usage
>> + and control domain can both process AP commands as well as be changed by an AP
>> + command sent to any usage domain assigned to the same system. Domains assigned
>> + only as control domains can not process AP commands but can be changed by AP
>> + commands sent to any usage domain assigned to the system.
> I'm struggling a bit with this paragraph. Does that mean that you can
> use control domains as the target of an instruction changing
> configuration on the system? (Or on the VM, if they are listed in the
> relevant control block?)
Yes. You can send an CPRB to a (usage) domain which includes
a command for controlling another (control) domain.
>
>> +
>> +* AP Queue
>> +
>> + An AP queue is the means by which an AP command-request message is sent to a
>> + usage domain inside a specific adapter. An AP queue is identified by a tuple
>> + comprised of an AP adapter ID (APID) and an AP queue index (APQI). The
>> + APQI corresponds to a given usage domain number within the adapter. This tuple
>> + forms an AP Queue Number (APQN) uniquely identifying an AP queue. AP
>> + instructions include a field containing the APQN to identify the AP queue to
>> + which the AP command-request message is to be sent for processing.
>> +
>> + The AP bus will create a sysfs device for each APQN that can be derived from
>> + the cross product of the AP adapter and usage domain numbers detected when the
>> + AP bus module is loaded. For example, if adapters 4 and 10 (0x0a) and usage
>> + domains 6 and 71 (0x47) are assigned to the LPAR, the AP bus will create the
>> + following sysfs entries:
>> +
>> + /sys/devices/ap/card04/04.0006
>> + /sys/devices/ap/card04/04.0047
>> + /sys/devices/ap/card0a/0a.0006
>> + /sys/devices/ap/card0a/0a.0047
>> +
>> + The following symbolic links to these devices will be created in the AP bus
>> + devices subdirectory:
>> +
>> + /sys/bus/ap/devices/[04.0006]
>> + /sys/bus/ap/devices/[04.0047]
>> + /sys/bus/ap/devices/[0a.0006]
>> + /sys/bus/ap/devices/[0a.0047]
>> +
>> +* AP Instructions:
>> +
>> + There are three AP instructions:
>> +
>> + * NQAP: to enqueue an AP command-request message to a queue
>> + * DQAP: to dequeue an AP command-reply message from a queue
>> + * PQAP: to administer the queues
> So, NQAP/DQAP need usage domains, while PQAP needs a control domain? Or
> is it that all of them need usage domains, but PQAP can target a control
> domain as well?
>
> [I don't want to dive deeply into the AP architecture here, just far
> enough to really understand the design implications.]
Well, to be honest, nobody ever tried this under Linux. Theoretically
one should be able to send a CPRB to a usage domain where inside
the CPRB another domain (the control domain) is addressed. However,
as of now I am only aware of applications controlling the same usage
domain. I don't know any application which is able to address another
control domain and I am not sure if the zcrypt device driver would
handle such a CPRB correctly. NQAP, DQAP and PQAP always address
a usage domain. But the CPRB send down the pipe via NQAP may
address some control thing on another domain. I am not sure which
code and where do the sorting out here. There are two candidates:
the firmware layer in the CEC and the crypto card code.
>
>> +
>> +AP and SIE:
>> +==========
>> +Let's now take a look at how AP instructions executed on a guest are interpreted
>> +by the hardware.
>> +
>> +A satellite control block called the Crypto Control Block (CRYCB) is attached to
>> +our main hardware virtualization control block. The CRYCB contains three fields
>> +to identify the adapters, usage domains and control domains assigned to the KVM
>> +guest:
>> +
>> +* The AP Mask (APM) field is a bit mask that identifies the AP adapters assigned
>> + to the KVM guest. Each bit in the mask, from most significant to least
>> + significant bit, corresponds to an APID from 0-255. If a bit is set, the
>> + corresponding adapter is valid for use by the KVM guest.
>> +
>> +* The AP Queue Mask (AQM) field is a bit mask identifying the AP usage domains
>> + assigned to the KVM guest. Each bit in the mask, from most significant to
>> + least significant bit, corresponds to an AP queue index (APQI) from 0-255. If
>> + a bit is set, the corresponding queue is valid for use by the KVM guest.
>> +
>> +* The AP Domain Mask field is a bit mask that identifies the AP control domains
>> + assigned to the KVM guest. The ADM bit mask controls which domains can be
>> + changed by an AP command-request message sent to a usage domain from the
>> + guest. Each bit in the mask, from least significant to most significant bit,
>> + corresponds to a domain from 0-255. If a bit is set, the corresponding domain
>> + can be modified by an AP command-request message sent to a usage domain
>> + configured for the KVM guest.
> OK, that seems to imply that you modify a control domain by sending a
> request to (any) usage domain? I do not doubt that, but the whole
> architecture is really confusing :)
>
>> +
>> +If you recall from the description of an AP Queue, AP instructions include
>> +an APQN to identify the AP adapter and AP queue to which an AP command-request
>> +message is to be sent (NQAP and PQAP instructions), or from which a
>> +command-reply message is to be received (DQAP instruction). The validity of an
>> +APQN is defined by the matrix calculated from the APM and AQM; it is the
>> +cross product of all assigned adapter numbers (APM) with all assigned queue
>> +indexes (AQM). For example, if adapters 1 and 2 and usage domains 5 and 6 are
>> +assigned to a guest, the APQNs (1,5), (1,6), (2,5) and (2,6) will be valid for
>> +the guest.
> How does the control domain mask interact with that? Can you send a
> command to an APQN valid for the guest to modify any control domain
> specified in the mask? Does the SIE complain if you specify a control
> domain that the host does not have access to (I'd guess so)?
>
>> +
>> +The APQNs can provide secure key functionality - i.e., a private key is stored
>> +on the adapter card for each of its domains - so each APQN must be assigned to
>> +at most one guest or to the linux host.
>> +
>> + Example 1: Valid configuration:
>> + ------------------------------
>> + Guest1: adapters 1,2 domains 5,6
>> + Guest2: adapter 1,2 domain 7
>> +
>> + This is valid because both guests have a unique set of APQNs: Guest1 has
>> + APQNs (1,5), (1,6), (2,5) and (2,6); Guest2 has APQNs (1,7) and (2,7).
>> +
>> + Example 2: Invalid configuration:
>> + Guest1: adapters 1,2 domains 5,6
>> + Guest2: adapter 1 domains 6,7
>> +
>> + This is an invalid configuration because both guests have access to
>> + APQN (1,6).
> So, the adapters or the domains can overlap , but the cross product
> mustn't? If I had
>
> Guest1: adapters 1,2 domains 5,6
> Guest2: adapters 3,4 domains 5,6
>
> would that be fine?
>
> Is there any rule about shared control domains?
>
> (...)
>
>> +Limitations
>> +===========
>> +* The KVM/kernel interfaces do not provide a way to prevent unbinding an AP
>> + queue that is still assigned to a mediated device. Even if the device
>> + 'remove' callback returns an error, the device core detaches the AP
>> + queue from the VFIO AP driver. It is therefore incumbent upon the
>> + administrator to make sure there is no mediated device to which the
>> + APQN - for the AP queue being unbound - is assigned.
>> +
>> +* Hot plug/unplug of AP devices is not supported for guests.
> Not sure what that sentence means. Adding/removing devices by the
> hypervisor is not supported? Or some guest actions, respectively
> injecting notifications that would trigger some actions on the real
> hardware?
>
> Do you want to add (some of) this in the future?
>
>> +
>> +* Live guest migration is not supported for guests using AP devices.
> Migration and vfio is an interesting area in general :) Would be great
> if vfio-ap could benefit from any generic efforts in that area, but
> that probably requires that someone with access to documentation and
> hardware keeps an eye on developments.
>
>> \ No newline at end of file
> Please add one :)
>

2018-08-21 13:37:42

[permalink] [raw]

Subject: Re: [PATCH v9 07/22] KVM: s390: refactor crypto initialization

On 08/20/2018 04:41 PM, David Hildenbrand wrote:
>>>> + if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_AP))
>>>> + vcpu->arch.sie_block->eca |= ECA_APIE;
>>>>
>>>> - vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
>>>> + /* If MSAX3 is installed on the guest, set up protected key support */
>>>> + if (test_kvm_facility(vcpu->kvm, 76)) {
>>>> + if (vcpu->kvm->arch.crypto.aes_kw)
>>>> + vcpu->arch.sie_block->ecb3 |= ECB3_AES;
>>>> + if (vcpu->kvm->arch.crypto.dea_kw)
>>>> + vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
>>>> + }
>>> As the feature can never change, and aes_kw/dea_kw are only set to 1 in
>>> case we have test_kvm_facility(vcpu->kvm, 76), this change is not needed.
>>>
>>> I think this function can be pretty much left alone. Just add the
>>> KVM_S390_VM_CPU_FEAT_AP handling.
>> I disagree, what about the case where the KVM_S390_VM_CPU_FEAT_AP is
>> configured for the guest but the MSAX3 facility (76) is not?
> Then aes_kw/dea_kw can never be set.
>
> kvm_s390_vm_set_crypto() and kvm_s390_crypto_init() correctly test for
> facility 76.
>
> Or am I missing a case?

I stand corrected, you are right. I'll remove the test.

>
>>>> }
>>>>
>>>> void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
>>>>
>

2018-08-21 16:35:08

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On Mon, 20 Aug 2018 13:41:32 -0400
Tony Krowiak <[email protected]> wrote:

> On 08/20/2018 10:23 AM, Cornelia Huck wrote:
> > On Mon, 13 Aug 2018 17:48:09 -0400
> > Tony Krowiak <[email protected]> wrote:
> >
> >> From: Tony Krowiak <[email protected]>
> >>
> >> Provides the sysfs interfaces for:
> >>
> >> 1. Assigning AP control domains to the mediated matrix device
> >>
> >> 2. Unassigning AP control domains from a mediated matrix device
> >>
> >> 3. Displaying the control domains assigned to a mediated matrix
> >> device
> >>
> >> The IDs of the AP control domains assigned to the mediated matrix
> >> device are stored in an AP domain mask (ADM). The bits in the ADM,
> >> from most significant to least significant bit, correspond to
> >> AP domain numbers 0 to 255. On some systems, the maximum allowable
> >> domain number may be less than 255 - depending upon the host's
> >> AP configuration - and assignment may be rejected if the input
> >> domain ID exceeds the limit.
> > Please remind me of the relationship between control domains and usage
> > domains... IIRC, usage domains allow both requests and configuration,
> > while control domains allow only configuration, and are by convention a
> > superset of usage domains.
>
> A usage domain is a domain to which an AP command-request message can be
> submitted for processing. A control domain is a domain that can
> be changed by an AP command request message submitted to a usage domain.
> AP command request messages to configure a domain will contain the domain
> number of the domain to be modified. The AP firmware will check the
> control domain mask (ADM) and will allow the request to proceed only if
> the corresponding bit in the ADM is set.

Thanks to you and Halil for the explanation.

>
> >
> > Is there a hard requirement somewhere in there, or can the admin
> > cheerfully use different masks for usage domains and control domains
> > without the SIE choking on it?
>
> There is no hard requirement that control domains must be a superset of
> the usage domains, it is merely an architectural convention. AFAIK,
> SIE doesn't enforce this and will not break if the convention is not
> enforced externally. Having said that, you should note that the AQM
> and ADM masks configured for the mediated matrix device will be logically
> OR'd together to create the ADM stored in the CRYCB referenced from the
> guest's SIE state description. In other words, we are enforcing the
> convention in our software.

Hm, that's interesting, as Halil argued that we should not enforce it
in the kernel. Might be somewhat surprising as well. If that is really
the way to do it, this needs to be documented clearly.

2018-08-21 16:46:05

[permalink] [raw]

Subject: Re: [PATCH v9 22/22] s390: doc: detailed specifications for AP virtualization

On Mon, 20 Aug 2018 16:16:15 -0400
Tony Krowiak <[email protected]> wrote:

> On 08/20/2018 12:03 PM, Cornelia Huck wrote:
> > On Mon, 13 Aug 2018 17:48:19 -0400
> > Tony Krowiak <[email protected]> wrote:

> >> +AP Architectural Overview:
> >> +=========================
> >> +To facilitate the comprehension of the design, let's start with some
> >> +definitions:
> >> +
> >> +* AP adapter
> >> +
> >> + An AP adapter is an IBM Z adapter card that can perform cryptographic
> >> + functions. There can be from 0 to 256 adapters assigned to an LPAR. Adapters
> >> + assigned to the LPAR in which a linux host is running will be available to
> >> + the linux host. Each adapter is identified by a number from 0 to 255. When
> >> + installed, an AP adapter is accessed by AP instructions executed by any CPU.
> >> +
> >> + The AP adapter cards are assigned to a given LPAR via the system's Activation
> >> + Profile which can be edited via the HMC. When the system is IPL'd, the AP bus
> > There's lots of s390 jargon in here... but one hopes that someone
> > trying to understand AP is already familiar with the basics...
>
> I'm not quite sure how one can describe s390-specific devices that can
> be installed
> only on an s390 system without using s390 jargon. I would think that one
> who is
> administering a linux host or guest running on an s390 system would have
> some
> basic knowledge of s390. If you have any suggestions, I'd be happy to
> entertain them.

I fear the jargon is mostly unavoidable :(

> >> +* AP Instructions:
> >> +
> >> + There are three AP instructions:
> >> +
> >> + * NQAP: to enqueue an AP command-request message to a queue
> >> + * DQAP: to dequeue an AP command-reply message from a queue
> >> + * PQAP: to administer the queues
> > So, NQAP/DQAP need usage domains, while PQAP needs a control domain? Or
> > is it that all of them need usage domains, but PQAP can target a control
> > domain as well?
>
> All AP instructions - the lone exception being the PQAP(QCI) subfunction -
> identify the usage domain that is the target of the instruction. I think
> using the term 'control domain' is the source of much confusion. It makes
> it seem as if there are two types of domains that serve different purposes.
> That is simply not true. A domain is a partition within an AP adapter that
> can process AP command request messages. All AP commands are sent to a
> domain. Configuring a domain as a usage domain means it can be used to
> process AP commands; in other words, it can be the target of an AP
> instruction. Configuring a domain as a control domain means it can be
> changed by an AP command. AP commands that change a domain are sent to
> a usage domain, but the domain to be changed is specified in the payload
> of the AP command message. The domain thus specified must be
> identified via the AP configuration as a control domain, or the AP command
> will be rejected.

Yes, the 'control domain' term is a source of much confusion :(

>
> >
> > [I don't want to dive deeply into the AP architecture here, just far
> > enough to really understand the design implications.]
>
> Are you suggesting some of the above should be removed? If so, what?

Not removed. What about an explanation like the following somewhere:

"AP instructions identify the domain that is targeted to process the
command: This must be one of the usage domains. They may modify a
domain that is not one of the usage domains, but the modified domain
must be one of the control domains."

I hope that is both correct and understandable ;)

> > Does the SIE complain if you specify a control
> > domain that the host does not have access to (I'd guess so)?
>
> The SIE does not complain if you specify a domain to which the host - or a
> lower level guest - does not have access. The firmware performs a logical
> AND of the guest's and hosts's - or lower level guest's - APMs, AQMs and
> ADMs
> to create effective masks EAPM, EAQM and EADM. Only devices corresponding to
> the bits set in the EAPM, EAQM and EADM will be accessible by the guest.

OK, so the guest effectively won't see the domain. That makes sense.

>
> >
> >> +
> >> +The APQNs can provide secure key functionality - i.e., a private key is stored
> >> +on the adapter card for each of its domains - so each APQN must be assigned to
> >> +at most one guest or to the linux host.
> >> +
> >> + Example 1: Valid configuration:
> >> + ------------------------------
> >> + Guest1: adapters 1,2 domains 5,6
> >> + Guest2: adapter 1,2 domain 7
> >> +
> >> + This is valid because both guests have a unique set of APQNs: Guest1 has
> >> + APQNs (1,5), (1,6), (2,5) and (2,6); Guest2 has APQNs (1,7) and (2,7).
> >> +
> >> + Example 2: Invalid configuration:
> >> + Guest1: adapters 1,2 domains 5,6
> >> + Guest2: adapter 1 domains 6,7
> >> +
> >> + This is an invalid configuration because both guests have access to
> >> + APQN (1,6).
> > So, the adapters or the domains can overlap , but the cross product
> > mustn't? If I had
> >
> > Guest1: adapters 1,2 domains 5,6
> > Guest2: adapters 3,4 domains 5,6
> >
> > would that be fine?
>
> Yes, that would be fine because Guest1 would have access to APQNs
> (1,5), (1,6), (2,5) and (2,6) while Guest2 would have access to
> (3,5), (3,6), (4,5) AND (4,6), but neither would have access to
> the same APQN.

Might be a good idea to add this as an additional example.

>
> >
> > Is there any rule about shared control domains?
>
> AFAIK there isn't, but I will consult with Reinhard about that.
>
> >
> > (...)
> >
> >> +Limitations
> >> +===========
> >> +* The KVM/kernel interfaces do not provide a way to prevent unbinding an AP
> >> + queue that is still assigned to a mediated device. Even if the device
> >> + 'remove' callback returns an error, the device core detaches the AP
> >> + queue from the VFIO AP driver. It is therefore incumbent upon the
> >> + administrator to make sure there is no mediated device to which the
> >> + APQN - for the AP queue being unbound - is assigned.
> >> +
> >> +* Hot plug/unplug of AP devices is not supported for guests.
> > Not sure what that sentence means. Adding/removing devices by the
> > hypervisor is not supported? Or some guest actions, respectively
> > injecting notifications that would trigger some actions on the real
> > hardware?
>
> No means is provided to modify a guest's AP matrix - i.e., APM, AQM
> and ADM - while a guest is running. Once a guest is running, its AP
> configuration can not be changed dynamically.
>
> >
> > Do you want to add (some of) this in the future?
>
> Yes, we plan to introduce dynamic configurations in future releases.

What about the following sentence:

"Dynamically modifying the AP matrix for a running guest (which would
amount to hot(un)plug of AP devices for the guest) is currently not
supported."

>
> >
> >> +
> >> +* Live guest migration is not supported for guests using AP devices.
> > Migration and vfio is an interesting area in general :) Would be great
> > if vfio-ap could benefit from any generic efforts in that area, but
> > that probably requires that someone with access to documentation and
> > hardware keeps an eye on developments.
>
> I have briefly looked at some of the articles talking about live migration
> of passthrough devices, but nothing seemed applicable to AP architecture.

Most of the approaches to live migration of vfio devices are focused on
pci devices; even ccw devices have different needs. Any halfway generic
approach would need a common part and a backend-specific part anyway, I
think.

> From my limited perspective, it would seem that architectural changes
> would have to be implemented to fully support live migration of in-process
> AP queues.

From what I have seen of the AP virtualization architecture, this may
very well be the case. I'll keep AP in the back of my head, but it's
probably better to focus on the easier targets first.

2018-08-21 16:53:43

[permalink] [raw]

Subject: Re: [PATCH v9 22/22] s390: doc: detailed specifications for AP virtualization

On Tue, 21 Aug 2018 11:00:00 +0200
Harald Freudenberger <[email protected]> wrote:

> On 20.08.2018 18:03, Cornelia Huck wrote:
> > On Mon, 13 Aug 2018 17:48:19 -0400
> > Tony Krowiak <[email protected]> wrote:

> >> +* AP Instructions:
> >> +
> >> + There are three AP instructions:
> >> +
> >> + * NQAP: to enqueue an AP command-request message to a queue
> >> + * DQAP: to dequeue an AP command-reply message from a queue
> >> + * PQAP: to administer the queues
> > So, NQAP/DQAP need usage domains, while PQAP needs a control domain? Or
> > is it that all of them need usage domains, but PQAP can target a control
> > domain as well?
> >
> > [I don't want to dive deeply into the AP architecture here, just far
> > enough to really understand the design implications.]
> Well, to be honest, nobody ever tried this under Linux. Theoretically
> one should be able to send a CPRB to a usage domain where inside
> the CPRB another domain (the control domain) is addressed. However,
> as of now I am only aware of applications controlling the same usage
> domain. I don't know any application which is able to address another
> control domain and I am not sure if the zcrypt device driver would
> handle such a CPRB correctly. NQAP, DQAP and PQAP always address
> a usage domain. But the CPRB send down the pipe via NQAP may
> address some control thing on another domain. I am not sure which
> code and where do the sorting out here. There are two candidates:
> the firmware layer in the CEC and the crypto card code.

OK, so it's possible as by the architecture, but at least Linux does
not (currently) do it?

Perhaps we should simply not overthink that whole control domain
thingy :) It's mostly yet another knob, and as long as the design does
not go against the general architecture, it's probably fine, I guess.

2018-08-21 17:10:26

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On 08/21/2018 11:25 AM, Cornelia Huck wrote:
> On Mon, 20 Aug 2018 13:41:32 -0400
> Tony Krowiak <[email protected]> wrote:
>
>> On 08/20/2018 10:23 AM, Cornelia Huck wrote:
>>> On Mon, 13 Aug 2018 17:48:09 -0400
>>> Tony Krowiak <[email protected]> wrote:
>>>
>>>> From: Tony Krowiak <[email protected]>
>>>>
>>>> Provides the sysfs interfaces for:
>>>>
>>>> 1. Assigning AP control domains to the mediated matrix device
>>>>
>>>> 2. Unassigning AP control domains from a mediated matrix device
>>>>
>>>> 3. Displaying the control domains assigned to a mediated matrix
>>>> device
>>>>
>>>> The IDs of the AP control domains assigned to the mediated matrix
>>>> device are stored in an AP domain mask (ADM). The bits in the ADM,
>>>> from most significant to least significant bit, correspond to
>>>> AP domain numbers 0 to 255. On some systems, the maximum allowable
>>>> domain number may be less than 255 - depending upon the host's
>>>> AP configuration - and assignment may be rejected if the input
>>>> domain ID exceeds the limit.
>>> Please remind me of the relationship between control domains and usage
>>> domains... IIRC, usage domains allow both requests and configuration,
>>> while control domains allow only configuration, and are by convention a
>>> superset of usage domains.
>> A usage domain is a domain to which an AP command-request message can be
>> submitted for processing. A control domain is a domain that can
>> be changed by an AP command request message submitted to a usage domain.
>> AP command request messages to configure a domain will contain the domain
>> number of the domain to be modified. The AP firmware will check the
>> control domain mask (ADM) and will allow the request to proceed only if
>> the corresponding bit in the ADM is set.
> Thanks to you and Halil for the explanation.
>
>>> Is there a hard requirement somewhere in there, or can the admin
>>> cheerfully use different masks for usage domains and control domains
>>> without the SIE choking on it?
>> There is no hard requirement that control domains must be a superset of
>> the usage domains, it is merely an architectural convention. AFAIK,
>> SIE doesn't enforce this and will not break if the convention is not
>> enforced externally. Having said that, you should note that the AQM
>> and ADM masks configured for the mediated matrix device will be logically
>> OR'd together to create the ADM stored in the CRYCB referenced from the
>> guest's SIE state description. In other words, we are enforcing the
>> convention in our software.
> Hm, that's interesting, as Halil argued that we should not enforce it
> in the kernel. Might be somewhat surprising as well. If that is really
> the way to do it, this needs to be documented clearly.

This convention has been enforced by the kernel since v1. This is also
enforced by both the LPAR as well as in z/VM. The following is from the
PR/SM Planning Guide:

Control Domain
A logical partition's control domains are those cryptographic domains
for which remote secure
administration functions can be established and administered from this
logical partition. This
logical partition’s control domains must include its usage domains. For
each index selected in the
usage domain index list, you must select the same index in the control
domain index list

Consequently, I'm going to opt for ensuring this is clearly documented.
Based on the fact you've
requested clarification of many points described in this section of the
doc, I
think I'll try putting my meager skills as a wordsmith to work to
hopefully clarify things.
I'll run it by you when I complete that task to see if I've succeeded:)

>

2018-08-21 19:28:56

[permalink] [raw]

Subject: Re: [PATCH v9 22/22] s390: doc: detailed specifications for AP virtualization

On 08/20/2018 10:16 PM, Tony Krowiak wrote:
>> Does the SIE complain if you specify a control
>> domain that the host does not have access to (I'd guess so)?
>
> The SIE does not complain if you specify a domain to which the host - or a
> lower level guest - does not have access. The firmware performs a logical
> AND of the guest's and hosts's - or lower level guest's - APMs, AQMs and ADMs

Rather a bit-wise AND, I guess (of the same type masks corresponding to Guest 1 and
Guest 2). The result of a logical AND is a logical value (true or false) as
far as I remember.

> to create effective masks EAPM, EAQM and EADM. Only devices corresponding to
> the bits set in the EAPM, EAQM and EADM will be accessible by the guest.

I'm not sure what is the intended meaning of 'the SIE complains'. If it means
getting out of (SIE when interpreting lets say an NQAP under the discussed
circumstances) with some sort of error code, I think Tony's answer, ' SIE does not complain'
makes a lot of sense. It's the guest that's is trying to stretch further than
the blanket reaches, and it's the guest that needs to be educated on this fact.

AFAIR SIE does the right thing (whatever the right thing is) and we don't have to
worry about it.

As a matter of fact I can't recall exactly what is supposed to happen
when a guest tries to modify a domain such that the guest does not
have privileges to modify (in terms of EADM, either because the
guest or because the host does not have the corresponding bit set). I'm sure
I did not try it out. Tony did you test this scenario? (BTW my best guess
at the moment is, that the situation is handled via the command-reply.)

Regards,
Halil

2018-08-21 19:53:56

[permalink] [raw]

Subject: Re: [PATCH v9 22/22] s390: doc: detailed specifications for AP virtualization

On 08/21/2018 12:13 PM, Cornelia Huck wrote:
> On Mon, 20 Aug 2018 16:16:15 -0400
> Tony Krowiak <[email protected]> wrote:
>
>> On 08/20/2018 12:03 PM, Cornelia Huck wrote:
>>> On Mon, 13 Aug 2018 17:48:19 -0400
>>> Tony Krowiak <[email protected]> wrote:
>>>> +AP Architectural Overview:
>>>> +=========================
>>>> +To facilitate the comprehension of the design, let's start with some
>>>> +definitions:
>>>> +
>>>> +* AP adapter
>>>> +
>>>> + An AP adapter is an IBM Z adapter card that can perform cryptographic
>>>> + functions. There can be from 0 to 256 adapters assigned to an LPAR. Adapters
>>>> + assigned to the LPAR in which a linux host is running will be available to
>>>> + the linux host. Each adapter is identified by a number from 0 to 255. When
>>>> + installed, an AP adapter is accessed by AP instructions executed by any CPU.
>>>> +
>>>> + The AP adapter cards are assigned to a given LPAR via the system's Activation
>>>> + Profile which can be edited via the HMC. When the system is IPL'd, the AP bus
>>> There's lots of s390 jargon in here... but one hopes that someone
>>> trying to understand AP is already familiar with the basics...
>> I'm not quite sure how one can describe s390-specific devices that can
>> be installed
>> only on an s390 system without using s390 jargon. I would think that one
>> who is
>> administering a linux host or guest running on an s390 system would have
>> some
>> basic knowledge of s390. If you have any suggestions, I'd be happy to
>> entertain them.
> I fear the jargon is mostly unavoidable :(
>
>>>> +* AP Instructions:
>>>> +
>>>> + There are three AP instructions:
>>>> +
>>>> + * NQAP: to enqueue an AP command-request message to a queue
>>>> + * DQAP: to dequeue an AP command-reply message from a queue
>>>> + * PQAP: to administer the queues
>>> So, NQAP/DQAP need usage domains, while PQAP needs a control domain? Or
>>> is it that all of them need usage domains, but PQAP can target a control
>>> domain as well?
>> All AP instructions - the lone exception being the PQAP(QCI) subfunction -
>> identify the usage domain that is the target of the instruction. I think
>> using the term 'control domain' is the source of much confusion. It makes
>> it seem as if there are two types of domains that serve different purposes.
>> That is simply not true. A domain is a partition within an AP adapter that
>> can process AP command request messages. All AP commands are sent to a
>> domain. Configuring a domain as a usage domain means it can be used to
>> process AP commands; in other words, it can be the target of an AP
>> instruction. Configuring a domain as a control domain means it can be
>> changed by an AP command. AP commands that change a domain are sent to
>> a usage domain, but the domain to be changed is specified in the payload
>> of the AP command message. The domain thus specified must be
>> identified via the AP configuration as a control domain, or the AP command
>> will be rejected.
> Yes, the 'control domain' term is a source of much confusion :(
>
>>> [I don't want to dive deeply into the AP architecture here, just far
>>> enough to really understand the design implications.]
>> Are you suggesting some of the above should be removed? If so, what?
> Not removed. What about an explanation like the following somewhere:
>
> "AP instructions identify the domain that is targeted to process the
> command: This must be one of the usage domains. They may modify a
> domain that is not one of the usage domains, but the modified domain
> must be one of the control domains."
>
> I hope that is both correct and understandable ;)

Yes, it is both correct and understandable.

>
>>> Does the SIE complain if you specify a control
>>> domain that the host does not have access to (I'd guess so)?
>> The SIE does not complain if you specify a domain to which the host - or a
>> lower level guest - does not have access. The firmware performs a logical
>> AND of the guest's and hosts's - or lower level guest's - APMs, AQMs and
>> ADMs
>> to create effective masks EAPM, EAQM and EADM. Only devices corresponding to
>> the bits set in the EAPM, EAQM and EADM will be accessible by the guest.
> OK, so the guest effectively won't see the domain. That makes sense.

It is one of the positive aspects of the architecture.

>
>>>
>>>> +
>>>> +The APQNs can provide secure key functionality - i.e., a private key is stored
>>>> +on the adapter card for each of its domains - so each APQN must be assigned to
>>>> +at most one guest or to the linux host.
>>>> +
>>>> + Example 1: Valid configuration:
>>>> + ------------------------------
>>>> + Guest1: adapters 1,2 domains 5,6
>>>> + Guest2: adapter 1,2 domain 7
>>>> +
>>>> + This is valid because both guests have a unique set of APQNs: Guest1 has
>>>> + APQNs (1,5), (1,6), (2,5) and (2,6); Guest2 has APQNs (1,7) and (2,7).
>>>> +
>>>> + Example 2: Invalid configuration:
>>>> + Guest1: adapters 1,2 domains 5,6
>>>> + Guest2: adapter 1 domains 6,7
>>>> +
>>>> + This is an invalid configuration because both guests have access to
>>>> + APQN (1,6).
>>> So, the adapters or the domains can overlap , but the cross product
>>> mustn't? If I had
>>>
>>> Guest1: adapters 1,2 domains 5,6
>>> Guest2: adapters 3,4 domains 5,6
>>>
>>> would that be fine?
>> Yes, that would be fine because Guest1 would have access to APQNs
>> (1,5), (1,6), (2,5) and (2,6) while Guest2 would have access to
>> (3,5), (3,6), (4,5) AND (4,6), but neither would have access to
>> the same APQN.
> Might be a good idea to add this as an additional example.

Will do

>
>>> Is there any rule about shared control domains?
>> AFAIK there isn't, but I will consult with Reinhard about that.
>>
>>> (...)
>>>
>>>> +Limitations
>>>> +===========
>>>> +* The KVM/kernel interfaces do not provide a way to prevent unbinding an AP
>>>> + queue that is still assigned to a mediated device. Even if the device
>>>> + 'remove' callback returns an error, the device core detaches the AP
>>>> + queue from the VFIO AP driver. It is therefore incumbent upon the
>>>> + administrator to make sure there is no mediated device to which the
>>>> + APQN - for the AP queue being unbound - is assigned.
>>>> +
>>>> +* Hot plug/unplug of AP devices is not supported for guests.
>>> Not sure what that sentence means. Adding/removing devices by the
>>> hypervisor is not supported? Or some guest actions, respectively
>>> injecting notifications that would trigger some actions on the real
>>> hardware?
>> No means is provided to modify a guest's AP matrix - i.e., APM, AQM
>> and ADM - while a guest is running. Once a guest is running, its AP
>> configuration can not be changed dynamically.
>>
>>> Do you want to add (some of) this in the future?
>> Yes, we plan to introduce dynamic configurations in future releases.
> What about the following sentence:
>
> "Dynamically modifying the AP matrix for a running guest (which would
> amount to hot(un)plug of AP devices for the guest) is currently not
> supported."

Sounds fine to me.

>
>>>
>>>> +
>>>> +* Live guest migration is not supported for guests using AP devices.
>>> Migration and vfio is an interesting area in general :) Would be great
>>> if vfio-ap could benefit from any generic efforts in that area, but
>>> that probably requires that someone with access to documentation and
>>> hardware keeps an eye on developments.
>> I have briefly looked at some of the articles talking about live migration
>> of passthrough devices, but nothing seemed applicable to AP architecture.
> Most of the approaches to live migration of vfio devices are focused on
> pci devices; even ccw devices have different needs. Any halfway generic
> approach would need a common part and a backend-specific part anyway, I
> think.

Yes, that would seem to be the case.

>
>> From my limited perspective, it would seem that architectural changes
>> would have to be implemented to fully support live migration of in-process
>> AP queues.
> From what I have seen of the AP virtualization architecture, this may
> very well be the case. I'll keep AP in the back of my head, but it's
> probably better to focus on the easier targets first.

That has been our goal from the start.

>

2018-08-21 23:20:28

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On 08/21/2018 07:07 PM, Tony Krowiak wrote:
> On 08/21/2018 11:25 AM, Cornelia Huck wrote:
>> On Mon, 20 Aug 2018 13:41:32 -0400
>> Tony Krowiak <[email protected]> wrote:
>>
>>> On 08/20/2018 10:23 AM, Cornelia Huck wrote:
>>>> On Mon, 13 Aug 2018 17:48:09 -0400
>>>> Tony Krowiak <[email protected]> wrote:
>>>>> From: Tony Krowiak <[email protected]>
>>>>>
>>>>> Provides the sysfs interfaces for:
>>>>>
>>>>> 1. Assigning AP control domains to the mediated matrix device
>>>>>
>>>>> 2. Unassigning AP control domains from a mediated matrix device
>>>>>
>>>>> 3. Displaying the control domains assigned to a mediated matrix
>>>>> device
>>>>>
>>>>> The IDs of the AP control domains assigned to the mediated matrix
>>>>> device are stored in an AP domain mask (ADM). The bits in the ADM,
>>>>> from most significant to least significant bit, correspond to
>>>>> AP domain numbers 0 to 255. On some systems, the maximum allowable
>>>>> domain number may be less than 255 - depending upon the host's
>>>>> AP configuration - and assignment may be rejected if the input
>>>>> domain ID exceeds the limit.
>>>> Please remind me of the relationship between control domains and usage
>>>> domains... IIRC, usage domains allow both requests and configuration,
>>>> while control domains allow only configuration, and are by convention a
>>>> superset of usage domains.
>>> A usage domain is a domain to which an AP command-request message can be
>>> submitted for processing. A control domain is a domain that can
>>> be changed by an AP command request message submitted to a usage domain.
>>> AP command request messages to configure a domain will contain the domain
>>> number of the domain to be modified. The AP firmware will check the
>>> control domain mask (ADM) and will allow the request to proceed only if
>>> the corresponding bit in the ADM is set.
>> Thanks to you and Halil for the explanation.
>>
>>>> Is there a hard requirement somewhere in there, or can the admin
>>>> cheerfully use different masks for usage domains and control domains
>>>> without the SIE choking on it?
>>> There is no hard requirement that control domains must be a superset of
>>> the usage domains, it is merely an architectural convention. AFAIK,
>>> SIE doesn't enforce this and will not break if the convention is not
>>> enforced externally. Having said that, you should note that the AQM
>>> and ADM masks configured for the mediated matrix device will be logically
>>> OR'd together to create the ADM stored in the CRYCB referenced from the
>>> guest's SIE state description. In other words, we are enforcing the
>>> convention in our software.
>> Hm, that's interesting, as Halil argued that we should not enforce it
>> in the kernel. Might be somewhat surprising as well. If that is really
>> the way to do it, this needs to be documented clearly.
>
> This convention has been enforced by the kernel since v1. This is also
> enforced by both the LPAR as well as in z/VM. The following is from the
> PR/SM Planning Guide:
>
> Control Domain
> A logical partition's control domains are those cryptographic domains for which remote secure
> administration functions can be established and administered from this logical partition. This
> logical partition’s control domains must include its usage domains. For each index selected in the
> usage domain index list, you must select the same index in the control domain index list
>

IMHO this quote is quite a half-full half-empty cup one:
* it mandates the set of usage domains is a subset of the set
of the control domains, but
* it speaks of independent controls, namely about the 'usage domain index'
and the 'control domain index list' and makes the enforcement of the rule
a job of the administrator (instead of codifying it in the controls).

>
> Consequently, I'm going to opt for ensuring this is clearly documented. Based on the fact you've
> requested clarification of many points described in this section of the doc, I
> think I'll try putting my meager skills as a wordsmith to work to hopefully clarify things.
> I'll run it by you when I complete that task to see if I've succeeded:)

I don't think just a doc update will do. Let me explain why.

What describe as "... note that the AQM and ADM masks configured for the
mediated matrix device will be logically OR'd together to create the ADM
stored in the CRYCB referenced from the guest's SIE state description."
is a gotcha at best. The member of struct ap_matrix and the member of the
respective apcb in the crycb are both called 'adm', but ap_matrix.adm is
not an ADM as we know it from the architecture, but rather ~ AQM & ADM.

I feel pretty strongly about this one. If we want to keep the enforcement
in the kernel, I guess, the assign_domain should set the bit corresponding
bit not only in ap_matrix.aqm but also in ap_matrix.adm. When the
ap_matrix is committed into the crycb no further manipulating the masks
should take place.

I don't feel strongly about whether to enforce this convention about AQM
and ADM in the kernel or not. Frankly, I don't know what is behind the
rule. Since I can't tell if any problems are to be expected if this
convention is violated, I would feel more comfortable if the rule was
accommodated higher in the management stack.

Regards,
Halil

>
>>
>

2018-08-22 07:07:58

by Harald Freudenberger

[permalink] [raw]

Subject: Re: [PATCH v9 22/22] s390: doc: detailed specifications for AP virtualization

On 21.08.2018 17:53, Cornelia Huck wrote:
> On Tue, 21 Aug 2018 11:00:00 +0200
> Harald Freudenberger <[email protected]> wrote:
>
>> On 20.08.2018 18:03, Cornelia Huck wrote:
>>> On Mon, 13 Aug 2018 17:48:19 -0400
>>> Tony Krowiak <[email protected]> wrote:
>>>> +* AP Instructions:
>>>> +
>>>> + There are three AP instructions:
>>>> +
>>>> + * NQAP: to enqueue an AP command-request message to a queue
>>>> + * DQAP: to dequeue an AP command-reply message from a queue
>>>> + * PQAP: to administer the queues
>>> So, NQAP/DQAP need usage domains, while PQAP needs a control domain? Or
>>> is it that all of them need usage domains, but PQAP can target a control
>>> domain as well?
>>>
>>> [I don't want to dive deeply into the AP architecture here, just far
>>> enough to really understand the design implications.]
>> Well, to be honest, nobody ever tried this under Linux. Theoretically
>> one should be able to send a CPRB to a usage domain where inside
>> the CPRB another domain (the control domain) is addressed. However,
>> as of now I am only aware of applications controlling the same usage
>> domain. I don't know any application which is able to address another
>> control domain and I am not sure if the zcrypt device driver would
>> handle such a CPRB correctly. NQAP, DQAP and PQAP always address
>> a usage domain. But the CPRB send down the pipe via NQAP may
>> address some control thing on another domain. I am not sure which
>> code and where do the sorting out here. There are two candidates:
>> the firmware layer in the CEC and the crypto card code.
> OK, so it's possible as by the architecture, but at least Linux does
> not (currently) do it?
>
> Perhaps we should simply not overthink that whole control domain
> thingy :) It's mostly yet another knob, and as long as the design does
> not go against the general architecture, it's probably fine, I guess.
Well, sooner or later this has to work. Yesterday we tested the control
domain thing with trying to pull some simple data from a 'controlled' domain
to the TKE - doesn't work with a Linux LPAR. I will investigate the details in the
next weeks. However, long-term it should be possible to run scenarios
like having one KVM guest control all the domains used by other KVM guests.
With respect to the KVM vfio driver, currently there should be just the
rule that for a guest the control domain mask should be equal or a superset
of the usage domain mask. This is by convention as the architecture is
not so clear here, but this is enforced on every place which deals with
usage and control domains (SE, TKE).

regards Harald Freudenberger

2018-08-22 08:01:05

[permalink] [raw]

Subject: Re: [PATCH v9 22/22] s390: doc: detailed specifications for AP virtualization

On Tue, 21 Aug 2018 20:54:49 +0200
Halil Pasic <[email protected]> wrote:

> On 08/20/2018 10:16 PM, Tony Krowiak wrote:
> >> Does the SIE complain if you specify a control
> >> domain that the host does not have access to (I'd guess so)?
> >
> > The SIE does not complain if you specify a domain to which the host - or a
> > lower level guest - does not have access. The firmware performs a logical
> > AND of the guest's and hosts's - or lower level guest's - APMs, AQMs and ADMs
>
> Rather a bit-wise AND, I guess (of the same type masks corresponding to Guest 1 and
> Guest 2). The result of a logical AND is a logical value (true or false) as
> far as I remember.
>
> > to create effective masks EAPM, EAQM and EADM. Only devices corresponding to
> > the bits set in the EAPM, EAQM and EADM will be accessible by the guest.
>
> I'm not sure what is the intended meaning of 'the SIE complains'. If it means
> getting out of (SIE when interpreting lets say an NQAP under the discussed
> circumstances) with some sort of error code, I think Tony's answer, ' SIE does not complain'
> makes a lot of sense. It's the guest that's is trying to stretch further than
> the blanket reaches, and it's the guest that needs to be educated on this fact.

Yep, that's what I meant. If the hypervisor can call the SIE with that
config, but the guest gets an error if it tries to use something that
it cannot use, that's fine.

2018-08-22 09:44:40

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On Wed, 22 Aug 2018 01:18:20 +0200
Halil Pasic <[email protected]> wrote:

> On 08/21/2018 07:07 PM, Tony Krowiak wrote:
> > This convention has been enforced by the kernel since v1. This is also
> > enforced by both the LPAR as well as in z/VM. The following is from the
> > PR/SM Planning Guide:
> >
> > Control Domain
> > A logical partition's control domains are those cryptographic domains for which remote secure
> > administration functions can be established and administered from this logical partition. This
> > logical partition’s control domains must include its usage domains. For each index selected in the
> > usage domain index list, you must select the same index in the control domain index list
> >

That's interesting.

>
> IMHO this quote is quite a half-full half-empty cup one:
> * it mandates the set of usage domains is a subset of the set
> of the control domains, but
> * it speaks of independent controls, namely about the 'usage domain index'
> and the 'control domain index list' and makes the enforcement of the rule
> a job of the administrator (instead of codifying it in the controls).

I'm wondering if a configuration with a usage domain that is not also a
control domain is rejected outright? Anybody tried that? :)

>
> >
> > Consequently, I'm going to opt for ensuring this is clearly documented. Based on the fact you've
> > requested clarification of many points described in this section of the doc, I
> > think I'll try putting my meager skills as a wordsmith to work to hopefully clarify things.
> > I'll run it by you when I complete that task to see if I've succeeded:)
>
> I don't think just a doc update will do. Let me explain why.
>
> What describe as "... note that the AQM and ADM masks configured for the
> mediated matrix device will be logically OR'd together to create the ADM
> stored in the CRYCB referenced from the guest's SIE state description."
> is a gotcha at best. The member of struct ap_matrix and the member of the
> respective apcb in the crycb are both called 'adm', but ap_matrix.adm is
> not an ADM as we know it from the architecture, but rather ~ AQM & ADM.
>
> I feel pretty strongly about this one. If we want to keep the enforcement
> in the kernel, I guess, the assign_domain should set the bit corresponding
> bit not only in ap_matrix.aqm but also in ap_matrix.adm. When the
> ap_matrix is committed into the crycb no further manipulating the masks
> should take place.

Would you be fine if the control domain interface stated that it is
used to configure _additional_ control domains and the usage domain
interface stated that it is used to define usage and implicitly also
control domains? (And make the usage domain interface also set the
equivalent bit in the control domain mask.)

>
> I don't feel strongly about whether to enforce this convention about AQM
> and ADM in the kernel or not. Frankly, I don't know what is behind the
> rule. Since I can't tell if any problems are to be expected if this
> convention is violated, I would feel more comfortable if the rule was
> accommodated higher in the management stack.

I guess it depends:

- If this is a case of: "Don't configure control domains that are not
also usage domains. You are likely to go through
{code,firmware,hardware} paths that are generally not used.",
configure it in the kernel.
- If this rather is "Everybody is doing that, it's a general
convention.", configure it higher up in the stack (libvirt?)

2018-08-22 10:20:01

by Harald Freudenberger

[permalink] [raw]

Subject: Re: [PATCH v9 22/22] s390: doc: detailed specifications for AP virtualization

... about control domains

Talked with the s390 firmware guys. The convention that the control domain
mask is a superset of the usage domain mask is only true for 1st level guests.

It is absolutely valid to run a kvm guest with restricted control domain
mask bitmap in the CRYCB. It is valid to have an empty control domain mask
and the guest should be able to run crypto CPRBs on the usage domain(s) without
any problems. However, nobody has tried this.

regards
Harald Freudenberger

2018-08-22 10:21:37

[permalink] [raw]

Subject: Re: [PATCH v9 00/22] guest dedicated crypto adapters

On Mon, 13 Aug 2018 17:47:57 -0400
Tony Krowiak <[email protected]> wrote:

> From: Tony Krowiak <[email protected]>
>
> Notes:
> =====
>
> Patches 1-4 (by Harald) posted with this series are forthcoming via
> Martins tree and are based on changes in the ap driver/bus that we use as a
> foundation. They have been included here because some of the functions
> in this patch series are dependent upon them.
>
> Patches 5-6 (by David) are posted with this series because they are not
> currently in our master branch. Patches 19 and 20 of this series are
> dependent upon them. I believe David's patches are available in the
> mainline now.

What's the current status of David's patches, BTW?

It's probably easiest to rebase this upon master when the patches have
hit mainline.

I don't have any comments beyond what I already wrote; we'll hopefully
be able to figure out that control domain stuff soon.

Would be great to get some LGTM from the vfio-mdev folks :)

I believe Pierre is also doing some fixing in the existing crypto vsie
code right now?

>
> This patch series works with the v6 QEMU patches. There is no new QEMU
> patchset version yet because there have been no review comments worthy of
> creating a new series; only a couple of extremely minor nits.

Probably best to repost that once the kernel part is basically through,
if just to solicit more comments.

2018-08-22 10:54:11

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On 08/22/2018 11:42 AM, Cornelia Huck wrote:
> On Wed, 22 Aug 2018 01:18:20 +0200
> Halil Pasic <[email protected]> wrote:
>
>> On 08/21/2018 07:07 PM, Tony Krowiak wrote:
>>> This convention has been enforced by the kernel since v1. This is also
>>> enforced by both the LPAR as well as in z/VM. The following is from the
>>> PR/SM Planning Guide:
>>>
>>> Control Domain
>>> A logical partition's control domains are those cryptographic domains for which remote secure
>>> administration functions can be established and administered from this logical partition. This
>>> logical partition’s control domains must include its usage domains. For each index selected in the
>>> usage domain index list, you must select the same index in the control domain index list
>>>
>
> That's interesting.
>
>>
>> IMHO this quote is quite a half-full half-empty cup one:
>> * it mandates the set of usage domains is a subset of the set
>> of the control domains, but
>> * it speaks of independent controls, namely about the 'usage domain index'
>> and the 'control domain index list' and makes the enforcement of the rule
>> a job of the administrator (instead of codifying it in the controls).
>
> I'm wondering if a configuration with a usage domain that is not also a
> control domain is rejected outright? Anybody tried that? :)
>

I did not. This is my first exposure to the PR/SM Planning Guide. As I
stated previously the HMC interface enforces the convention by UI design:
in the activation profile you can either configure a domain as 'control' or
'control and usage' domain -- think radio button. I have no idea how
is this information feed into PR/SM and same goes for alternatives to
specify it. So I'm also very curious about this.

Another interesting question: On what level does z/VM and PR/SM
enforce the convention (i.e. on privilege level does the code
doing the enforcement run)?

>>
>>>
>>> Consequently, I'm going to opt for ensuring this is clearly documented. Based on the fact you've
>>> requested clarification of many points described in this section of the doc, I
>>> think I'll try putting my meager skills as a wordsmith to work to hopefully clarify things.
>>> I'll run it by you when I complete that task to see if I've succeeded:)
>>
>> I don't think just a doc update will do. Let me explain why.
>>
>> What describe as "... note that the AQM and ADM masks configured for the
>> mediated matrix device will be logically OR'd together to create the ADM
>> stored in the CRYCB referenced from the guest's SIE state description."
>> is a gotcha at best. The member of struct ap_matrix and the member of the
>> respective apcb in the crycb are both called 'adm', but ap_matrix.adm is
>> not an ADM as we know it from the architecture, but rather ~ AQM & ADM.
>>
>> I feel pretty strongly about this one. If we want to keep the enforcement
>> in the kernel, I guess, the assign_domain should set the bit corresponding
>> bit not only in ap_matrix.aqm but also in ap_matrix.adm. When the
>> ap_matrix is committed into the crycb no further manipulating the masks
>> should take place.
>
> Would you be fine if the control domain interface stated that it is
> used to configure _additional_ control domains and the usage domain
> interface stated that it is used to define usage and implicitly also
> control domains? (And make the usage domain interface also set the
> equivalent bit in the control domain mask.)
>

I'm fine with the interface, otherwise I would not have r-b-ed the patch.
What I strongly dislike is the implementation is IMHO very confusing (along
the lines "surprise, surprise it is called adm but it ain't adm"). This
implementation detail however can be changed any time, so I did not want
to start a big discussion as we wanted to get this out ASAP. But since
it was brought up, I decided to put in my two cents.

>>
>> I don't feel strongly about whether to enforce this convention about AQM
>> and ADM in the kernel or not. Frankly, I don't know what is behind the
>> rule. Since I can't tell if any problems are to be expected if this
>> convention is violated, I would feel more comfortable if the rule was
>> accommodated higher in the management stack.
>
> I guess it depends:
>
> - If this is a case of: "Don't configure control domains that are not
> also usage domains. You are likely to go through
> {code,firmware,hardware} paths that are generally not used.",
> configure it in the kernel.
> - If this rather is "Everybody is doing that, it's a general
> convention.", configure it higher up in the stack (libvirt?)
>

My guess is that it's the second, but I really don't know. Maybe
somebody else will help us answer this question, and also tell
what is the rationale behind the rule.

Regards,
Halil

2018-08-22 11:00:41

[permalink] [raw]

Subject: Re: [PATCH v9 00/22] guest dedicated crypto adapters

On 22/08/2018 12:19, Cornelia Huck wrote:
> On Mon, 13 Aug 2018 17:47:57 -0400
> Tony Krowiak <[email protected]> wrote:
>
>> From: Tony Krowiak <[email protected]>
>>
>> Notes:
>> =====
>>
>> Patches 1-4 (by Harald) posted with this series are forthcoming via
>> Martins tree and are based on changes in the ap driver/bus that we use as a
>> foundation. They have been included here because some of the functions
>> in this patch series are dependent upon them.
>>
>> Patches 5-6 (by David) are posted with this series because they are not
>> currently in our master branch. Patches 19 and 20 of this series are
>> dependent upon them. I believe David's patches are available in the
>> mainline now.
> What's the current status of David's patches, BTW?
>
> It's probably easiest to rebase this upon master when the patches have
> hit mainline.
>
> I don't have any comments beyond what I already wrote; we'll hopefully
> be able to figure out that control domain stuff soon.
>
> Would be great to get some LGTM from the vfio-mdev folks :)
>
> I believe Pierre is also doing some fixing in the existing crypto vsie
> code right now?
yes

Regards,
Pierre

--
Pierre Morel
Linux/KVM/QEMU in Böblingen - Germany

2018-08-22 11:04:54

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On 22/08/2018 11:42, Cornelia Huck wrote:
> On Wed, 22 Aug 2018 01:18:20 +0200
> Halil Pasic <[email protected]> wrote:
>
>> On 08/21/2018 07:07 PM, Tony Krowiak wrote:
>>> This convention has been enforced by the kernel since v1. This is also
>>> enforced by both the LPAR as well as in z/VM. The following is from the
>>> PR/SM Planning Guide:
>>>
>>> Control Domain
>>> A logical partition's control domains are those cryptographic domains for which remote secure
>>> administration functions can be established and administered from this logical partition. This
>>> logical partition’s control domains must include its usage domains. For each index selected in the
>>> usage domain index list, you must select the same index in the control domain index list
>>>
> That's interesting.
>
>> IMHO this quote is quite a half-full half-empty cup one:
>> * it mandates the set of usage domains is a subset of the set
>> of the control domains, but
>> * it speaks of independent controls, namely about the 'usage domain index'
>> and the 'control domain index list' and makes the enforcement of the rule
>> a job of the administrator (instead of codifying it in the controls).
> I'm wondering if a configuration with a usage domain that is not also a
> control domain is rejected outright? Anybody tried that? :)

Yes, and no it is not.
We can use a queue (usage domain) to a AP card for SHA-512 or RSA without
having to define the queue as a control domain.

Regards,
Pierre

--
Pierre Morel
Linux/KVM/QEMU in Böblingen - Germany

2018-08-22 11:26:24

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

On 22.08.2018 13:19, David Hildenbrand wrote:
> On 13.08.2018 23:48, Tony Krowiak wrote:
>> From: Tony Krowiak <[email protected]>
>>
>> Introduces a new CPU model feature and two CPU model
>> facilities to support AP virtualization for KVM guests.
>>
>> CPU model feature:
>>
>> The KVM_S390_VM_CPU_FEAT_AP feature indicates that
>> AP instructions are available on the guest. This
>> feature will be enabled by the kernel only if the AP
>> instructions are installed on the linux host. This feature
>> must be specifically turned on for the KVM guest from
>> userspace to use the VFIO AP device driver for guest
>> access to AP devices.
>>
>> CPU model facilities:
>>
>> 1. AP Query Configuration Information (QCI) facility is installed.
>>
>> This is indicated by setting facilities bit 12 for
>> the guest. The kernel will not enable this facility
>> for the guest if it is not set on the host.
>>
>> If this facility is not set for the KVM guest, then only
>> APQNs with an APQI less than 16 will be used by a Linux
>> guest regardless of the matrix configuration for the virtual
>> machine. This is a limitation of the Linux AP bus.
>>
>> 2. AP Facilities Test facility (APFT) is installed.
>>
>> This is indicated by setting facilities bit 15 for
>> the guest. The kernel will not enable this facility for
>> the guest if it is not set on the host.
>>
>> If this facility is not set for the KVM guest, then no
>> AP devices will be available to the guest regardless of
>> the guest's matrix configuration for the virtual
>> machine. This is a limitation of the Linux AP bus.
>>
>> Signed-off-by: Tony Krowiak <[email protected]>
>> Reviewed-by: Christian Borntraeger <[email protected]>
>> Reviewed-by: Halil Pasic <[email protected]>
>> Tested-by: Michael Mueller <[email protected]>
>> Tested-by: Farhan Ali <[email protected]>
>> Signed-off-by: Christian Borntraeger <[email protected]>
>> ---
>> arch/s390/kvm/kvm-s390.c | 5 +++++
>> arch/s390/tools/gen_facilities.c | 2 ++
>> 2 files changed, 7 insertions(+), 0 deletions(-)
>>
>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
>> index 1e8cb67..d5e04d2 100644
>> --- a/arch/s390/kvm/kvm-s390.c
>> +++ b/arch/s390/kvm/kvm-s390.c
>> @@ -367,6 +367,11 @@ static void kvm_s390_cpu_feat_init(void)
>>
>> if (MACHINE_HAS_ESOP)
>> allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
>> +
>> + /* Check if AP instructions installed on host */
>> + if (ap_instructions_available())
>> + allow_cpu_feat(KVM_S390_VM_CPU_FEAT_AP);
>> +
>> /*
>> * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
>> * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
>> diff --git a/arch/s390/tools/gen_facilities.c b/arch/s390/tools/gen_facilities.c
>> index 90a8c9e..a52290b 100644
>> --- a/arch/s390/tools/gen_facilities.c
>> +++ b/arch/s390/tools/gen_facilities.c
>> @@ -106,6 +106,8 @@ struct facility_def {
>>
>> .name = "FACILITIES_KVM_CPUMODEL",
>> .bits = (int[]){
>> + 12, /* AP Query Configuration Information */
>> + 15, /* AP Facilities Test */
>> -1 /* END */
>> }
>> },
>>
>
> I really wonder if we should also export the APXA facility.
>
> We can probe and allow that CPU feature. However, we cannot disable it
> (as of now).
>
> We have other CPU features where it is the same case (basically all
> subfunctions). See kvm_s390_get_processor_subfunc(). We probe them and
> export them, but support to disable them has never been implemented.
>
> On a high level, we could then e.g. deny to start a QEMU guest if APXA
> is available but has been disabled. (until we know that disabling it
> actually works - if ever).
>
> This helps to catch nasty migration bugs (e.g. APXA suddenly
> disappearing). Although unlikely, definitely possible.
>
>
> Are there any other AP related facilities that the guest can from now on
> probe that should also become part of the CPU model?
>

To be more precise, shouldn't PQAP(QCI) be handled just like other
subfunctions? (I remember it should)

That would imply that there is actually theoretically a way to fake away
certain AP facilities.

--

Thanks,

David / dhildenb

2018-08-22 11:34:20

[permalink] [raw]

Subject: Re: [PATCH v9 22/22] s390: doc: detailed specifications for AP virtualization

On Wed, 22 Aug 2018 09:04:13 +0200
Harald Freudenberger <[email protected]> wrote:

> Well, sooner or later this has to work. Yesterday we tested the control
> domain thing with trying to pull some simple data from a 'controlled' domain
> to the TKE - doesn't work with a Linux LPAR. I will investigate the details in the
> next weeks. However, long-term it should be possible to run scenarios
> like having one KVM guest control all the domains used by other KVM guests.
> With respect to the KVM vfio driver, currently there should be just the
> rule that for a guest the control domain mask should be equal or a superset
> of the usage domain mask. This is by convention as the architecture is
> not so clear here, but this is enforced on every place which deals with
> usage and control domains (SE, TKE).

Thanks for the update; this makes me think we really should fiddle with
the masks in the kernel (as opposed to doing it higher up in the stack).

2018-08-22 11:49:14

[permalink] [raw]

Subject: Re: [PATCH v9 22/22] s390: doc: detailed specifications for AP virtualization

On 08/22/2018 12:13 PM, Harald Freudenberger wrote:
> ... about control domains
>
> Talked with the s390 firmware guys. The convention that the control domain
> mask is a superset of the usage domain mask is only true for 1st level guests.
>
> It is absolutely valid to run a kvm guest with restricted control domain
> mask bitmap in the CRYCB. It is valid to have an empty control domain mask
> and the guest should be able to run crypto CPRBs on the usage domain(s) without
> any problems. However, nobody has tried this.

I did try this ;).

>
> regards
> Harald Freudenberger
>
>

2018-08-22 12:02:42

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

On 13.08.2018 23:48, Tony Krowiak wrote:
> From: Tony Krowiak <[email protected]>
>
> Introduces a new CPU model feature and two CPU model
> facilities to support AP virtualization for KVM guests.
>
> CPU model feature:
>
> The KVM_S390_VM_CPU_FEAT_AP feature indicates that
> AP instructions are available on the guest. This
> feature will be enabled by the kernel only if the AP
> instructions are installed on the linux host. This feature
> must be specifically turned on for the KVM guest from
> userspace to use the VFIO AP device driver for guest
> access to AP devices.
>
> CPU model facilities:
>
> 1. AP Query Configuration Information (QCI) facility is installed.
>
> This is indicated by setting facilities bit 12 for
> the guest. The kernel will not enable this facility
> for the guest if it is not set on the host.
>
> If this facility is not set for the KVM guest, then only
> APQNs with an APQI less than 16 will be used by a Linux
> guest regardless of the matrix configuration for the virtual
> machine. This is a limitation of the Linux AP bus.
>
> 2. AP Facilities Test facility (APFT) is installed.
>
> This is indicated by setting facilities bit 15 for
> the guest. The kernel will not enable this facility for
> the guest if it is not set on the host.
>
> If this facility is not set for the KVM guest, then no
> AP devices will be available to the guest regardless of
> the guest's matrix configuration for the virtual
> machine. This is a limitation of the Linux AP bus.
>
> Signed-off-by: Tony Krowiak <[email protected]>
> Reviewed-by: Christian Borntraeger <[email protected]>
> Reviewed-by: Halil Pasic <[email protected]>
> Tested-by: Michael Mueller <[email protected]>
> Tested-by: Farhan Ali <[email protected]>
> Signed-off-by: Christian Borntraeger <[email protected]>
> ---
> arch/s390/kvm/kvm-s390.c | 5 +++++
> arch/s390/tools/gen_facilities.c | 2 ++
> 2 files changed, 7 insertions(+), 0 deletions(-)
>
> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
> index 1e8cb67..d5e04d2 100644
> --- a/arch/s390/kvm/kvm-s390.c
> +++ b/arch/s390/kvm/kvm-s390.c
> @@ -367,6 +367,11 @@ static void kvm_s390_cpu_feat_init(void)
>
> if (MACHINE_HAS_ESOP)
> allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
> +
> + /* Check if AP instructions installed on host */
> + if (ap_instructions_available())
> + allow_cpu_feat(KVM_S390_VM_CPU_FEAT_AP);
> +
> /*
> * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
> * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
> diff --git a/arch/s390/tools/gen_facilities.c b/arch/s390/tools/gen_facilities.c
> index 90a8c9e..a52290b 100644
> --- a/arch/s390/tools/gen_facilities.c
> +++ b/arch/s390/tools/gen_facilities.c
> @@ -106,6 +106,8 @@ struct facility_def {
>
> .name = "FACILITIES_KVM_CPUMODEL",
> .bits = (int[]){
> + 12, /* AP Query Configuration Information */
> + 15, /* AP Facilities Test */
> -1 /* END */
> }
> },
>

I really wonder if we should also export the APXA facility.

We can probe and allow that CPU feature. However, we cannot disable it
(as of now).

We have other CPU features where it is the same case (basically all
subfunctions). See kvm_s390_get_processor_subfunc(). We probe them and
export them, but support to disable them has never been implemented.

On a high level, we could then e.g. deny to start a QEMU guest if APXA
is available but has been disabled. (until we know that disabling it
actually works - if ever).

This helps to catch nasty migration bugs (e.g. APXA suddenly
disappearing). Although unlikely, definitely possible.

Are there any other AP related facilities that the guest can from now on
probe that should also become part of the CPU model?

--

Thanks,

David / dhildenb

2018-08-22 14:33:24

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On 08/21/2018 07:18 PM, Halil Pasic wrote:
>
>
> On 08/21/2018 07:07 PM, Tony Krowiak wrote:
>> On 08/21/2018 11:25 AM, Cornelia Huck wrote:
>>> On Mon, 20 Aug 2018 13:41:32 -0400
>>> Tony Krowiak <[email protected]> wrote:
>>>
>>>> On 08/20/2018 10:23 AM, Cornelia Huck wrote:
>>>>> On Mon, 13 Aug 2018 17:48:09 -0400
>>>>> Tony Krowiak <[email protected]> wrote:
>>>>>> From: Tony Krowiak <[email protected]>
>>>>>>
>>>>>> Provides the sysfs interfaces for:
>>>>>>
>>>>>> 1. Assigning AP control domains to the mediated matrix device
>>>>>>
>>>>>> 2. Unassigning AP control domains from a mediated matrix device
>>>>>>
>>>>>> 3. Displaying the control domains assigned to a mediated matrix
>>>>>> device
>>>>>>
>>>>>> The IDs of the AP control domains assigned to the mediated matrix
>>>>>> device are stored in an AP domain mask (ADM). The bits in the ADM,
>>>>>> from most significant to least significant bit, correspond to
>>>>>> AP domain numbers 0 to 255. On some systems, the maximum allowable
>>>>>> domain number may be less than 255 - depending upon the host's
>>>>>> AP configuration - and assignment may be rejected if the input
>>>>>> domain ID exceeds the limit.
>>>>> Please remind me of the relationship between control domains and
>>>>> usage
>>>>> domains... IIRC, usage domains allow both requests and configuration,
>>>>> while control domains allow only configuration, and are by
>>>>> convention a
>>>>> superset of usage domains.
>>>> A usage domain is a domain to which an AP command-request message
>>>> can be
>>>> submitted for processing. A control domain is a domain that can
>>>> be changed by an AP command request message submitted to a usage
>>>> domain.
>>>> AP command request messages to configure a domain will contain the
>>>> domain
>>>> number of the domain to be modified. The AP firmware will check the
>>>> control domain mask (ADM) and will allow the request to proceed
>>>> only if
>>>> the corresponding bit in the ADM is set.
>>> Thanks to you and Halil for the explanation.
>>>
>>>>> Is there a hard requirement somewhere in there, or can the admin
>>>>> cheerfully use different masks for usage domains and control domains
>>>>> without the SIE choking on it?
>>>> There is no hard requirement that control domains must be a
>>>> superset of
>>>> the usage domains, it is merely an architectural convention. AFAIK,
>>>> SIE doesn't enforce this and will not break if the convention is not
>>>> enforced externally. Having said that, you should note that the AQM
>>>> and ADM masks configured for the mediated matrix device will be
>>>> logically
>>>> OR'd together to create the ADM stored in the CRYCB referenced from
>>>> the
>>>> guest's SIE state description. In other words, we are enforcing the
>>>> convention in our software.
>>> Hm, that's interesting, as Halil argued that we should not enforce it
>>> in the kernel. Might be somewhat surprising as well. If that is really
>>> the way to do it, this needs to be documented clearly.
>>
>> This convention has been enforced by the kernel since v1. This is also
>> enforced by both the LPAR as well as in z/VM. The following is from the
>> PR/SM Planning Guide:
>>
>> Control Domain
>> A logical partition's control domains are those cryptographic domains
>> for which remote secure
>> administration functions can be established and administered from
>> this logical partition. This
>> logical partition’s control domains must include its usage domains.
>> For each index selected in the
>> usage domain index list, you must select the same index in the
>> control domain index list
>>
>
> IMHO this quote is quite a half-full half-empty cup one:
> * it mandates the set of usage domains is a subset of the set
> of the control domains, but
> * it speaks of independent controls, namely about the 'usage domain
> index'
> and the 'control domain index list' and makes the enforcement of the rule
> a job of the administrator (instead of codifying it in the controls).

For what it's worth, I spoke with the z/VM developers about dedicated crypto
in z/VM. In z/VM dedicated crypto, control domains are not even
configured by
the admin. All configured usage domains are also configured as control
domains.

>
>
>>
>> Consequently, I'm going to opt for ensuring this is clearly
>> documented. Based on the fact you've
>> requested clarification of many points described in this section of
>> the doc, I
>> think I'll try putting my meager skills as a wordsmith to work to
>> hopefully clarify things.
>> I'll run it by you when I complete that task to see if I've succeeded:)
>
> I don't think just a doc update will do. Let me explain why.
>
> What describe as "... note that the AQM and ADM masks configured for the
> mediated matrix device will be logically OR'd together to create the ADM
> stored in the CRYCB referenced from the guest's SIE state description."
> is a gotcha at best. The member of struct ap_matrix and the member of the
> respective apcb in the crycb are both called 'adm', but ap_matrix.adm is
> not an ADM as we know it from the architecture, but rather ~ AQM & ADM.
>
> I feel pretty strongly about this one. If we want to keep the enforcement
> in the kernel, I guess, the assign_domain should set the bit
> corresponding
> bit not only in ap_matrix.aqm but also in ap_matrix.adm. When the
> ap_matrix is committed into the crycb no further manipulating the masks
> should take place.

I have no problem with this and considered implementing it that way at one
time.

>
> I don't feel strongly about whether to enforce this convention about AQM
> and ADM in the kernel or not. Frankly, I don't know what is behind the
> rule. Since I can't tell if any problems are to be expected if this
> convention is violated, I would feel more comfortable if the rule was
> accommodated higher in the management stack.

I wouldn't describe it as a rule. It is described in the architecture doc
as an architectural convention; in other words, it is agreed upon that all
usage domains should also be control domains. Based on my discussions with
the z/VM developers, I believe the reason for the convention is to ensure a
system has control over its own usage domains, but that is just my
interpretation.

>
>
> Regards,
> Halil
>
>>
>>>
>>

2018-08-22 14:35:34

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

On 22/08/2018 13:19, David Hildenbrand wrote:
> On 13.08.2018 23:48, Tony Krowiak wrote:
>> From: Tony Krowiak <[email protected]>
>>
>> Introduces a new CPU model feature and two CPU model
>> facilities to support AP virtualization for KVM guests.
>>
>> CPU model feature:
>>
>> The KVM_S390_VM_CPU_FEAT_AP feature indicates that
>> AP instructions are available on the guest. This
>> feature will be enabled by the kernel only if the AP
>> instructions are installed on the linux host. This feature
>> must be specifically turned on for the KVM guest from
>> userspace to use the VFIO AP device driver for guest
>> access to AP devices.
>>
>> CPU model facilities:
>>
>> 1. AP Query Configuration Information (QCI) facility is installed.
>>
>> This is indicated by setting facilities bit 12 for
>> the guest. The kernel will not enable this facility
>> for the guest if it is not set on the host.
>>
>> If this facility is not set for the KVM guest, then only
>> APQNs with an APQI less than 16 will be used by a Linux
>> guest regardless of the matrix configuration for the virtual
>> machine. This is a limitation of the Linux AP bus.
>>
>> 2. AP Facilities Test facility (APFT) is installed.
>>
>> This is indicated by setting facilities bit 15 for
>> the guest. The kernel will not enable this facility for
>> the guest if it is not set on the host.
>>
>> If this facility is not set for the KVM guest, then no
>> AP devices will be available to the guest regardless of
>> the guest's matrix configuration for the virtual
>> machine. This is a limitation of the Linux AP bus.
>>
>> Signed-off-by: Tony Krowiak <[email protected]>
>> Reviewed-by: Christian Borntraeger <[email protected]>
>> Reviewed-by: Halil Pasic <[email protected]>
>> Tested-by: Michael Mueller <[email protected]>
>> Tested-by: Farhan Ali <[email protected]>
>> Signed-off-by: Christian Borntraeger <[email protected]>
>> ---
>> arch/s390/kvm/kvm-s390.c | 5 +++++
>> arch/s390/tools/gen_facilities.c | 2 ++
>> 2 files changed, 7 insertions(+), 0 deletions(-)
>>
>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
>> index 1e8cb67..d5e04d2 100644
>> --- a/arch/s390/kvm/kvm-s390.c
>> +++ b/arch/s390/kvm/kvm-s390.c
>> @@ -367,6 +367,11 @@ static void kvm_s390_cpu_feat_init(void)
>>
>> if (MACHINE_HAS_ESOP)
>> allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
>> +
>> + /* Check if AP instructions installed on host */
>> + if (ap_instructions_available())
>> + allow_cpu_feat(KVM_S390_VM_CPU_FEAT_AP);
>> +
>> /*
>> * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
>> * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
>> diff --git a/arch/s390/tools/gen_facilities.c b/arch/s390/tools/gen_facilities.c
>> index 90a8c9e..a52290b 100644
>> --- a/arch/s390/tools/gen_facilities.c
>> +++ b/arch/s390/tools/gen_facilities.c
>> @@ -106,6 +106,8 @@ struct facility_def {
>>
>> .name = "FACILITIES_KVM_CPUMODEL",
>> .bits = (int[]){
>> + 12, /* AP Query Configuration Information */
>> + 15, /* AP Facilities Test */
>> -1 /* END */
>> }
>> },
>>
>
> I really wonder if we should also export the APXA facility.
>
> We can probe and allow that CPU feature. However, we cannot disable it
> (as of now).
>
> We have other CPU features where it is the same case (basically all
> subfunctions). See kvm_s390_get_processor_subfunc(). We probe them and
> export them, but support to disable them has never been implemented.
>
> On a high level, we could then e.g. deny to start a QEMU guest if APXA
> is available but has been disabled. (until we know that disabling it
> actually works - if ever).
>
> This helps to catch nasty migration bugs (e.g. APXA suddenly
> disappearing). Although unlikely, definitely possible. >
>
> Are there any other AP related facilities that the guest can from now on
> probe that should also become part of the CPU model?
>

Before going too far in a discussion on features which we do not really
need, we can make clear that we only support beginning with z13 and only
in the Z architecture mode as host and as guest.

We then need to abort the VFIO driver if APXA is not installed.

In this case we will have no problem with older guests not having idea
about APXA.

Would it be a solution?

Regards,
Pierre

--
Pierre Morel
Linux/KVM/QEMU in Böblingen - Germany

2018-08-22 15:18:04

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

On 22.08.2018 16:33, Pierre Morel wrote:
> On 22/08/2018 13:19, David Hildenbrand wrote:
>> On 13.08.2018 23:48, Tony Krowiak wrote:
>>> From: Tony Krowiak <[email protected]>
>>>
>>> Introduces a new CPU model feature and two CPU model
>>> facilities to support AP virtualization for KVM guests.
>>>
>>> CPU model feature:
>>>
>>> The KVM_S390_VM_CPU_FEAT_AP feature indicates that
>>> AP instructions are available on the guest. This
>>> feature will be enabled by the kernel only if the AP
>>> instructions are installed on the linux host. This feature
>>> must be specifically turned on for the KVM guest from
>>> userspace to use the VFIO AP device driver for guest
>>> access to AP devices.
>>>
>>> CPU model facilities:
>>>
>>> 1. AP Query Configuration Information (QCI) facility is installed.
>>>
>>> This is indicated by setting facilities bit 12 for
>>> the guest. The kernel will not enable this facility
>>> for the guest if it is not set on the host.
>>>
>>> If this facility is not set for the KVM guest, then only
>>> APQNs with an APQI less than 16 will be used by a Linux
>>> guest regardless of the matrix configuration for the virtual
>>> machine. This is a limitation of the Linux AP bus.
>>>
>>> 2. AP Facilities Test facility (APFT) is installed.
>>>
>>> This is indicated by setting facilities bit 15 for
>>> the guest. The kernel will not enable this facility for
>>> the guest if it is not set on the host.
>>>
>>> If this facility is not set for the KVM guest, then no
>>> AP devices will be available to the guest regardless of
>>> the guest's matrix configuration for the virtual
>>> machine. This is a limitation of the Linux AP bus.
>>>
>>> Signed-off-by: Tony Krowiak <[email protected]>
>>> Reviewed-by: Christian Borntraeger <[email protected]>
>>> Reviewed-by: Halil Pasic <[email protected]>
>>> Tested-by: Michael Mueller <[email protected]>
>>> Tested-by: Farhan Ali <[email protected]>
>>> Signed-off-by: Christian Borntraeger <[email protected]>
>>> ---
>>> arch/s390/kvm/kvm-s390.c | 5 +++++
>>> arch/s390/tools/gen_facilities.c | 2 ++
>>> 2 files changed, 7 insertions(+), 0 deletions(-)
>>>
>>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
>>> index 1e8cb67..d5e04d2 100644
>>> --- a/arch/s390/kvm/kvm-s390.c
>>> +++ b/arch/s390/kvm/kvm-s390.c
>>> @@ -367,6 +367,11 @@ static void kvm_s390_cpu_feat_init(void)
>>>
>>> if (MACHINE_HAS_ESOP)
>>> allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
>>> +
>>> + /* Check if AP instructions installed on host */
>>> + if (ap_instructions_available())
>>> + allow_cpu_feat(KVM_S390_VM_CPU_FEAT_AP);
>>> +
>>> /*
>>> * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
>>> * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
>>> diff --git a/arch/s390/tools/gen_facilities.c b/arch/s390/tools/gen_facilities.c
>>> index 90a8c9e..a52290b 100644
>>> --- a/arch/s390/tools/gen_facilities.c
>>> +++ b/arch/s390/tools/gen_facilities.c
>>> @@ -106,6 +106,8 @@ struct facility_def {
>>>
>>> .name = "FACILITIES_KVM_CPUMODEL",
>>> .bits = (int[]){
>>> + 12, /* AP Query Configuration Information */
>>> + 15, /* AP Facilities Test */
>>> -1 /* END */
>>> }
>>> },
>>>
>>
>> I really wonder if we should also export the APXA facility.
>>
>> We can probe and allow that CPU feature. However, we cannot disable it
>> (as of now).
>>
>> We have other CPU features where it is the same case (basically all
>> subfunctions). See kvm_s390_get_processor_subfunc(). We probe them and
>> export them, but support to disable them has never been implemented.
>>
>> On a high level, we could then e.g. deny to start a QEMU guest if APXA
>> is available but has been disabled. (until we know that disabling it
>> actually works - if ever).
>>
>> This helps to catch nasty migration bugs (e.g. APXA suddenly
>> disappearing). Although unlikely, definitely possible. >
>>
>> Are there any other AP related facilities that the guest can from now on
>> probe that should also become part of the CPU model?
>>
>
>
>
> Before going too far in a discussion on features which we do not really
> need, we can make clear that we only support beginning with z13 and only
> in the Z architecture mode as host and as guest.

Easy answer:

The CPU model should be prepared for all eventualities. We have handled
it that way since the beginning.

The minimal thing I expect to have is all relevant features probed and
exported to user space. Just like we do with all the MSA/PTFF/PLO
subfunctions. I expect (and remember) this to be the same for PQAP.

(there are some very special cases regarding subfunctions that are not
indicated)

Why? The CPU model is not KVM specific.

>
> We then need to abort the VFIO driver if APXA is not installed.

While you should do that, the CPU model is more generic. This would only
imply that as of now, the APXA feature would always be available if the
AP feature is available.

In addition, it makes the vSIE handling code easier - there is always
APXA and for now it cannot be disabled.

>
> In this case we will have no problem with older guests not having idea
> about APXA.
>
> Would it be a solution?

Any feature the guest sees, should be part of the CPU model. The whole
environment for cpu subfunctions is already in place both in KVM and
QEMU. Only disabling subfunctions in KVM is not implemented yet.

You can exclude any subfunctions/facilities that are only valid on LPAR
level and cannot be used in some guest either way. (that makes life
sometimes easier)

I know that this might sound a little bit complicated, but it really
isn't. Boils down to modifying kvm_s390_cpu_feat_init() and specifying
some features+feature groups in QEMU.

>
>
> Regards,
> Pierre
>

--

Thanks,

David / dhildenb

2018-08-22 15:19:18

by Christian Borntraeger

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On 08/22/2018 01:03 PM, Pierre Morel wrote:
>> That's interesting.
>>
>>> IMHO this quote is quite a half-full half-empty cup one:
>>> * it mandates the set of usage domains is a subset of the set
>>> of the control domains, but
>>> * it speaks of independent controls, namely about the 'usage domain index'
>>> and the 'control domain index list' and makes the enforcement of the rule
>>> a job of the administrator (instead of codifying it in the controls).
>> I'm wondering if a configuration with a usage domain that is not also a
>> control domain is rejected outright? Anybody tried that? :)
>
> Yes, and no it is not.
> We can use a queue (usage domain) to a AP card for SHA-512 or RSA without
> having to define the queue as a control domain.

Huh? My HMC allows to add a domain as
- control only domain
- control and usage domain.

But I am not able to configure a usage-only domain for my LPAR. That seems to match
the current code, no?

2018-08-22 15:21:21

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On 08/22/2018 05:42 AM, Cornelia Huck wrote:
> On Wed, 22 Aug 2018 01:18:20 +0200
> Halil Pasic <[email protected]> wrote:
>
>> On 08/21/2018 07:07 PM, Tony Krowiak wrote:
>>> This convention has been enforced by the kernel since v1. This is also
>>> enforced by both the LPAR as well as in z/VM. The following is from the
>>> PR/SM Planning Guide:
>>>
>>> Control Domain
>>> A logical partition's control domains are those cryptographic domains for which remote secure
>>> administration functions can be established and administered from this logical partition. This
>>> logical partition’s control domains must include its usage domains. For each index selected in the
>>> usage domain index list, you must select the same index in the control domain index list
>>>
> That's interesting.
>
>> IMHO this quote is quite a half-full half-empty cup one:
>> * it mandates the set of usage domains is a subset of the set
>> of the control domains, but
>> * it speaks of independent controls, namely about the 'usage domain index'
>> and the 'control domain index list' and makes the enforcement of the rule
>> a job of the administrator (instead of codifying it in the controls).
> I'm wondering if a configuration with a usage domain that is not also a
> control domain is rejected outright? Anybody tried that? :)

That's been tried and is not rejected.

>
>>> Consequently, I'm going to opt for ensuring this is clearly documented. Based on the fact you've
>>> requested clarification of many points described in this section of the doc, I
>>> think I'll try putting my meager skills as a wordsmith to work to hopefully clarify things.
>>> I'll run it by you when I complete that task to see if I've succeeded:)
>> I don't think just a doc update will do. Let me explain why.
>>
>> What describe as "... note that the AQM and ADM masks configured for the
>> mediated matrix device will be logically OR'd together to create the ADM
>> stored in the CRYCB referenced from the guest's SIE state description."
>> is a gotcha at best. The member of struct ap_matrix and the member of the
>> respective apcb in the crycb are both called 'adm', but ap_matrix.adm is
>> not an ADM as we know it from the architecture, but rather ~ AQM & ADM.
>>
>> I feel pretty strongly about this one. If we want to keep the enforcement
>> in the kernel, I guess, the assign_domain should set the bit corresponding
>> bit not only in ap_matrix.aqm but also in ap_matrix.adm. When the
>> ap_matrix is committed into the crycb no further manipulating the masks
>> should take place.
> Would you be fine if the control domain interface stated that it is
> used to configure _additional_ control domains and the usage domain
> interface stated that it is used to define usage and implicitly also
> control domains? (And make the usage domain interface also set the
> equivalent bit in the control domain mask.)

I think that is the better way to go and is something Halil recommended
in another post.

>
>> I don't feel strongly about whether to enforce this convention about AQM
>> and ADM in the kernel or not. Frankly, I don't know what is behind the
>> rule. Since I can't tell if any problems are to be expected if this
>> convention is violated, I would feel more comfortable if the rule was
>> accommodated higher in the management stack.
> I guess it depends:
>
> - If this is a case of: "Don't configure control domains that are not
> also usage domains. You are likely to go through
> {code,firmware,hardware} paths that are generally not used.",
> configure it in the kernel.
> - If this rather is "Everybody is doing that, it's a general
> convention.", configure it higher up in the stack (libvirt?)

I have come to the conclusion that the convention should be enforced
in the sysfs interfaces of the mediated matrix device as follows:

1. All domains assigned as usage domains will also be implicitly
assigned as control domains.

2. Control domains that are not usage domains may be assigned via
the assign_control_domain interface.

My reason is to maintain consistency across platforms, because:

1. The architecture doc states that control domains are a superset
of the usage domains.

2. The HMC interface for assigning domains to the LPAR enforces
the convention.

3. The PR/SM documentation states the same.

>

2018-08-22 15:36:41

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On 22/08/2018 17:11, Christian Borntraeger wrote:
>
>
> On 08/22/2018 01:03 PM, Pierre Morel wrote:
>>> That's interesting.
>>>
>>>> IMHO this quote is quite a half-full half-empty cup one:
>>>> * it mandates the set of usage domains is a subset of the set
>>>> of the control domains, but
>>>> * it speaks of independent controls, namely about the 'usage domain index'
>>>> and the 'control domain index list' and makes the enforcement of the rule
>>>> a job of the administrator (instead of codifying it in the controls).
>>> I'm wondering if a configuration with a usage domain that is not also a
>>> control domain is rejected outright? Anybody tried that? :)
>>
>> Yes, and no it is not.
>> We can use a queue (usage domain) to a AP card for SHA-512 or RSA without
>> having to define the queue as a control domain.
>
> Huh? My HMC allows to add a domain as
> - control only domain
> - control and usage domain.
>
> But I am not able to configure a usage-only domain for my LPAR. That seems to match
> the current code, no?
>

Yes, it may not be configurable by the HMC but if we start a guest with
no control domain it is not a problem to access the hardware through the
usage domain.

I tested this a long time ago, but tested again today to be sure on my LPAR.

AFAIU adding a control only domain and a control and usage domain
allows say:
control and usage domain 1
control only domain 2

Allow to send a message to domain 2 using queue 1

Allow also to send a domain modifying message to domain 1 using queue 1

control domain are domain which are controlled
It seems that the HMC enforce the LPARs to have access to their usage
domain (AFAIU from Harald)

--
Pierre Morel
Linux/KVM/QEMU in Böblingen - Germany

2018-08-22 15:50:36

by Christian Borntraeger

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On 08/22/2018 05:34 PM, Pierre Morel wrote:
> On 22/08/2018 17:11, Christian Borntraeger wrote:
>>
>>
>> On 08/22/2018 01:03 PM, Pierre Morel wrote:
>>>> That's interesting.
>>>>
>>>>> IMHO this quote is quite a half-full half-empty cup one:
>>>>> * it mandates the set of usage domains is a subset of the set
>>>>> of the control domains, but
>>>>> * it speaks of independent controls, namely about the 'usage domain index'
>>>>> and the 'control domain index list' and makes the enforcement of the rule
>>>>> a job of the administrator (instead of codifying it in the controls).
>>>> I'm wondering if a configuration with a usage domain that is not also a
>>>> control domain is rejected outright? Anybody tried that? :)
>>>
>>> Yes, and no it is not.
>>> We can use a queue (usage domain) to a AP card for SHA-512 or RSA without
>>> having to define the queue as a control domain.
>>
>> Huh? My HMC allows to add a domain as
>> - control only domain
>> - control and usage domain.
>>
>> But I am not able to configure a usage-only domain for my LPAR. That seems to match
>> the current code, no?
>>
>
> Yes, it may not be configurable by the HMC but if we start a guest with no control domain it is not a problem to access the hardware through the usage domain.
>
> I tested this a long time ago, but tested again today to be sure on my LPAR.
>
> AFAIU adding a control only domain and a control and usage domain
> allows say:
> control and usage domain 1
> control only domain 2
>
> Allow to send a message to domain 2 using queue 1
>
> Allow also to send a domain modifying message to domain 1 using queue 1
>
> control domain are domain which are controlled

So you have changed the code to not automatically make a usage domain a
control domain in the bitfield (and you could still use it as a usage
domain). Correct?
I think this is probably expected. the "usage implies control" seems to
be a convention implemented by HMC (lpar) and z/VM but millicode offers
the bits to have usage-only domains. As LPAR and z/VM will always enable
any usage-domain to also be a control domain we should do the same.

> It seems that the HMC enforce the LPARs to have access to their usage domain (AFAIU from Harald)

2018-08-22 15:52:22

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

On 22/08/2018 17:04, David Hildenbrand wrote:
> On 22.08.2018 16:33, Pierre Morel wrote:
>> On 22/08/2018 13:19, David Hildenbrand wrote:
>>> On 13.08.2018 23:48, Tony Krowiak wrote:
>>>> From: Tony Krowiak <[email protected]>
>>>>
>>>> Introduces a new CPU model feature and two CPU model
>>>> facilities to support AP virtualization for KVM guests.
>>>>
>>>> CPU model feature:
>>>>
>>>> The KVM_S390_VM_CPU_FEAT_AP feature indicates that
>>>> AP instructions are available on the guest. This
>>>> feature will be enabled by the kernel only if the AP
>>>> instructions are installed on the linux host. This feature
>>>> must be specifically turned on for the KVM guest from
>>>> userspace to use the VFIO AP device driver for guest
>>>> access to AP devices.
>>>>
>>>> CPU model facilities:
>>>>
>>>> 1. AP Query Configuration Information (QCI) facility is installed.
>>>>
>>>> This is indicated by setting facilities bit 12 for
>>>> the guest. The kernel will not enable this facility
>>>> for the guest if it is not set on the host.
>>>>
>>>> If this facility is not set for the KVM guest, then only
>>>> APQNs with an APQI less than 16 will be used by a Linux
>>>> guest regardless of the matrix configuration for the virtual
>>>> machine. This is a limitation of the Linux AP bus.
>>>>
>>>> 2. AP Facilities Test facility (APFT) is installed.
>>>>
>>>> This is indicated by setting facilities bit 15 for
>>>> the guest. The kernel will not enable this facility for
>>>> the guest if it is not set on the host.
>>>>
>>>> If this facility is not set for the KVM guest, then no
>>>> AP devices will be available to the guest regardless of
>>>> the guest's matrix configuration for the virtual
>>>> machine. This is a limitation of the Linux AP bus.
>>>>
>>>> Signed-off-by: Tony Krowiak <[email protected]>
>>>> Reviewed-by: Christian Borntraeger <[email protected]>
>>>> Reviewed-by: Halil Pasic <[email protected]>
>>>> Tested-by: Michael Mueller <[email protected]>
>>>> Tested-by: Farhan Ali <[email protected]>
>>>> Signed-off-by: Christian Borntraeger <[email protected]>
>>>> ---
>>>> arch/s390/kvm/kvm-s390.c | 5 +++++
>>>> arch/s390/tools/gen_facilities.c | 2 ++
>>>> 2 files changed, 7 insertions(+), 0 deletions(-)
>>>>
>>>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
>>>> index 1e8cb67..d5e04d2 100644
>>>> --- a/arch/s390/kvm/kvm-s390.c
>>>> +++ b/arch/s390/kvm/kvm-s390.c
>>>> @@ -367,6 +367,11 @@ static void kvm_s390_cpu_feat_init(void)
>>>>
>>>> if (MACHINE_HAS_ESOP)
>>>> allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
>>>> +
>>>> + /* Check if AP instructions installed on host */
>>>> + if (ap_instructions_available())
>>>> + allow_cpu_feat(KVM_S390_VM_CPU_FEAT_AP);
>>>> +
>>>> /*
>>>> * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
>>>> * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
>>>> diff --git a/arch/s390/tools/gen_facilities.c b/arch/s390/tools/gen_facilities.c
>>>> index 90a8c9e..a52290b 100644
>>>> --- a/arch/s390/tools/gen_facilities.c
>>>> +++ b/arch/s390/tools/gen_facilities.c
>>>> @@ -106,6 +106,8 @@ struct facility_def {
>>>>
>>>> .name = "FACILITIES_KVM_CPUMODEL",
>>>> .bits = (int[]){
>>>> + 12, /* AP Query Configuration Information */
>>>> + 15, /* AP Facilities Test */
>>>> -1 /* END */
>>>> }
>>>> },
>>>>
>>>
>>> I really wonder if we should also export the APXA facility.
>>>
>>> We can probe and allow that CPU feature. However, we cannot disable it
>>> (as of now).
>>>
>>> We have other CPU features where it is the same case (basically all
>>> subfunctions). See kvm_s390_get_processor_subfunc(). We probe them and
>>> export them, but support to disable them has never been implemented.
>>>
>>> On a high level, we could then e.g. deny to start a QEMU guest if APXA
>>> is available but has been disabled. (until we know that disabling it
>>> actually works - if ever).
>>>
>>> This helps to catch nasty migration bugs (e.g. APXA suddenly
>>> disappearing). Although unlikely, definitely possible. >
>>>
>>> Are there any other AP related facilities that the guest can from now on
>>> probe that should also become part of the CPU model?
>>>
>>
>>
>>
>> Before going too far in a discussion on features which we do not really
>> need, we can make clear that we only support beginning with z13 and only
>> in the Z architecture mode as host and as guest.
>
> Easy answer:
>
> The CPU model should be prepared for all eventualities. We have handled
> it that way since the beginning.
>
> The minimal thing I expect to have is all relevant features probed and
> exported to user space. Just like we do with all the MSA/PTFF/PLO
> subfunctions. I expect (and remember) this to be the same for PQAP.

OK, we will need a separate patch.

>
> (there are some very special cases regarding subfunctions that are not
> indicated)
>
> Why? The CPU model is not KVM specific.

Is it true for privilege instructions?

>
>>
>> We then need to abort the VFIO driver if APXA is not installed.
>
> While you should do that, the CPU model is more generic. This would only
> imply that as of now, the APXA feature would always be available if the
> AP feature is available.

yes

>
> In addition, it makes the vSIE handling code easier - there is always
> APXA and for now it cannot be disabled.
>

yes :)

>>
>> In this case we will have no problem with older guests not having idea
>> about APXA.
>>
>> Would it be a solution?
>
> Any feature the guest sees, should be part of the CPU model. The whole
> environment for cpu subfunctions is already in place both in KVM and
> QEMU. Only disabling subfunctions in KVM is not implemented yet.
>
> You can exclude any subfunctions/facilities that are only valid on LPAR
> level and cannot be used in some guest either way. (that makes life
> sometimes easier)
>
>
> I know that this might sound a little bit complicated, but it really
> isn't. Boils down to modifying kvm_s390_cpu_feat_init() and specifying
> some features+feature groups in QEMU.

OK, we definitively need another patch/patch-set, to handle this.
Do you think it can be done in another series since if we always support
APXA when we have AP instructions, we already have an indication that
APXA exist: the AP facility.

Regards,
Pierre

--
Pierre Morel
Linux/KVM/QEMU in Böblingen - Germany

2018-08-22 16:07:30

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On 22/08/2018 17:48, Christian Borntraeger wrote:
> On 08/22/2018 05:34 PM, Pierre Morel wrote:
>> On 22/08/2018 17:11, Christian Borntraeger wrote:
>>>
>>>
>>> On 08/22/2018 01:03 PM, Pierre Morel wrote:
>>>>> That's interesting.
>>>>>
>>>>>> IMHO this quote is quite a half-full half-empty cup one:
>>>>>> * it mandates the set of usage domains is a subset of the set
>>>>>> of the control domains, but
>>>>>> * it speaks of independent controls, namely about the 'usage domain index'
>>>>>> and the 'control domain index list' and makes the enforcement of the rule
>>>>>> a job of the administrator (instead of codifying it in the controls).
>>>>> I'm wondering if a configuration with a usage domain that is not also a
>>>>> control domain is rejected outright? Anybody tried that? :)
>>>>
>>>> Yes, and no it is not.
>>>> We can use a queue (usage domain) to a AP card for SHA-512 or RSA without
>>>> having to define the queue as a control domain.
>>>
>>> Huh? My HMC allows to add a domain as
>>> - control only domain
>>> - control and usage domain.
>>>
>>> But I am not able to configure a usage-only domain for my LPAR. That seems to match
>>> the current code, no?
>>>
>>
>> Yes, it may not be configurable by the HMC but if we start a guest with no control domain it is not a problem to access the hardware through the usage domain.
>>
>> I tested this a long time ago, but tested again today to be sure on my LPAR.
>>
>> AFAIU adding a control only domain and a control and usage domain
>> allows say:
>> control and usage domain 1
>> control only domain 2
>>
>> Allow to send a message to domain 2 using queue 1
>>
>> Allow also to send a domain modifying message to domain 1 using queue 1
>>
>> control domain are domain which are controlled
>
> So you have changed the code to not automatically make a usage domain a
> control domain in the bitfield (and you could still use it as a usage
> domain). Correct?

yes
and I used Harald's libica tests to verify it in the guest.

> I think this is probably expected. the "usage implies control" seems to
> be a convention implemented by HMC (lpar) and z/VM but millicode offers
> the bits to have usage-only domains. As LPAR and z/VM will always enable
> any usage-domain to also be a control domain we should do the same.
>
>
>> It seems that the HMC enforce the LPARs to have access to their usage domain (AFAIU from Harald)

--
Pierre Morel
Linux/KVM/QEMU in Böblingen - Germany

2018-08-22 17:00:13

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

>>>
>>> In this case we will have no problem with older guests not having idea
>>> about APXA.
>>>
>>> Would it be a solution?
>>
>> Any feature the guest sees, should be part of the CPU model. The whole
>> environment for cpu subfunctions is already in place both in KVM and
>> QEMU. Only disabling subfunctions in KVM is not implemented yet.
>>
>> You can exclude any subfunctions/facilities that are only valid on LPAR
>> level and cannot be used in some guest either way. (that makes life
>> sometimes easier)
>>
>>
>> I know that this might sound a little bit complicated, but it really
>> isn't. Boils down to modifying kvm_s390_cpu_feat_init() and specifying
>> some features+feature groups in QEMU.
>
> OK, we definitively need another patch/patch-set, to handle this.
> Do you think it can be done in another series since if we always support
> APXA when we have AP instructions, we already have an indication that
> APXA exist: the AP facility.
>

Please implement the subfunction stuff right away. This will allow to
handle all future facilities transparently from a kernel POV.

Implementing that should be easy - and I don't like gluing features
together in such a way.

You can always assure that consistent data (e.g. AP + APXA availability)
is reported from KVM to QEMU.

> Regards,
> Pierre
>
>
>
>

--

Thanks,

David / dhildenb

2018-08-22 17:13:59

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On 08/22/2018 05:48 PM, Christian Borntraeger wrote:
> On 08/22/2018 05:34 PM, Pierre Morel wrote:
>> On 22/08/2018 17:11, Christian Borntraeger wrote:
>>>
>>>
>>> On 08/22/2018 01:03 PM, Pierre Morel wrote:
>>>>> That's interesting.
>>>>>
>>>>>> IMHO this quote is quite a half-full half-empty cup one:
>>>>>> * it mandates the set of usage domains is a subset of the set
>>>>>> of the control domains, but
>>>>>> * it speaks of independent controls, namely about the 'usage domain index'
>>>>>> and the 'control domain index list' and makes the enforcement of the rule
>>>>>> a job of the administrator (instead of codifying it in the controls).
>>>>> I'm wondering if a configuration with a usage domain that is not also a
>>>>> control domain is rejected outright? Anybody tried that? :)
>>>>
>>>> Yes, and no it is not.
>>>> We can use a queue (usage domain) to a AP card for SHA-512 or RSA without
>>>> having to define the queue as a control domain.
>>>
>>> Huh? My HMC allows to add a domain as
>>> - control only domain
>>> - control and usage domain.
>>>
>>> But I am not able to configure a usage-only domain for my LPAR. That seems to match
>>> the current code, no?
>>>
>>
>> Yes, it may not be configurable by the HMC but if we start a guest with no control domain it is not a problem to access the hardware through the usage domain.
>>
>> I tested this a long time ago, but tested again today to be sure on my LPAR.
>>
>> AFAIU adding a control only domain and a control and usage domain
>> allows say:
>> control and usage domain 1
>> control only domain 2
>>
>> Allow to send a message to domain 2 using queue 1
>>
>> Allow also to send a domain modifying message to domain 1 using queue 1
>>
>> control domain are domain which are controlled
>
> So you have changed the code to not automatically make a usage domain a
> control domain in the bitfield (and you could still use it as a usage
> domain). Correct?

I tested basically the same yesterday, with the same results.

> I think this is probably expected. the "usage implies control" seems to
> be a convention implemented by HMC (lpar) and z/VM but millicode offers
> the bits to have usage-only domains. As LPAR and z/VM will always enable
> any usage-domain to also be a control domain we should do the same.

I'm fine either way, but slightly prefer higher level management software
and not the kernel accommodating this convention.

Please consider a quote from Harald's mail in another sub-thread

"""
... about control domains

Talked with the s390 firmware guys. The convention that the control domain
mask is a superset of the usage domain mask is only true for 1st level guests.

It is absolutely valid to run a kvm guest with restricted control domain
mask bitmap in the CRYCB. It is valid to have an empty control domain mask
and the guest should be able to run crypto CPRBs on the usage domain(s) without
any problems. However, nobody has tried this.
"""

I'm yet to get an explanation why was this convention established in the first
place. And I can not figure it out myself. For me a setup where I know that
the domains used by some guest can not be modified by the same guest makes
perfect sense. If I try to think in analogies, I kind of compare modification
(that is control domain) with write access, and usage (that is usage domain)
with read access to, let's say a regular file. For me, all options (rw, r, and w)
do make sense, and if I had to pick the one that makes the least sense I would
pick write only. The convention is in these terms making read-only illegal. But
should 'usage only domains' ever get identified as something somebody wants to do
we can just add an attribute for that. So I'm fine either way.

Still I find the commit message for this patch, the implementation of
assign_control_domain() and also the documentation slightly misleading regarding
what does one get from assign_domain.

Regards,
Halil

>
>
>> It seems that the HMC enforce the LPARs to have access to their usage domain (AFAIU from Harald)
>

2018-08-22 19:01:07

[permalink] [raw]

Subject: Re: [PATCH v9 00/22] guest dedicated crypto adapters

On 08/22/2018 06:19 AM, Cornelia Huck wrote:
> On Mon, 13 Aug 2018 17:47:57 -0400
> Tony Krowiak <[email protected]> wrote:
>
>> From: Tony Krowiak <[email protected]>
>>
>> Notes:
>> =====
>>
>> Patches 1-4 (by Harald) posted with this series are forthcoming via
>> Martins tree and are based on changes in the ap driver/bus that we use as a
>> foundation. They have been included here because some of the functions
>> in this patch series are dependent upon them.
>>
>> Patches 5-6 (by David) are posted with this series because they are not
>> currently in our master branch. Patches 19 and 20 of this series are
>> dependent upon them. I believe David's patches are available in the
>> mainline now.
> What's the current status of David's patches, BTW?
>
> It's probably easiest to rebase this upon master when the patches have
> hit mainline.
>
> I don't have any comments beyond what I already wrote; we'll hopefully
> be able to figure out that control domain stuff soon.
>
> Would be great to get some LGTM from the vfio-mdev folks :)
>
> I believe Pierre is also doing some fixing in the existing crypto vsie
> code right now?
Yes. I will roll his patches into my branch when available prior to
posting v10.

>
>> This patch series works with the v6 QEMU patches. There is no new QEMU
>> patchset version yet because there have been no review comments worthy of
>> creating a new series; only a couple of extremely minor nits.
> Probably best to repost that once the kernel part is basically through,
> if just to solicit more comments.

Will do.

>

2018-08-22 19:27:54

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On 08/22/2018 01:11 PM, Halil Pasic wrote:
>
>
> On 08/22/2018 05:48 PM, Christian Borntraeger wrote:
>> On 08/22/2018 05:34 PM, Pierre Morel wrote:
>>> On 22/08/2018 17:11, Christian Borntraeger wrote:
>>>>
>>>>
>>>> On 08/22/2018 01:03 PM, Pierre Morel wrote:
>>>>>> That's interesting.
>>>>>>
>>>>>>> IMHO this quote is quite a half-full half-empty cup one:
>>>>>>> * it mandates the set of usage domains is a subset of the set
>>>>>>> of the control domains, but
>>>>>>> * it speaks of independent controls, namely about the 'usage
>>>>>>> domain index'
>>>>>>> and the 'control domain index list' and makes the enforcement of
>>>>>>> the rule
>>>>>>> a job of the administrator (instead of codifying it in the
>>>>>>> controls).
>>>>>> I'm wondering if a configuration with a usage domain that is not
>>>>>> also a
>>>>>> control domain is rejected outright? Anybody tried that? :)
>>>>>
>>>>> Yes, and no it is not.
>>>>> We can use a queue (usage domain) to a AP card for SHA-512 or RSA
>>>>> without
>>>>> having to define the queue as a control domain.
>>>>
>>>> Huh? My HMC allows to add a domain as
>>>> - control only domain
>>>> - control and usage domain.
>>>>
>>>> But I am not able to configure a usage-only domain for my LPAR.
>>>> That seems to match
>>>> the current code, no?
>>>>
>>>
>>> Yes, it may not be configurable by the HMC but if we start a guest
>>> with no control domain it is not a problem to access the hardware
>>> through the usage domain.
>>>
>>> I tested this a long time ago, but tested again today to be sure on
>>> my LPAR.
>>>
>>> AFAIU adding a control only domain and a control and usage domain
>>> allows say:
>>> control and usage domain 1
>>> control only domain 2
>>>
>>> Allow to send a message to domain 2 using queue 1
>>>
>>> Allow also to send a domain modifying message to domain 1 using queue 1
>>>
>>> control domain are domain which are controlled
>>
>> So you have changed the code to not automatically make a usage domain a
>> control domain in the bitfield (and you could still use it as a usage
>> domain). Correct?
>
> I tested basically the same yesterday, with the same results.
>
>> I think this is probably expected. the "usage implies control" seems to
>> be a convention implemented by HMC (lpar) and z/VM but millicode offers
>> the bits to have usage-only domains. As LPAR and z/VM will always enable
>> any usage-domain to also be a control domain we should do the same.
>
> I'm fine either way, but slightly prefer higher level management software
> and not the kernel accommodating this convention.
>
> Please consider a quote from Harald's mail in another sub-thread
>
>
> """
> ... about control domains
>
> Talked with the s390 firmware guys. The convention that the control
> domain
> mask is a superset of the usage domain mask is only true for 1st level
> guests.
>
> It is absolutely valid to run a kvm guest with restricted control domain
> mask bitmap in the CRYCB. It is valid to have an empty control domain
> mask
> and the guest should be able to run crypto CPRBs on the usage
> domain(s) without
> any problems. However, nobody has tried this.
> """
>
> I'm yet to get an explanation why was this convention established in
> the first
> place. And I can not figure it out myself. For me a setup where I know
> that
> the domains used by some guest can not be modified by the same guest
> makes
> perfect sense. If I try to think in analogies, I kind of compare
> modification
> (that is control domain) with write access, and usage (that is usage
> domain)
> with read access to, let's say a regular file. For me, all options
> (rw, r, and w)
> do make sense, and if I had to pick the one that makes the least sense
> I would
> pick write only. The convention is in these terms making read-only
> illegal. But
> should 'usage only domains' ever get identified as something somebody
> wants to do
> we can just add an attribute for that. So I'm fine either way.

One of the things I suggested in a private conversation with Christian
earlier
today was to provide an additional rw sysfs attribute - a boolean - that
indicates
whether all usage domains should also be control domains. The default
could be
true. This would allow one to configure guests with usage-only domains
as well
as satisfy the convention.

>
>
> Still I find the commit message for this patch, the implementation of
> assign_control_domain() and also the documentation slightly misleading
> regarding
> what does one get from assign_domain.

>
>
> Regards,
> Halil
>
>
>
>
>
>>
>>
>>> It seems that the HMC enforce the LPARs to have access to their
>>> usage domain (AFAIU from Harald)
>>

2018-08-22 20:18:01

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

On 08/22/2018 07:24 AM, David Hildenbrand wrote:
> On 22.08.2018 13:19, David Hildenbrand wrote:
>> On 13.08.2018 23:48, Tony Krowiak wrote:
>>> From: Tony Krowiak <[email protected]>
>>>
>>> Introduces a new CPU model feature and two CPU model
>>> facilities to support AP virtualization for KVM guests.
>>>
>>> CPU model feature:
>>>
>>> The KVM_S390_VM_CPU_FEAT_AP feature indicates that
>>> AP instructions are available on the guest. This
>>> feature will be enabled by the kernel only if the AP
>>> instructions are installed on the linux host. This feature
>>> must be specifically turned on for the KVM guest from
>>> userspace to use the VFIO AP device driver for guest
>>> access to AP devices.
>>>
>>> CPU model facilities:
>>>
>>> 1. AP Query Configuration Information (QCI) facility is installed.
>>>
>>> This is indicated by setting facilities bit 12 for
>>> the guest. The kernel will not enable this facility
>>> for the guest if it is not set on the host.
>>>
>>> If this facility is not set for the KVM guest, then only
>>> APQNs with an APQI less than 16 will be used by a Linux
>>> guest regardless of the matrix configuration for the virtual
>>> machine. This is a limitation of the Linux AP bus.
>>>
>>> 2. AP Facilities Test facility (APFT) is installed.
>>>
>>> This is indicated by setting facilities bit 15 for
>>> the guest. The kernel will not enable this facility for
>>> the guest if it is not set on the host.
>>>
>>> If this facility is not set for the KVM guest, then no
>>> AP devices will be available to the guest regardless of
>>> the guest's matrix configuration for the virtual
>>> machine. This is a limitation of the Linux AP bus.
>>>
>>> Signed-off-by: Tony Krowiak <[email protected]>
>>> Reviewed-by: Christian Borntraeger <[email protected]>
>>> Reviewed-by: Halil Pasic <[email protected]>
>>> Tested-by: Michael Mueller <[email protected]>
>>> Tested-by: Farhan Ali <[email protected]>
>>> Signed-off-by: Christian Borntraeger <[email protected]>
>>> ---
>>> arch/s390/kvm/kvm-s390.c | 5 +++++
>>> arch/s390/tools/gen_facilities.c | 2 ++
>>> 2 files changed, 7 insertions(+), 0 deletions(-)
>>>
>>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
>>> index 1e8cb67..d5e04d2 100644
>>> --- a/arch/s390/kvm/kvm-s390.c
>>> +++ b/arch/s390/kvm/kvm-s390.c
>>> @@ -367,6 +367,11 @@ static void kvm_s390_cpu_feat_init(void)
>>>
>>> if (MACHINE_HAS_ESOP)
>>> allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
>>> +
>>> + /* Check if AP instructions installed on host */
>>> + if (ap_instructions_available())
>>> + allow_cpu_feat(KVM_S390_VM_CPU_FEAT_AP);
>>> +
>>> /*
>>> * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
>>> * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
>>> diff --git a/arch/s390/tools/gen_facilities.c b/arch/s390/tools/gen_facilities.c
>>> index 90a8c9e..a52290b 100644
>>> --- a/arch/s390/tools/gen_facilities.c
>>> +++ b/arch/s390/tools/gen_facilities.c
>>> @@ -106,6 +106,8 @@ struct facility_def {
>>>
>>> .name = "FACILITIES_KVM_CPUMODEL",
>>> .bits = (int[]){
>>> + 12, /* AP Query Configuration Information */
>>> + 15, /* AP Facilities Test */
>>> -1 /* END */
>>> }
>>> },
>>>
>> I really wonder if we should also export the APXA facility.
>>
>> We can probe and allow that CPU feature. However, we cannot disable it
>> (as of now).
>>
>> We have other CPU features where it is the same case (basically all
>> subfunctions). See kvm_s390_get_processor_subfunc(). We probe them and
>> export them, but support to disable them has never been implemented.
>>
>> On a high level, we could then e.g. deny to start a QEMU guest if APXA
>> is available but has been disabled. (until we know that disabling it
>> actually works - if ever).
>>
>> This helps to catch nasty migration bugs (e.g. APXA suddenly
>> disappearing). Although unlikely, definitely possible.
>>
>>
>> Are there any other AP related facilities that the guest can from now on
>> probe that should also become part of the CPU model?
>>
> To be more precise, shouldn't PQAP(QCI) be handled just like other
> subfunctions? (I remember it should)

When you suggest PQAP(QCI) be handled like other subfunctions, are you
suggesting that there should be a field in struct kvm_s390_vm_cpu_subfunc
with a bit indicating the QCI subfunction is available? The availability
of the QCI subfunction of the PQAP instruction is determined by facilities
bit 12. Is it not enough to export facilities bit 12?

>
> That would imply that there is actually theoretically a way to fake away
> certain AP facilities.
>

2018-08-22 20:57:19

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

On 08/22/2018 07:19 AM, David Hildenbrand wrote:
> On 13.08.2018 23:48, Tony Krowiak wrote:
>> From: Tony Krowiak <[email protected]>
>>
>> Introduces a new CPU model feature and two CPU model
>> facilities to support AP virtualization for KVM guests.
>>
>> CPU model feature:
>>
>> The KVM_S390_VM_CPU_FEAT_AP feature indicates that
>> AP instructions are available on the guest. This
>> feature will be enabled by the kernel only if the AP
>> instructions are installed on the linux host. This feature
>> must be specifically turned on for the KVM guest from
>> userspace to use the VFIO AP device driver for guest
>> access to AP devices.
>>
>> CPU model facilities:
>>
>> 1. AP Query Configuration Information (QCI) facility is installed.
>>
>> This is indicated by setting facilities bit 12 for
>> the guest. The kernel will not enable this facility
>> for the guest if it is not set on the host.
>>
>> If this facility is not set for the KVM guest, then only
>> APQNs with an APQI less than 16 will be used by a Linux
>> guest regardless of the matrix configuration for the virtual
>> machine. This is a limitation of the Linux AP bus.
>>
>> 2. AP Facilities Test facility (APFT) is installed.
>>
>> This is indicated by setting facilities bit 15 for
>> the guest. The kernel will not enable this facility for
>> the guest if it is not set on the host.
>>
>> If this facility is not set for the KVM guest, then no
>> AP devices will be available to the guest regardless of
>> the guest's matrix configuration for the virtual
>> machine. This is a limitation of the Linux AP bus.
>>
>> Signed-off-by: Tony Krowiak <[email protected]>
>> Reviewed-by: Christian Borntraeger <[email protected]>
>> Reviewed-by: Halil Pasic <[email protected]>
>> Tested-by: Michael Mueller <[email protected]>
>> Tested-by: Farhan Ali <[email protected]>
>> Signed-off-by: Christian Borntraeger <[email protected]>
>> ---
>> arch/s390/kvm/kvm-s390.c | 5 +++++
>> arch/s390/tools/gen_facilities.c | 2 ++
>> 2 files changed, 7 insertions(+), 0 deletions(-)
>>
>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
>> index 1e8cb67..d5e04d2 100644
>> --- a/arch/s390/kvm/kvm-s390.c
>> +++ b/arch/s390/kvm/kvm-s390.c
>> @@ -367,6 +367,11 @@ static void kvm_s390_cpu_feat_init(void)
>>
>> if (MACHINE_HAS_ESOP)
>> allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
>> +
>> + /* Check if AP instructions installed on host */
>> + if (ap_instructions_available())
>> + allow_cpu_feat(KVM_S390_VM_CPU_FEAT_AP);
>> +
>> /*
>> * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
>> * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
>> diff --git a/arch/s390/tools/gen_facilities.c b/arch/s390/tools/gen_facilities.c
>> index 90a8c9e..a52290b 100644
>> --- a/arch/s390/tools/gen_facilities.c
>> +++ b/arch/s390/tools/gen_facilities.c
>> @@ -106,6 +106,8 @@ struct facility_def {
>>
>> .name = "FACILITIES_KVM_CPUMODEL",
>> .bits = (int[]){
>> + 12, /* AP Query Configuration Information */
>> + 15, /* AP Facilities Test */
>> -1 /* END */
>> }
>> },
>>
> I really wonder if we should also export the APXA facility.

Given this comment is made within the context of the
FACILITIES_KVM_CPUMODEL I might point out that APXA is not
indicated by a facilities bit. It is indicated by a bit in
the QCI control block returned from the PQAP(QCI)
instruction to indicate that APXA is installed on all CPUs.

> We can probe and allow that CPU feature. However, we cannot disable it
> (as of now).

Given this patch series implements passthrough devices,
the output of the PQAP(QCI) will always be from a real
device - i.e., there will be no way to disable it.

>
> We have other CPU features where it is the same case (basically all
> subfunctions). See kvm_s390_get_processor_subfunc(). We probe them and
> export them, but support to disable them has never been implemented.
>
> On a high level, we could then e.g. deny to start a QEMU guest if APXA
> is available but has been disabled. (until we know that disabling it
> actually works - if ever).
>
> This helps to catch nasty migration bugs (e.g. APXA suddenly
> disappearing). Although unlikely, definitely possible.

Migration of AP devices is not supported by this patch series, so this
should
not be an issue.

>
>
> Are there any other AP related facilities that the guest can from now on
> probe that should also become part of the CPU model?
>

2018-08-22 21:07:07

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

On 08/22/2018 12:57 PM, David Hildenbrand wrote:
>>>> In this case we will have no problem with older guests not having idea
>>>> about APXA.
>>>>
>>>> Would it be a solution?
>>> Any feature the guest sees, should be part of the CPU model. The whole
>>> environment for cpu subfunctions is already in place both in KVM and
>>> QEMU. Only disabling subfunctions in KVM is not implemented yet.
>>>
>>> You can exclude any subfunctions/facilities that are only valid on LPAR
>>> level and cannot be used in some guest either way. (that makes life
>>> sometimes easier)
>>>
>>>
>>> I know that this might sound a little bit complicated, but it really
>>> isn't. Boils down to modifying kvm_s390_cpu_feat_init() and specifying
>>> some features+feature groups in QEMU.
>> OK, we definitively need another patch/patch-set, to handle this.
>> Do you think it can be done in another series since if we always support
>> APXA when we have AP instructions, we already have an indication that
>> APXA exist: the AP facility.
>>
> Please implement the subfunction stuff right away. This will allow to
> handle all future facilities transparently from a kernel POV.

I find your use of the term 'subfunction' confusing here. In the
kvm_s390_cpu_feat_init(void) function, it looks like the
kvm_s390_available_subfunc structure is filled in with bits
returned from CPACF queries of various MSA facilities to indicate
which CPACF functions are supported. APXA is not a subfunction but
a facility that is indicated by a bit returned from the PQAP(QCI)
instruction. If we are to implement this, wouldn't it be done as
a CPU model feature as opposed to a subfunction? Am I
misunderstanding what you are asking for?

>
> Implementing that should be easy - and I don't like gluing features
> together in such a way.
>
> You can always assure that consistent data (e.g. AP + APXA availability)
> is reported from KVM to QEMU.
>
>> Regards,
>> Pierre
>>
>>
>>
>>
>

2018-08-23 07:43:44

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

On 22.08.2018 23:05, Tony Krowiak wrote:
> On 08/22/2018 12:57 PM, David Hildenbrand wrote:
>>>>> In this case we will have no problem with older guests not having idea
>>>>> about APXA.
>>>>>
>>>>> Would it be a solution?
>>>> Any feature the guest sees, should be part of the CPU model. The whole
>>>> environment for cpu subfunctions is already in place both in KVM and
>>>> QEMU. Only disabling subfunctions in KVM is not implemented yet.
>>>>
>>>> You can exclude any subfunctions/facilities that are only valid on LPAR
>>>> level and cannot be used in some guest either way. (that makes life
>>>> sometimes easier)
>>>>
>>>>
>>>> I know that this might sound a little bit complicated, but it really
>>>> isn't. Boils down to modifying kvm_s390_cpu_feat_init() and specifying
>>>> some features+feature groups in QEMU.
>>> OK, we definitively need another patch/patch-set, to handle this.
>>> Do you think it can be done in another series since if we always support
>>> APXA when we have AP instructions, we already have an indication that
>>> APXA exist: the AP facility.
>>>
>> Please implement the subfunction stuff right away. This will allow to
>> handle all future facilities transparently from a kernel POV.
>
> I find your use of the term 'subfunction' confusing here. In the
> kvm_s390_cpu_feat_init(void) function, it looks like the
> kvm_s390_available_subfunc structure is filled in with bits
> returned from CPACF queries of various MSA facilities to indicate
> which CPACF functions are supported. APXA is not a subfunction but
> a facility that is indicated by a bit returned from the PQAP(QCI)
> instruction. If we are to implement this, wouldn't it be done as
> a CPU model feature as opposed to a subfunction? Am I
> misunderstanding what you are asking for?

Yes, "subfunction" is a confusing terminology. (I once called it
subfeature/sufacility, but ended up using subfunction).

From a high level perspective, these are just feature bits - "can I use
feature X" / "is feature X available".

What all of these "query" blocks (MSA, PLO, PQAP(QCI) ...) have in
common is:
- in contrast to STFL(E), they are as a default not modified by the
hypervisor but silently passed through
- dropping one of the bits (e.g. APXA) can break the guest - guest
visible ABI
- any newly added feature/facility in such a block (new HW generation)
should be transparently handled by HW and not require modifications in
the hypervisor - because they are right away presented to the guest.
(unfortunately with minor exceptions - e.g. APXA might be such a
candidate, but that was rather a design error back then)
- there is a way we can overwrite which features are presented to the
guest

The nice thing about that "blob" exported to user space (in contrast to
features) is that it does not have to be fixed up in KVM every time a
new feature/facility is added. As they should be transparently handled.
Only QEMU has to be thought about the new feature - which can be done
right away when introducing the new CPU model.

That's why the natural choice for PQAP(QCI) is also exposing it as
subfunctions, and not as CPU model features (kvm interface).

--

Thanks,

David / dhildenb

2018-08-23 07:46:42

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

On 22.08.2018 22:16, Tony Krowiak wrote:
> On 08/22/2018 07:24 AM, David Hildenbrand wrote:
>> On 22.08.2018 13:19, David Hildenbrand wrote:
>>> On 13.08.2018 23:48, Tony Krowiak wrote:
>>>> From: Tony Krowiak <[email protected]>
>>>>
>>>> Introduces a new CPU model feature and two CPU model
>>>> facilities to support AP virtualization for KVM guests.
>>>>
>>>> CPU model feature:
>>>>
>>>> The KVM_S390_VM_CPU_FEAT_AP feature indicates that
>>>> AP instructions are available on the guest. This
>>>> feature will be enabled by the kernel only if the AP
>>>> instructions are installed on the linux host. This feature
>>>> must be specifically turned on for the KVM guest from
>>>> userspace to use the VFIO AP device driver for guest
>>>> access to AP devices.
>>>>
>>>> CPU model facilities:
>>>>
>>>> 1. AP Query Configuration Information (QCI) facility is installed.
>>>>
>>>> This is indicated by setting facilities bit 12 for
>>>> the guest. The kernel will not enable this facility
>>>> for the guest if it is not set on the host.
>>>>
>>>> If this facility is not set for the KVM guest, then only
>>>> APQNs with an APQI less than 16 will be used by a Linux
>>>> guest regardless of the matrix configuration for the virtual
>>>> machine. This is a limitation of the Linux AP bus.
>>>>
>>>> 2. AP Facilities Test facility (APFT) is installed.
>>>>
>>>> This is indicated by setting facilities bit 15 for
>>>> the guest. The kernel will not enable this facility for
>>>> the guest if it is not set on the host.
>>>>
>>>> If this facility is not set for the KVM guest, then no
>>>> AP devices will be available to the guest regardless of
>>>> the guest's matrix configuration for the virtual
>>>> machine. This is a limitation of the Linux AP bus.
>>>>
>>>> Signed-off-by: Tony Krowiak <[email protected]>
>>>> Reviewed-by: Christian Borntraeger <[email protected]>
>>>> Reviewed-by: Halil Pasic <[email protected]>
>>>> Tested-by: Michael Mueller <[email protected]>
>>>> Tested-by: Farhan Ali <[email protected]>
>>>> Signed-off-by: Christian Borntraeger <[email protected]>
>>>> ---
>>>> arch/s390/kvm/kvm-s390.c | 5 +++++
>>>> arch/s390/tools/gen_facilities.c | 2 ++
>>>> 2 files changed, 7 insertions(+), 0 deletions(-)
>>>>
>>>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
>>>> index 1e8cb67..d5e04d2 100644
>>>> --- a/arch/s390/kvm/kvm-s390.c
>>>> +++ b/arch/s390/kvm/kvm-s390.c
>>>> @@ -367,6 +367,11 @@ static void kvm_s390_cpu_feat_init(void)
>>>>
>>>> if (MACHINE_HAS_ESOP)
>>>> allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
>>>> +
>>>> + /* Check if AP instructions installed on host */
>>>> + if (ap_instructions_available())
>>>> + allow_cpu_feat(KVM_S390_VM_CPU_FEAT_AP);
>>>> +
>>>> /*
>>>> * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
>>>> * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
>>>> diff --git a/arch/s390/tools/gen_facilities.c b/arch/s390/tools/gen_facilities.c
>>>> index 90a8c9e..a52290b 100644
>>>> --- a/arch/s390/tools/gen_facilities.c
>>>> +++ b/arch/s390/tools/gen_facilities.c
>>>> @@ -106,6 +106,8 @@ struct facility_def {
>>>>
>>>> .name = "FACILITIES_KVM_CPUMODEL",
>>>> .bits = (int[]){
>>>> + 12, /* AP Query Configuration Information */
>>>> + 15, /* AP Facilities Test */
>>>> -1 /* END */
>>>> }
>>>> },
>>>>
>>> I really wonder if we should also export the APXA facility.
>>>
>>> We can probe and allow that CPU feature. However, we cannot disable it
>>> (as of now).
>>>
>>> We have other CPU features where it is the same case (basically all
>>> subfunctions). See kvm_s390_get_processor_subfunc(). We probe them and
>>> export them, but support to disable them has never been implemented.
>>>
>>> On a high level, we could then e.g. deny to start a QEMU guest if APXA
>>> is available but has been disabled. (until we know that disabling it
>>> actually works - if ever).
>>>
>>> This helps to catch nasty migration bugs (e.g. APXA suddenly
>>> disappearing). Although unlikely, definitely possible.
>>>
>>>
>>> Are there any other AP related facilities that the guest can from now on
>>> probe that should also become part of the CPU model?
>>>
>> To be more precise, shouldn't PQAP(QCI) be handled just like other
>> subfunctions? (I remember it should)
>
> When you suggest PQAP(QCI) be handled like other subfunctions, are you
> suggesting that there should be a field in struct kvm_s390_vm_cpu_subfunc
> with a bit indicating the QCI subfunction is available? The availability
> of the QCI subfunction of the PQAP instruction is determined by facilities
> bit 12. Is it not enough to export facilities bit 12?

The feature block (128 bit) from PQAP(QCI) should be passed through a
subfunction block to QEMU.

So it is about passing e.g. APXA availability, not QCI itself. (as you
correctly said, that is stfl 12)

--

Thanks,

David / dhildenb

2018-08-23 07:50:17

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

>>>
>> I really wonder if we should also export the APXA facility.
>
> Given this comment is made within the context of the
> FACILITIES_KVM_CPUMODEL I might point out that APXA is not
> indicated by a facilities bit. It is indicated by a bit in
> the QCI control block returned from the PQAP(QCI)
> instruction to indicate that APXA is installed on all CPUs.
>
>> We can probe and allow that CPU feature. However, we cannot disable it
>> (as of now).
>
> Given this patch series implements passthrough devices,
> the output of the PQAP(QCI) will always be from a real
> device - i.e., there will be no way to disable it.
>

see below

>>
>> We have other CPU features where it is the same case (basically all
>> subfunctions). See kvm_s390_get_processor_subfunc(). We probe them and
>> export them, but support to disable them has never been implemented.
>>
>> On a high level, we could then e.g. deny to start a QEMU guest if APXA
>> is available but has been disabled. (until we know that disabling it
>> actually works - if ever).
>>
>> This helps to catch nasty migration bugs (e.g. APXA suddenly
>> disappearing). Although unlikely, definitely possible.
>
> Migration of AP devices is not supported by this patch series, so this
> should
> not be an issue.

Might not be a problem now, but could be later. As I said in a different
reply, the CPU model in QEMU does not care about KVM.

I want the QEMU CPU model and the KVM interfaces to be clean and future
proof. That's why my opinion is to handle PQAP(QCI) just like all the
other "feature blocks" we already have.

--

Thanks,

David / dhildenb

2018-08-23 08:28:17

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

On 23/08/2018 09:48, David Hildenbrand wrote:
>>>>
>>> I really wonder if we should also export the APXA facility.
>>
>> Given this comment is made within the context of the
>> FACILITIES_KVM_CPUMODEL I might point out that APXA is not
>> indicated by a facilities bit. It is indicated by a bit in
>> the QCI control block returned from the PQAP(QCI)
>> instruction to indicate that APXA is installed on all CPUs.
>>
>>> We can probe and allow that CPU feature. However, we cannot disable it
>>> (as of now).
>>
>> Given this patch series implements passthrough devices,
>> the output of the PQAP(QCI) will always be from a real
>> device - i.e., there will be no way to disable it.
>>
>
> see below
>
>>>
>>> We have other CPU features where it is the same case (basically all
>>> subfunctions). See kvm_s390_get_processor_subfunc(). We probe them and
>>> export them, but support to disable them has never been implemented.
>>>
>>> On a high level, we could then e.g. deny to start a QEMU guest if APXA
>>> is available but has been disabled. (until we know that disabling it
>>> actually works - if ever).
>>>
>>> This helps to catch nasty migration bugs (e.g. APXA suddenly
>>> disappearing). Although unlikely, definitely possible.
>>
>> Migration of AP devices is not supported by this patch series, so this
>> should
>> not be an issue.
>
> Might not be a problem now, but could be later. As I said in a different
> reply, the CPU model in QEMU does not care about KVM.
>
> I want the QEMU CPU model and the KVM interfaces to be clean and future
> proof. That's why my opinion is to handle PQAP(QCI) just like all the
> other "feature blocks" we already have.
>

Don't you mix with the TAPQ instruction which has
a T bit to specify interception.
It indeed is not in the subfunction list even it
has a T bit to indicate interception.

TAPQ-t is indicated through the APFT facility.

We can use this bit as an indication of the presence
of APXA, the documentation mention that both are implemented together.

regards,
Pierre

--
Pierre Morel
Linux/KVM/QEMU in Böblingen - Germany

2018-08-23 13:29:09

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

On Thu, 23 Aug 2018 09:48:48 +0200
David Hildenbrand <[email protected]> wrote:

> > Migration of AP devices is not supported by this patch series, so this
> > should
> > not be an issue.
>
> Might not be a problem now, but could be later. As I said in a different
> reply, the CPU model in QEMU does not care about KVM.
>
> I want the QEMU CPU model and the KVM interfaces to be clean and future
> proof. That's why my opinion is to handle PQAP(QCI) just like all the
> other "feature blocks" we already have.

+1 to that sentiment.

It's better to try to get this correct now than having to hack around
should we want to implement things in the future.

2018-08-23 15:57:38

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On 08/22/2018 09:16 PM, Tony Krowiak wrote:
> On 08/22/2018 01:11 PM, Halil Pasic wrote:
>>
>>
>> On 08/22/2018 05:48 PM, Christian Borntraeger wrote:
>>> On 08/22/2018 05:34 PM, Pierre Morel wrote:
>>>> On 22/08/2018 17:11, Christian Borntraeger wrote:
>>>>>
>>>>>
>>>>> On 08/22/2018 01:03 PM, Pierre Morel wrote:
>>>>>>> That's interesting.
>>>>>>>
>>>>>>>> IMHO this quote is quite a half-full half-empty cup one:
>>>>>>>> * it mandates the set of usage domains is a subset of the set
>>>>>>>> of the control domains, but
>>>>>>>> * it speaks of independent controls, namely about the 'usage domain index'
>>>>>>>> and the 'control domain index list' and makes the enforcement of the rule
>>>>>>>> a job of the administrator (instead of codifying it in the controls).
>>>>>>> I'm wondering if a configuration with a usage domain that is not also a
>>>>>>> control domain is rejected outright? Anybody tried that? :)
>>>>>>
>>>>>> Yes, and no it is not.
>>>>>> We can use a queue (usage domain) to a AP card for SHA-512 or RSA without
>>>>>> having to define the queue as a control domain.
>>>>>
>>>>> Huh? My HMC allows to add a domain as
>>>>> - control only domain
>>>>> - control and usage domain.
>>>>>
>>>>> But I am not able to configure a usage-only domain for my LPAR. That seems to match
>>>>> the current code, no?
>>>>>
>>>>
>>>> Yes, it may not be configurable by the HMC but if we start a guest with no control domain it is not a problem to access the hardware through the usage domain.
>>>>
>>>> I tested this a long time ago, but tested again today to be sure on my LPAR.
>>>>
>>>> AFAIU adding a control only domain and a control and usage domain
>>>> allows say:
>>>> control and usage domain 1
>>>> control only domain 2
>>>>
>>>> Allow to send a message to domain 2 using queue 1
>>>>
>>>> Allow also to send a domain modifying message to domain 1 using queue 1
>>>>
>>>> control domain are domain which are controlled
>>>
>>> So you have changed the code to not automatically make a usage domain a
>>> control domain in the bitfield (and you could still use it as a usage
>>> domain). Correct?
>>
>> I tested basically the same yesterday, with the same results.
>>
>>> I think this is probably expected. the "usage implies control" seems to
>>> be a convention implemented by HMC (lpar) and z/VM but millicode offers
>>> the bits to have usage-only domains. As LPAR and z/VM will always enable
>>> any usage-domain to also be a control domain we should do the same.
>>
>> I'm fine either way, but slightly prefer higher level management software
>> and not the kernel accommodating this convention.
>>
>> Please consider a quote from Harald's mail in another sub-thread
>>
>>
>> """
>> ... about control domains
>>
>> Talked with the s390 firmware guys. The convention that the control domain
>> mask is a superset of the usage domain mask is only true for 1st level guests.
>>
>> It is absolutely valid to run a kvm guest with restricted control domain
>> mask bitmap in the CRYCB. It is valid to have an empty control domain mask
>> and the guest should be able to run crypto CPRBs on the usage domain(s) without
>> any problems. However, nobody has tried this.
>> """
>>
>> I'm yet to get an explanation why was this convention established in the first
>> place. And I can not figure it out myself. For me a setup where I know that
>> the domains used by some guest can not be modified by the same guest makes
>> perfect sense. If I try to think in analogies, I kind of compare modification
>> (that is control domain) with write access, and usage (that is usage domain)
>> with read access to, let's say a regular file. For me, all options (rw, r, and w)
>> do make sense, and if I had to pick the one that makes the least sense I would
>> pick write only. The convention is in these terms making read-only illegal. But
>> should 'usage only domains' ever get identified as something somebody wants to do
>> we can just add an attribute for that. So I'm fine either way.
>
> One of the things I suggested in a private conversation with Christian earlier
> today was to provide an additional rw sysfs attribute - a boolean - that indicates
> whether all usage domains should also be control domains. The default could be
> true. This would allow one to configure guests with usage-only domains as well
> as satisfy the convention.
>

I prefer keeping the attributes as they are and adding a new let's say
(un)assign_usage_domain if the need arises over this boolean attribute
that changes how (un)assign_domain works.

Halil

2018-08-23 15:58:52

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

On 08/23/2018 09:44 AM, David Hildenbrand wrote:
> On 22.08.2018 22:16, Tony Krowiak wrote:
>> On 08/22/2018 07:24 AM, David Hildenbrand wrote:
>>> On 22.08.2018 13:19, David Hildenbrand wrote:
>>>> On 13.08.2018 23:48, Tony Krowiak wrote:
>>>>> From: Tony Krowiak <[email protected]>
>>>>>
>>>>> Introduces a new CPU model feature and two CPU model
>>>>> facilities to support AP virtualization for KVM guests.
>>>>>
>>>>> CPU model feature:
>>>>>
>>>>> The KVM_S390_VM_CPU_FEAT_AP feature indicates that
>>>>> AP instructions are available on the guest. This
>>>>> feature will be enabled by the kernel only if the AP
>>>>> instructions are installed on the linux host. This feature
>>>>> must be specifically turned on for the KVM guest from
>>>>> userspace to use the VFIO AP device driver for guest
>>>>> access to AP devices.
>>>>>
>>>>> CPU model facilities:
>>>>>
>>>>> 1. AP Query Configuration Information (QCI) facility is installed.
>>>>>
>>>>> This is indicated by setting facilities bit 12 for
>>>>> the guest. The kernel will not enable this facility
>>>>> for the guest if it is not set on the host.
>>>>>
>>>>> If this facility is not set for the KVM guest, then only
>>>>> APQNs with an APQI less than 16 will be used by a Linux
>>>>> guest regardless of the matrix configuration for the virtual
>>>>> machine. This is a limitation of the Linux AP bus.
>>>>>
>>>>> 2. AP Facilities Test facility (APFT) is installed.
>>>>>
>>>>> This is indicated by setting facilities bit 15 for
>>>>> the guest. The kernel will not enable this facility for
>>>>> the guest if it is not set on the host.
>>>>>
>>>>> If this facility is not set for the KVM guest, then no
>>>>> AP devices will be available to the guest regardless of
>>>>> the guest's matrix configuration for the virtual
>>>>> machine. This is a limitation of the Linux AP bus.
>>>>>
>>>>> Signed-off-by: Tony Krowiak <[email protected]>
>>>>> Reviewed-by: Christian Borntraeger <[email protected]>
>>>>> Reviewed-by: Halil Pasic <[email protected]>
>>>>> Tested-by: Michael Mueller <[email protected]>
>>>>> Tested-by: Farhan Ali <[email protected]>
>>>>> Signed-off-by: Christian Borntraeger <[email protected]>
>>>>> ---
>>>>> arch/s390/kvm/kvm-s390.c | 5 +++++
>>>>> arch/s390/tools/gen_facilities.c | 2 ++
>>>>> 2 files changed, 7 insertions(+), 0 deletions(-)
>>>>>
>>>>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
>>>>> index 1e8cb67..d5e04d2 100644
>>>>> --- a/arch/s390/kvm/kvm-s390.c
>>>>> +++ b/arch/s390/kvm/kvm-s390.c
>>>>> @@ -367,6 +367,11 @@ static void kvm_s390_cpu_feat_init(void)
>>>>>
>>>>> if (MACHINE_HAS_ESOP)
>>>>> allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
>>>>> +
>>>>> + /* Check if AP instructions installed on host */
>>>>> + if (ap_instructions_available())
>>>>> + allow_cpu_feat(KVM_S390_VM_CPU_FEAT_AP);
>>>>> +
>>>>> /*
>>>>> * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
>>>>> * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
>>>>> diff --git a/arch/s390/tools/gen_facilities.c b/arch/s390/tools/gen_facilities.c
>>>>> index 90a8c9e..a52290b 100644
>>>>> --- a/arch/s390/tools/gen_facilities.c
>>>>> +++ b/arch/s390/tools/gen_facilities.c
>>>>> @@ -106,6 +106,8 @@ struct facility_def {
>>>>>
>>>>> .name = "FACILITIES_KVM_CPUMODEL",
>>>>> .bits = (int[]){
>>>>> + 12, /* AP Query Configuration Information */
>>>>> + 15, /* AP Facilities Test */
>>>>> -1 /* END */
>>>>> }
>>>>> },
>>>>>
>>>> I really wonder if we should also export the APXA facility.
>>>>
>>>> We can probe and allow that CPU feature. However, we cannot disable it
>>>> (as of now).
>>>>
>>>> We have other CPU features where it is the same case (basically all
>>>> subfunctions). See kvm_s390_get_processor_subfunc(). We probe them and
>>>> export them, but support to disable them has never been implemented.
>>>>
>>>> On a high level, we could then e.g. deny to start a QEMU guest if APXA
>>>> is available but has been disabled. (until we know that disabling it
>>>> actually works - if ever).
>>>>
>>>> This helps to catch nasty migration bugs (e.g. APXA suddenly
>>>> disappearing). Although unlikely, definitely possible.
>>>>
>>>>
>>>> Are there any other AP related facilities that the guest can from now on
>>>> probe that should also become part of the CPU model?
>>>>
>>> To be more precise, shouldn't PQAP(QCI) be handled just like other
>>> subfunctions? (I remember it should)
>>
>> When you suggest PQAP(QCI) be handled like other subfunctions, are you
>> suggesting that there should be a field in struct kvm_s390_vm_cpu_subfunc
>> with a bit indicating the QCI subfunction is available? The availability
>> of the QCI subfunction of the PQAP instruction is determined by facilities
>> bit 12. Is it not enough to export facilities bit 12?
>
> The feature block (128 bit) from PQAP(QCI) should be passed through a
> subfunction block to QEMU.
>

I'm confused, which 128 bit?

> So it is about passing e.g. APXA availability, not QCI itself. (as you
> correctly said, that is stfl 12)
>

2018-08-23 16:01:19

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On Wed, 22 Aug 2018 15:16:19 -0400
Tony Krowiak <[email protected]> wrote:

> One of the things I suggested in a private conversation with Christian
> earlier
> today was to provide an additional rw sysfs attribute - a boolean - that
> indicates
> whether all usage domains should also be control domains. The default
> could be
> true. This would allow one to configure guests with usage-only domains
> as well
> as satisfy the convention.

Would this additional attribute then control "add usage domains to the
list of control domains automatically", or "don't allow to add a usage
domain if it has not already been added as a control domain"?

One thing I'm still unsure about is how libvirt comes into the picture
here. Will it consume the setting, or actively manipulate it?

[In general, I'm not very clear about how libvirt will interact with the
whole infrastructure...]

2018-08-23 16:03:57

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

On 23.08.2018 12:00, Halil Pasic wrote:
>
>
> On 08/23/2018 09:44 AM, David Hildenbrand wrote:
>> On 22.08.2018 22:16, Tony Krowiak wrote:
>>> On 08/22/2018 07:24 AM, David Hildenbrand wrote:
>>>> On 22.08.2018 13:19, David Hildenbrand wrote:
>>>>> On 13.08.2018 23:48, Tony Krowiak wrote:
>>>>>> From: Tony Krowiak <[email protected]>
>>>>>>
>>>>>> Introduces a new CPU model feature and two CPU model
>>>>>> facilities to support AP virtualization for KVM guests.
>>>>>>
>>>>>> CPU model feature:
>>>>>>
>>>>>> The KVM_S390_VM_CPU_FEAT_AP feature indicates that
>>>>>> AP instructions are available on the guest. This
>>>>>> feature will be enabled by the kernel only if the AP
>>>>>> instructions are installed on the linux host. This feature
>>>>>> must be specifically turned on for the KVM guest from
>>>>>> userspace to use the VFIO AP device driver for guest
>>>>>> access to AP devices.
>>>>>>
>>>>>> CPU model facilities:
>>>>>>
>>>>>> 1. AP Query Configuration Information (QCI) facility is installed.
>>>>>>
>>>>>> This is indicated by setting facilities bit 12 for
>>>>>> the guest. The kernel will not enable this facility
>>>>>> for the guest if it is not set on the host.
>>>>>>
>>>>>> If this facility is not set for the KVM guest, then only
>>>>>> APQNs with an APQI less than 16 will be used by a Linux
>>>>>> guest regardless of the matrix configuration for the virtual
>>>>>> machine. This is a limitation of the Linux AP bus.
>>>>>>
>>>>>> 2. AP Facilities Test facility (APFT) is installed.
>>>>>>
>>>>>> This is indicated by setting facilities bit 15 for
>>>>>> the guest. The kernel will not enable this facility for
>>>>>> the guest if it is not set on the host.
>>>>>>
>>>>>> If this facility is not set for the KVM guest, then no
>>>>>> AP devices will be available to the guest regardless of
>>>>>> the guest's matrix configuration for the virtual
>>>>>> machine. This is a limitation of the Linux AP bus.
>>>>>>
>>>>>> Signed-off-by: Tony Krowiak <[email protected]>
>>>>>> Reviewed-by: Christian Borntraeger <[email protected]>
>>>>>> Reviewed-by: Halil Pasic <[email protected]>
>>>>>> Tested-by: Michael Mueller <[email protected]>
>>>>>> Tested-by: Farhan Ali <[email protected]>
>>>>>> Signed-off-by: Christian Borntraeger <[email protected]>
>>>>>> ---
>>>>>> arch/s390/kvm/kvm-s390.c | 5 +++++
>>>>>> arch/s390/tools/gen_facilities.c | 2 ++
>>>>>> 2 files changed, 7 insertions(+), 0 deletions(-)
>>>>>>
>>>>>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
>>>>>> index 1e8cb67..d5e04d2 100644
>>>>>> --- a/arch/s390/kvm/kvm-s390.c
>>>>>> +++ b/arch/s390/kvm/kvm-s390.c
>>>>>> @@ -367,6 +367,11 @@ static void kvm_s390_cpu_feat_init(void)
>>>>>>
>>>>>> if (MACHINE_HAS_ESOP)
>>>>>> allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
>>>>>> +
>>>>>> + /* Check if AP instructions installed on host */
>>>>>> + if (ap_instructions_available())
>>>>>> + allow_cpu_feat(KVM_S390_VM_CPU_FEAT_AP);
>>>>>> +
>>>>>> /*
>>>>>> * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
>>>>>> * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
>>>>>> diff --git a/arch/s390/tools/gen_facilities.c b/arch/s390/tools/gen_facilities.c
>>>>>> index 90a8c9e..a52290b 100644
>>>>>> --- a/arch/s390/tools/gen_facilities.c
>>>>>> +++ b/arch/s390/tools/gen_facilities.c
>>>>>> @@ -106,6 +106,8 @@ struct facility_def {
>>>>>>
>>>>>> .name = "FACILITIES_KVM_CPUMODEL",
>>>>>> .bits = (int[]){
>>>>>> + 12, /* AP Query Configuration Information */
>>>>>> + 15, /* AP Facilities Test */
>>>>>> -1 /* END */
>>>>>> }
>>>>>> },
>>>>>>
>>>>> I really wonder if we should also export the APXA facility.
>>>>>
>>>>> We can probe and allow that CPU feature. However, we cannot disable it
>>>>> (as of now).
>>>>>
>>>>> We have other CPU features where it is the same case (basically all
>>>>> subfunctions). See kvm_s390_get_processor_subfunc(). We probe them and
>>>>> export them, but support to disable them has never been implemented.
>>>>>
>>>>> On a high level, we could then e.g. deny to start a QEMU guest if APXA
>>>>> is available but has been disabled. (until we know that disabling it
>>>>> actually works - if ever).
>>>>>
>>>>> This helps to catch nasty migration bugs (e.g. APXA suddenly
>>>>> disappearing). Although unlikely, definitely possible.
>>>>>
>>>>>
>>>>> Are there any other AP related facilities that the guest can from now on
>>>>> probe that should also become part of the CPU model?
>>>>>
>>>> To be more precise, shouldn't PQAP(QCI) be handled just like other
>>>> subfunctions? (I remember it should)
>>>
>>> When you suggest PQAP(QCI) be handled like other subfunctions, are you
>>> suggesting that there should be a field in struct kvm_s390_vm_cpu_subfunc
>>> with a bit indicating the QCI subfunction is available? The availability
>>> of the QCI subfunction of the PQAP instruction is determined by facilities
>>> bit 12. Is it not enough to export facilities bit 12?
>>
>> The feature block (128 bit) from PQAP(QCI) should be passed through a
>> subfunction block to QEMU.
>>
>
> I'm confused, which 128 bit?

Me too :) , I was assuming this block to be 128bit, but the qci block
has 128 bytes....

And looking at arch/s390/include/asm/ap.h, there is a lot of information
contained that is definitely not of interest for CPU models...

I wonder if there is somewhere defined which bits are reserved for
future features/facilities, compared to ap masks and such.

This is really hard to understand/plan without access to documentation.

You (Halil, Tony, Pier, ...) should have a look if what I described
related to PQAP(QCI) containing features that should get part of the CPU
model makes sense or not. For now I was thinking that there is some part
inside of QCI that is strictly reserved for facilities/features that we
can use.

--

Thanks,

David / dhildenb

2018-08-23 16:05:15

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On 23/08/2018 11:26, Halil Pasic wrote:
>
>
> On 08/22/2018 09:16 PM, Tony Krowiak wrote:
>> On 08/22/2018 01:11 PM, Halil Pasic wrote:
>>>
>>>
>>> On 08/22/2018 05:48 PM, Christian Borntraeger wrote:
>>>> On 08/22/2018 05:34 PM, Pierre Morel wrote:
>>>>> On 22/08/2018 17:11, Christian Borntraeger wrote:
>>>>>>
>>>>>>
>>>>>> On 08/22/2018 01:03 PM, Pierre Morel wrote:
>>>>>>>> That's interesting.
>>>>>>>>
...
>>>>
>>>> So you have changed the code to not automatically make a usage domain a
>>>> control domain in the bitfield (and you could still use it as a usage
>>>> domain). Correct?

Yes.

>>>
>>> I tested basically the same yesterday, with the same results.
>>>
>>>> I think this is probably expected. the "usage implies control" seems to
>>>> be a convention implemented by HMC (lpar) and z/VM but millicode offers
>>>> the bits to have usage-only domains. As LPAR and z/VM will always
>>>> enable
>>>> any usage-domain to also be a control domain we should do the same.

I think it is the reasonable thing to do.

>>>
>>> I'm fine either way, but slightly prefer higher level management
>>> software
>>> and not the kernel accommodating this convention.

Please, we do not need this in a first version just make it easy
stick with what HMC does.

>>> with read access to, let's say a regular file. For me, all options
>>> (rw, r, and w)
>>> do make sense, and if I had to pick the one that makes the least
>>> sense I would
>>> pick write only. The convention is in these terms making read-only
>>> illegal. But
>>> should 'usage only domains' ever get identified as something somebody
>>> wants to do
>>> we can just add an attribute for that. So I'm fine either way.

We do not need to introduce new features now.

regards,
Pierre

--
Pierre Morel
Linux/KVM/QEMU in Böblingen - Germany

2018-08-23 16:07:15

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On 23/08/2018 12:25, Cornelia Huck wrote:
> On Wed, 22 Aug 2018 15:16:19 -0400
> Tony Krowiak <[email protected]> wrote:
>
>> One of the things I suggested in a private conversation with Christian
>> earlier
>> today was to provide an additional rw sysfs attribute - a boolean - that
>> indicates
>> whether all usage domains should also be control domains. The default
>> could be
>> true. This would allow one to configure guests with usage-only domains
>> as well
>> as satisfy the convention.
>
> Would this additional attribute then control "add usage domains to the
> list of control domains automatically", or "don't allow to add a usage
> domain if it has not already been added as a control domain"?
>
> One thing I'm still unsure about is how libvirt comes into the picture
> here. Will it consume the setting, or actively manipulate it?
>
> [In general, I'm not very clear about how libvirt will interact with the
> whole infrastructure...]
>

When I read you it convince me that it is not wise to change anything
that has been already discuss and could impact the Libvirt.

Regards,
Pierre

--
Pierre Morel
Linux/KVM/QEMU in Böblingen - Germany

2018-08-23 16:09:34

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

On 23.08.2018 13:10, Pierre Morel wrote:
> On 23/08/2018 12:28, David Hildenbrand wrote:
>> On 23.08.2018 12:00, Halil Pasic wrote:
>>>
>>>
>>> On 08/23/2018 09:44 AM, David Hildenbrand wrote:
>>>> On 22.08.2018 22:16, Tony Krowiak wrote:
>>>>> On 08/22/2018 07:24 AM, David Hildenbrand wrote:
>>>>>> On 22.08.2018 13:19, David Hildenbrand wrote:
>>>>>>> On 13.08.2018 23:48, Tony Krowiak wrote:
>>>>>>>> From: Tony Krowiak <[email protected]>
>>>>>>>>
>>>>>>>> Introduces a new CPU model feature and two CPU model
>>>>>>>> facilities to support AP virtualization for KVM guests.
>>>>>>>>
>>>>>>>> CPU model feature:
>>>>>>>>
>>>>>>>> The KVM_S390_VM_CPU_FEAT_AP feature indicates that
>>>>>>>> AP instructions are available on the guest. This
>>>>>>>> feature will be enabled by the kernel only if the AP
>>>>>>>> instructions are installed on the linux host. This feature
>>>>>>>> must be specifically turned on for the KVM guest from
>>>>>>>> userspace to use the VFIO AP device driver for guest
>>>>>>>> access to AP devices.
>>>>>>>>
>>>>>>>> CPU model facilities:
>>>>>>>>
>>>>>>>> 1. AP Query Configuration Information (QCI) facility is installed.
>>>>>>>>
>>>>>>>> This is indicated by setting facilities bit 12 for
>>>>>>>> the guest. The kernel will not enable this facility
>>>>>>>> for the guest if it is not set on the host.
>>>>>>>>
>>>>>>>> If this facility is not set for the KVM guest, then only
>>>>>>>> APQNs with an APQI less than 16 will be used by a Linux
>>>>>>>> guest regardless of the matrix configuration for the virtual
>>>>>>>> machine. This is a limitation of the Linux AP bus.
>>>>>>>>
>>>>>>>> 2. AP Facilities Test facility (APFT) is installed.
>>>>>>>>
>>>>>>>> This is indicated by setting facilities bit 15 for
>>>>>>>> the guest. The kernel will not enable this facility for
>>>>>>>> the guest if it is not set on the host.
>>>>>>>>
>>>>>>>> If this facility is not set for the KVM guest, then no
>>>>>>>> AP devices will be available to the guest regardless of
>>>>>>>> the guest's matrix configuration for the virtual
>>>>>>>> machine. This is a limitation of the Linux AP bus.
>>>>>>>>
>>>>>>>> Signed-off-by: Tony Krowiak <[email protected]>
>>>>>>>> Reviewed-by: Christian Borntraeger <[email protected]>
>>>>>>>> Reviewed-by: Halil Pasic <[email protected]>
>>>>>>>> Tested-by: Michael Mueller <[email protected]>
>>>>>>>> Tested-by: Farhan Ali <[email protected]>
>>>>>>>> Signed-off-by: Christian Borntraeger <[email protected]>
>>>>>>>> ---
>>>>>>>> arch/s390/kvm/kvm-s390.c | 5 +++++
>>>>>>>> arch/s390/tools/gen_facilities.c | 2 ++
>>>>>>>> 2 files changed, 7 insertions(+), 0 deletions(-)
>>>>>>>>
>>>>>>>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
>>>>>>>> index 1e8cb67..d5e04d2 100644
>>>>>>>> --- a/arch/s390/kvm/kvm-s390.c
>>>>>>>> +++ b/arch/s390/kvm/kvm-s390.c
>>>>>>>> @@ -367,6 +367,11 @@ static void kvm_s390_cpu_feat_init(void)
>>>>>>>>
>>>>>>>> if (MACHINE_HAS_ESOP)
>>>>>>>> allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
>>>>>>>> +
>>>>>>>> + /* Check if AP instructions installed on host */
>>>>>>>> + if (ap_instructions_available())
>>>>>>>> + allow_cpu_feat(KVM_S390_VM_CPU_FEAT_AP);
>>>>>>>> +
>>>>>>>> /*
>>>>>>>> * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
>>>>>>>> * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
>>>>>>>> diff --git a/arch/s390/tools/gen_facilities.c b/arch/s390/tools/gen_facilities.c
>>>>>>>> index 90a8c9e..a52290b 100644
>>>>>>>> --- a/arch/s390/tools/gen_facilities.c
>>>>>>>> +++ b/arch/s390/tools/gen_facilities.c
>>>>>>>> @@ -106,6 +106,8 @@ struct facility_def {
>>>>>>>>
>>>>>>>> .name = "FACILITIES_KVM_CPUMODEL",
>>>>>>>> .bits = (int[]){
>>>>>>>> + 12, /* AP Query Configuration Information */
>>>>>>>> + 15, /* AP Facilities Test */
>>>>>>>> -1 /* END */
>>>>>>>> }
>>>>>>>> },
>>>>>>>>
>>>>>>> I really wonder if we should also export the APXA facility.
>>>>>>>
>>>>>>> We can probe and allow that CPU feature. However, we cannot disable it
>>>>>>> (as of now).
>>>>>>>
>>>>>>> We have other CPU features where it is the same case (basically all
>>>>>>> subfunctions). See kvm_s390_get_processor_subfunc(). We probe them and
>>>>>>> export them, but support to disable them has never been implemented.
>>>>>>>
>>>>>>> On a high level, we could then e.g. deny to start a QEMU guest if APXA
>>>>>>> is available but has been disabled. (until we know that disabling it
>>>>>>> actually works - if ever).
>>>>>>>
>>>>>>> This helps to catch nasty migration bugs (e.g. APXA suddenly
>>>>>>> disappearing). Although unlikely, definitely possible.
>>>>>>>
>>>>>>>
>>>>>>> Are there any other AP related facilities that the guest can from now on
>>>>>>> probe that should also become part of the CPU model?
>>>>>>>
>>>>>> To be more precise, shouldn't PQAP(QCI) be handled just like other
>>>>>> subfunctions? (I remember it should)
>>>>>
>>>>> When you suggest PQAP(QCI) be handled like other subfunctions, are you
>>>>> suggesting that there should be a field in struct kvm_s390_vm_cpu_subfunc
>>>>> with a bit indicating the QCI subfunction is available? The availability
>>>>> of the QCI subfunction of the PQAP instruction is determined by facilities
>>>>> bit 12. Is it not enough to export facilities bit 12?
>>>>
>>>> The feature block (128 bit) from PQAP(QCI) should be passed through a
>>>> subfunction block to QEMU.
>>>>
>>>
>>> I'm confused, which 128 bit?
>>
>>
>> Me too :) , I was assuming this block to be 128bit, but the qci block
>> has 128 bytes....
>>
>> And looking at arch/s390/include/asm/ap.h, there is a lot of information
>> contained that is definitely not of interest for CPU models...
>>
>> I wonder if there is somewhere defined which bits are reserved for
>> future features/facilities, compared to ap masks and such.
>>
>> This is really hard to understand/plan without access to documentation.
>>
>> You (Halil, Tony, Pier, ...) should have a look if what I described
>> related to PQAP(QCI) containing features that should get part of the CPU
>> model makes sense or not. For now I was thinking that there is some part
>> inside of QCI that is strictly reserved for facilities/features that we
>> can use.
>>
>
> David,
> I already answered to you on this subject.
>
> First,
> Are you sure you do not mistake QCI for TAPQ which has the t bit
> instruction interception bit as all the instructions you use as
> subfunctions?

Yes, I am pretty sure it is PQAP(QCI), please check with Christian /
architecture documentations.

>
> Second,
> The TAPQ interception bit is exposed through the facility bit 15
> and is documented as being installed when the APXA facility is installed.
>
> If we have the APFT, we have the APXA, problem seems solved to me.

What is apsc, qact, rc8a in the qci blocks? are the facility bits?

>
> Regards,
> Pierre
>

--

Thanks,

David / dhildenb

2018-08-23 16:10:32

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

On 23/08/2018 12:28, David Hildenbrand wrote:
> On 23.08.2018 12:00, Halil Pasic wrote:
>>
>>
>> On 08/23/2018 09:44 AM, David Hildenbrand wrote:
>>> On 22.08.2018 22:16, Tony Krowiak wrote:
>>>> On 08/22/2018 07:24 AM, David Hildenbrand wrote:
>>>>> On 22.08.2018 13:19, David Hildenbrand wrote:
>>>>>> On 13.08.2018 23:48, Tony Krowiak wrote:
>>>>>>> From: Tony Krowiak <[email protected]>
>>>>>>>
>>>>>>> Introduces a new CPU model feature and two CPU model
>>>>>>> facilities to support AP virtualization for KVM guests.
>>>>>>>
>>>>>>> CPU model feature:
>>>>>>>
>>>>>>> The KVM_S390_VM_CPU_FEAT_AP feature indicates that
>>>>>>> AP instructions are available on the guest. This
>>>>>>> feature will be enabled by the kernel only if the AP
>>>>>>> instructions are installed on the linux host. This feature
>>>>>>> must be specifically turned on for the KVM guest from
>>>>>>> userspace to use the VFIO AP device driver for guest
>>>>>>> access to AP devices.
>>>>>>>
>>>>>>> CPU model facilities:
>>>>>>>
>>>>>>> 1. AP Query Configuration Information (QCI) facility is installed.
>>>>>>>
>>>>>>> This is indicated by setting facilities bit 12 for
>>>>>>> the guest. The kernel will not enable this facility
>>>>>>> for the guest if it is not set on the host.
>>>>>>>
>>>>>>> If this facility is not set for the KVM guest, then only
>>>>>>> APQNs with an APQI less than 16 will be used by a Linux
>>>>>>> guest regardless of the matrix configuration for the virtual
>>>>>>> machine. This is a limitation of the Linux AP bus.
>>>>>>>
>>>>>>> 2. AP Facilities Test facility (APFT) is installed.
>>>>>>>
>>>>>>> This is indicated by setting facilities bit 15 for
>>>>>>> the guest. The kernel will not enable this facility for
>>>>>>> the guest if it is not set on the host.
>>>>>>>
>>>>>>> If this facility is not set for the KVM guest, then no
>>>>>>> AP devices will be available to the guest regardless of
>>>>>>> the guest's matrix configuration for the virtual
>>>>>>> machine. This is a limitation of the Linux AP bus.
>>>>>>>
>>>>>>> Signed-off-by: Tony Krowiak <[email protected]>
>>>>>>> Reviewed-by: Christian Borntraeger <[email protected]>
>>>>>>> Reviewed-by: Halil Pasic <[email protected]>
>>>>>>> Tested-by: Michael Mueller <[email protected]>
>>>>>>> Tested-by: Farhan Ali <[email protected]>
>>>>>>> Signed-off-by: Christian Borntraeger <[email protected]>
>>>>>>> ---
>>>>>>> arch/s390/kvm/kvm-s390.c | 5 +++++
>>>>>>> arch/s390/tools/gen_facilities.c | 2 ++
>>>>>>> 2 files changed, 7 insertions(+), 0 deletions(-)
>>>>>>>
>>>>>>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
>>>>>>> index 1e8cb67..d5e04d2 100644
>>>>>>> --- a/arch/s390/kvm/kvm-s390.c
>>>>>>> +++ b/arch/s390/kvm/kvm-s390.c
>>>>>>> @@ -367,6 +367,11 @@ static void kvm_s390_cpu_feat_init(void)
>>>>>>>
>>>>>>> if (MACHINE_HAS_ESOP)
>>>>>>> allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
>>>>>>> +
>>>>>>> + /* Check if AP instructions installed on host */
>>>>>>> + if (ap_instructions_available())
>>>>>>> + allow_cpu_feat(KVM_S390_VM_CPU_FEAT_AP);
>>>>>>> +
>>>>>>> /*
>>>>>>> * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
>>>>>>> * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
>>>>>>> diff --git a/arch/s390/tools/gen_facilities.c b/arch/s390/tools/gen_facilities.c
>>>>>>> index 90a8c9e..a52290b 100644
>>>>>>> --- a/arch/s390/tools/gen_facilities.c
>>>>>>> +++ b/arch/s390/tools/gen_facilities.c
>>>>>>> @@ -106,6 +106,8 @@ struct facility_def {
>>>>>>>
>>>>>>> .name = "FACILITIES_KVM_CPUMODEL",
>>>>>>> .bits = (int[]){
>>>>>>> + 12, /* AP Query Configuration Information */
>>>>>>> + 15, /* AP Facilities Test */
>>>>>>> -1 /* END */
>>>>>>> }
>>>>>>> },
>>>>>>>
>>>>>> I really wonder if we should also export the APXA facility.
>>>>>>
>>>>>> We can probe and allow that CPU feature. However, we cannot disable it
>>>>>> (as of now).
>>>>>>
>>>>>> We have other CPU features where it is the same case (basically all
>>>>>> subfunctions). See kvm_s390_get_processor_subfunc(). We probe them and
>>>>>> export them, but support to disable them has never been implemented.
>>>>>>
>>>>>> On a high level, we could then e.g. deny to start a QEMU guest if APXA
>>>>>> is available but has been disabled. (until we know that disabling it
>>>>>> actually works - if ever).
>>>>>>
>>>>>> This helps to catch nasty migration bugs (e.g. APXA suddenly
>>>>>> disappearing). Although unlikely, definitely possible.
>>>>>>
>>>>>>
>>>>>> Are there any other AP related facilities that the guest can from now on
>>>>>> probe that should also become part of the CPU model?
>>>>>>
>>>>> To be more precise, shouldn't PQAP(QCI) be handled just like other
>>>>> subfunctions? (I remember it should)
>>>>
>>>> When you suggest PQAP(QCI) be handled like other subfunctions, are you
>>>> suggesting that there should be a field in struct kvm_s390_vm_cpu_subfunc
>>>> with a bit indicating the QCI subfunction is available? The availability
>>>> of the QCI subfunction of the PQAP instruction is determined by facilities
>>>> bit 12. Is it not enough to export facilities bit 12?
>>>
>>> The feature block (128 bit) from PQAP(QCI) should be passed through a
>>> subfunction block to QEMU.
>>>
>>
>> I'm confused, which 128 bit?
>
>
> Me too :) , I was assuming this block to be 128bit, but the qci block
> has 128 bytes....
>
> And looking at arch/s390/include/asm/ap.h, there is a lot of information
> contained that is definitely not of interest for CPU models...
>
> I wonder if there is somewhere defined which bits are reserved for
> future features/facilities, compared to ap masks and such.
>
> This is really hard to understand/plan without access to documentation.
>
> You (Halil, Tony, Pier, ...) should have a look if what I described
> related to PQAP(QCI) containing features that should get part of the CPU
> model makes sense or not. For now I was thinking that there is some part
> inside of QCI that is strictly reserved for facilities/features that we
> can use.
>

David,
I already answered to you on this subject.

First,
Are you sure you do not mistake QCI for TAPQ which has the t bit
instruction interception bit as all the instructions you use as
subfunctions?

Second,
The TAPQ interception bit is exposed through the facility bit 15
and is documented as being installed when the APXA facility is installed.

If we have the APFT, we have the APXA, problem seems solved to me.

Regards,
Pierre

--
Pierre Morel
Linux/KVM/QEMU in Böblingen - Germany

2018-08-23 16:13:50

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On Thu, 23 Aug 2018 12:43:42 +0200
Pierre Morel <[email protected]> wrote:

> On 23/08/2018 12:25, Cornelia Huck wrote:
> > On Wed, 22 Aug 2018 15:16:19 -0400
> > Tony Krowiak <[email protected]> wrote:
> >
> >> One of the things I suggested in a private conversation with Christian
> >> earlier
> >> today was to provide an additional rw sysfs attribute - a boolean - that
> >> indicates
> >> whether all usage domains should also be control domains. The default
> >> could be
> >> true. This would allow one to configure guests with usage-only domains
> >> as well
> >> as satisfy the convention.
> >
> > Would this additional attribute then control "add usage domains to the
> > list of control domains automatically", or "don't allow to add a usage
> > domain if it has not already been added as a control domain"?
> >
> > One thing I'm still unsure about is how libvirt comes into the picture
> > here. Will it consume the setting, or actively manipulate it?
> >
> > [In general, I'm not very clear about how libvirt will interact with the
> > whole infrastructure...]
> >
>
> When I read you it convince me that it is not wise to change anything
> that has been already discuss and could impact the Libvirt.

My main point basically was that we should get feedback from a libvirt
POV :) The new attribute may make sense, or not; but I'm really feeling
a bit in the dark with regard to libvirt.

2018-08-23 16:16:46

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On 23/08/2018 13:31, Cornelia Huck wrote:
> On Thu, 23 Aug 2018 12:43:42 +0200
> Pierre Morel <[email protected]> wrote:
>
>> On 23/08/2018 12:25, Cornelia Huck wrote:
>>> On Wed, 22 Aug 2018 15:16:19 -0400
>>> Tony Krowiak <[email protected]> wrote:
>>>
>>>> One of the things I suggested in a private conversation with Christian
>>>> earlier
>>>> today was to provide an additional rw sysfs attribute - a boolean - that
>>>> indicates
>>>> whether all usage domains should also be control domains. The default
>>>> could be
>>>> true. This would allow one to configure guests with usage-only domains
>>>> as well
>>>> as satisfy the convention.
>>>
>>> Would this additional attribute then control "add usage domains to the
>>> list of control domains automatically", or "don't allow to add a usage
>>> domain if it has not already been added as a control domain"?
>>>
>>> One thing I'm still unsure about is how libvirt comes into the picture
>>> here. Will it consume the setting, or actively manipulate it?
>>>
>>> [In general, I'm not very clear about how libvirt will interact with the
>>> whole infrastructure...]
>>>
>>
>> When I read you it convince me that it is not wise to change anything
>> that has been already discuss and could impact the Libvirt.
>
> My main point basically was that we should get feedback from a libvirt
> POV :) The new attribute may make sense, or not; but I'm really feeling
> a bit in the dark with regard to libvirt.
>

Me too, this explains my conservative approach ;)

--
Pierre Morel
Linux/KVM/QEMU in Böblingen - Germany

2018-08-23 16:19:47

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

On 23/08/2018 13:12, David Hildenbrand wrote:
> On 23.08.2018 13:10, Pierre Morel wrote:
>> On 23/08/2018 12:28, David Hildenbrand wrote:
>>> On 23.08.2018 12:00, Halil Pasic wrote:
>>>>
>>>>
>>>> On 08/23/2018 09:44 AM, David Hildenbrand wrote:
>>>>> On 22.08.2018 22:16, Tony Krowiak wrote:
>>>>>> On 08/22/2018 07:24 AM, David Hildenbrand wrote:
>>>>>>> On 22.08.2018 13:19, David Hildenbrand wrote:
>>>>>>>> On 13.08.2018 23:48, Tony Krowiak wrote:
>>>>>>>>> From: Tony Krowiak <[email protected]>
>>>>>>>>>
>>>>>>>>> Introduces a new CPU model feature and two CPU model
>>>>>>>>> facilities to support AP virtualization for KVM guests.
>>>>>>>>>
>>>>>>>>> CPU model feature:
>>>>>>>>>
>>>>>>>>> The KVM_S390_VM_CPU_FEAT_AP feature indicates that
>>>>>>>>> AP instructions are available on the guest. This
>>>>>>>>> feature will be enabled by the kernel only if the AP
>>>>>>>>> instructions are installed on the linux host. This feature
>>>>>>>>> must be specifically turned on for the KVM guest from
>>>>>>>>> userspace to use the VFIO AP device driver for guest
>>>>>>>>> access to AP devices.
>>>>>>>>>
>>>>>>>>> CPU model facilities:
>>>>>>>>>
>>>>>>>>> 1. AP Query Configuration Information (QCI) facility is installed.
>>>>>>>>>
>>>>>>>>> This is indicated by setting facilities bit 12 for
>>>>>>>>> the guest. The kernel will not enable this facility
>>>>>>>>> for the guest if it is not set on the host.
>>>>>>>>>
>>>>>>>>> If this facility is not set for the KVM guest, then only
>>>>>>>>> APQNs with an APQI less than 16 will be used by a Linux
>>>>>>>>> guest regardless of the matrix configuration for the virtual
>>>>>>>>> machine. This is a limitation of the Linux AP bus.
>>>>>>>>>
>>>>>>>>> 2. AP Facilities Test facility (APFT) is installed.
>>>>>>>>>
>>>>>>>>> This is indicated by setting facilities bit 15 for
>>>>>>>>> the guest. The kernel will not enable this facility for
>>>>>>>>> the guest if it is not set on the host.
>>>>>>>>>
>>>>>>>>> If this facility is not set for the KVM guest, then no
>>>>>>>>> AP devices will be available to the guest regardless of
>>>>>>>>> the guest's matrix configuration for the virtual
>>>>>>>>> machine. This is a limitation of the Linux AP bus.
>>>>>>>>>
>>>>>>>>> Signed-off-by: Tony Krowiak <[email protected]>
>>>>>>>>> Reviewed-by: Christian Borntraeger <[email protected]>
>>>>>>>>> Reviewed-by: Halil Pasic <[email protected]>
>>>>>>>>> Tested-by: Michael Mueller <[email protected]>
>>>>>>>>> Tested-by: Farhan Ali <[email protected]>
>>>>>>>>> Signed-off-by: Christian Borntraeger <[email protected]>
>>>>>>>>> ---
>>>>>>>>> arch/s390/kvm/kvm-s390.c | 5 +++++
>>>>>>>>> arch/s390/tools/gen_facilities.c | 2 ++
>>>>>>>>> 2 files changed, 7 insertions(+), 0 deletions(-)
>>>>>>>>>
>>>>>>>>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
>>>>>>>>> index 1e8cb67..d5e04d2 100644
>>>>>>>>> --- a/arch/s390/kvm/kvm-s390.c
>>>>>>>>> +++ b/arch/s390/kvm/kvm-s390.c
>>>>>>>>> @@ -367,6 +367,11 @@ static void kvm_s390_cpu_feat_init(void)
>>>>>>>>>
>>>>>>>>> if (MACHINE_HAS_ESOP)
>>>>>>>>> allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
>>>>>>>>> +
>>>>>>>>> + /* Check if AP instructions installed on host */
>>>>>>>>> + if (ap_instructions_available())
>>>>>>>>> + allow_cpu_feat(KVM_S390_VM_CPU_FEAT_AP);
>>>>>>>>> +
>>>>>>>>> /*
>>>>>>>>> * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
>>>>>>>>> * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
>>>>>>>>> diff --git a/arch/s390/tools/gen_facilities.c b/arch/s390/tools/gen_facilities.c
>>>>>>>>> index 90a8c9e..a52290b 100644
>>>>>>>>> --- a/arch/s390/tools/gen_facilities.c
>>>>>>>>> +++ b/arch/s390/tools/gen_facilities.c
>>>>>>>>> @@ -106,6 +106,8 @@ struct facility_def {
>>>>>>>>>
>>>>>>>>> .name = "FACILITIES_KVM_CPUMODEL",
>>>>>>>>> .bits = (int[]){
>>>>>>>>> + 12, /* AP Query Configuration Information */
>>>>>>>>> + 15, /* AP Facilities Test */
>>>>>>>>> -1 /* END */
>>>>>>>>> }
>>>>>>>>> },
>>>>>>>>>
>>>>>>>> I really wonder if we should also export the APXA facility.
>>>>>>>>
>>>>>>>> We can probe and allow that CPU feature. However, we cannot disable it
>>>>>>>> (as of now).
>>>>>>>>
>>>>>>>> We have other CPU features where it is the same case (basically all
>>>>>>>> subfunctions). See kvm_s390_get_processor_subfunc(). We probe them and
>>>>>>>> export them, but support to disable them has never been implemented.
>>>>>>>>
>>>>>>>> On a high level, we could then e.g. deny to start a QEMU guest if APXA
>>>>>>>> is available but has been disabled. (until we know that disabling it
>>>>>>>> actually works - if ever).
>>>>>>>>
>>>>>>>> This helps to catch nasty migration bugs (e.g. APXA suddenly
>>>>>>>> disappearing). Although unlikely, definitely possible.
>>>>>>>>
>>>>>>>>
>>>>>>>> Are there any other AP related facilities that the guest can from now on
>>>>>>>> probe that should also become part of the CPU model?
>>>>>>>>
>>>>>>> To be more precise, shouldn't PQAP(QCI) be handled just like other
>>>>>>> subfunctions? (I remember it should)
>>>>>>
>>>>>> When you suggest PQAP(QCI) be handled like other subfunctions, are you
>>>>>> suggesting that there should be a field in struct kvm_s390_vm_cpu_subfunc
>>>>>> with a bit indicating the QCI subfunction is available? The availability
>>>>>> of the QCI subfunction of the PQAP instruction is determined by facilities
>>>>>> bit 12. Is it not enough to export facilities bit 12?
>>>>>
>>>>> The feature block (128 bit) from PQAP(QCI) should be passed through a
>>>>> subfunction block to QEMU.
>>>>>
>>>>
>>>> I'm confused, which 128 bit?
>>>
>>>
>>> Me too :) , I was assuming this block to be 128bit, but the qci block
>>> has 128 bytes....
>>>
>>> And looking at arch/s390/include/asm/ap.h, there is a lot of information
>>> contained that is definitely not of interest for CPU models...
>>>
>>> I wonder if there is somewhere defined which bits are reserved for
>>> future features/facilities, compared to ap masks and such.
>>>
>>> This is really hard to understand/plan without access to documentation.
>>>
>>> You (Halil, Tony, Pier, ...) should have a look if what I described
>>> related to PQAP(QCI) containing features that should get part of the CPU
>>> model makes sense or not. For now I was thinking that there is some part
>>> inside of QCI that is strictly reserved for facilities/features that we
>>> can use.
>>>
>>
>> David,
>> I already answered to you on this subject.
>>
>> First,
>> Are you sure you do not mistake QCI for TAPQ which has the t bit
>> instruction interception bit as all the instructions you use as
>> subfunctions?
>
> Yes, I am pretty sure it is PQAP(QCI), please check with Christian /
> architecture documentations.

OK.

>
>>
>> Second,
>> The TAPQ interception bit is exposed through the facility bit 15
>> and is documented as being installed when the APXA facility is installed.
>>
>> If we have the APFT, we have the APXA, problem seems solved to me.

hum. wrong, sorry, the assertion is in the wrong way...

>
> What is apsc, qact, rc8a in the qci blocks? are the facility bits?

Yes, facility bits concerning the AP instructions

>
>>
>> Regards,
>> Pierre
>>
>
>

--
Pierre Morel
Linux/KVM/QEMU in Böblingen - Germany

2018-08-23 16:22:59

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

On 08/23/2018 02:47 PM, Pierre Morel wrote:
> On 23/08/2018 13:12, David Hildenbrand wrote:
[..]
>>>>>
>>>>> I'm confused, which 128 bit?
>>>>
>>>>
>>>> Me too :) , I was assuming this block to be 128bit, but the qci block
>>>> has 128 bytes....
>>>>
>>>> And looking at arch/s390/include/asm/ap.h, there is a lot of information
>>>> contained that is definitely not of interest for CPU models...
>>>>
>>>> I wonder if there is somewhere defined which bits are reserved for
>>>> future features/facilities, compared to ap masks and such.
>>>>
>>>> This is really hard to understand/plan without access to documentation.
>>>>
>>>> You (Halil, Tony, Pier, ...) should have a look if what I described
>>>> related to PQAP(QCI) containing features that should get part of the CPU
>>>> model makes sense or not. For now I was thinking that there is some part
>>>> inside of QCI that is strictly reserved for facilities/features that we
>>>> can use.

No there is no such part. The architecture documentation is quite confusing
with some aspects (e.g. persistence) of how exactly some of these features
work and are indicated. I'm having a hard time finding my opinion. I may
end up asking some questions later, but for now i have to think first.

Just one hint. There is a programming note stating that if bit 2 of the
QCI block is one there is at least one AP card in the machine that actually
has APXA installed.

I read the architecture so that the APXA has a 'cpu part' (if we are
doing APXA the cpu can't spec exception on certain bits not being zor9)
and a 'card(s) part'.

Since the stuff seems quite difficult to sort out properly, I ask myself
are there real problems we must solve?

This ultimately seems to be about the migration, right? You say 'This helps
to catch nasty migration bugs (e.g. APXA suddenly disappearing).' at the very
beginning of the discussion. Yes, we don't have to have an vfio_ap device,
he guest can and will start looking for AP resources if
only the cpu model features installed. So the guest could observe
a disappearing APXA, but I don't think that would lead to problems (with
Linux at least).

And there ain't much AP a guest can sanely do without if no AP resources
are there.

I would really prefer not rushing a solution if we don't have to.

>
>
>>
>> What is apsc, qact, rc8a in the qci blocks? are the facility bits?
>
> Yes, facility bits concerning the AP instructions
>

According to the current AR document rc8a ain't a facility but bits
0-2 and 4-7 kind of are.

Regards,
Halil

2018-08-23 16:24:30

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

On 23.08.2018 15:22, Halil Pasic wrote:
>
>
> On 08/23/2018 02:47 PM, Pierre Morel wrote:
>> On 23/08/2018 13:12, David Hildenbrand wrote:
> [..]
>>>>>>
>>>>>> I'm confused, which 128 bit?
>>>>>
>>>>>
>>>>> Me too :) , I was assuming this block to be 128bit, but the qci block
>>>>> has 128 bytes....
>>>>>
>>>>> And looking at arch/s390/include/asm/ap.h, there is a lot of information
>>>>> contained that is definitely not of interest for CPU models...
>>>>>
>>>>> I wonder if there is somewhere defined which bits are reserved for
>>>>> future features/facilities, compared to ap masks and such.
>>>>>
>>>>> This is really hard to understand/plan without access to documentation.
>>>>>
>>>>> You (Halil, Tony, Pier, ...) should have a look if what I described
>>>>> related to PQAP(QCI) containing features that should get part of the CPU
>>>>> model makes sense or not. For now I was thinking that there is some part
>>>>> inside of QCI that is strictly reserved for facilities/features that we
>>>>> can use.
>
> No there is no such part. The architecture documentation is quite confusing
> with some aspects (e.g. persistence) of how exactly some of these features
> work and are indicated. I'm having a hard time finding my opinion. I may
> end up asking some questions later, but for now i have to think first.
>
> Just one hint. There is a programming note stating that if bit 2 of the
> QCI block is one there is at least one AP card in the machine that actually
> has APXA installed.
>
> I read the architecture so that the APXA has a 'cpu part' (if we are
> doing APXA the cpu can't spec exception on certain bits not being zor9)
> and a 'card(s) part'.
>
> Since the stuff seems quite difficult to sort out properly, I ask myself
> are there real problems we must solve?
>
> This ultimately seems to be about the migration, right? You say 'This helps
> to catch nasty migration bugs (e.g. APXA suddenly disappearing).' at the very
> beginning of the discussion. Yes, we don't have to have an vfio_ap device,
> he guest can and will start looking for AP resources if
> only the cpu model features installed. So the guest could observe
> a disappearing APXA, but I don't think that would lead to problems (with
> Linux at least).
>
> And there ain't much AP a guest can sanely do without if no AP resources
> are there.
>
> I would really prefer not rushing a solution if we don't have to.
>
>>
>>
>>>
>>> What is apsc, qact, rc8a in the qci blocks? are the facility bits?
>>
>> Yes, facility bits concerning the AP instructions
>>
>
> According to the current AR document rc8a ain't a facility but bits
> 0-2 and 4-7 kind of are.
>

Easy ( :) ) answer. Everything that is the CPU part should get into the
CPU model. Everything that is AP specific not. If APXA is not a CPU
facility, fine with me to leave it out.

Ack to not rushing, but also ack to not leaving out important things.
Ack that this stuff is hard to ficure out.

> Regards,
> Halil
>

--

Thanks,

David / dhildenb

2018-08-23 16:26:32

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On 08/23/2018 06:25 AM, Cornelia Huck wrote:
> On Wed, 22 Aug 2018 15:16:19 -0400
> Tony Krowiak <[email protected]> wrote:
>
>> One of the things I suggested in a private conversation with Christian
>> earlier
>> today was to provide an additional rw sysfs attribute - a boolean - that
>> indicates
>> whether all usage domains should also be control domains. The default
>> could be
>> true. This would allow one to configure guests with usage-only domains
>> as well
>> as satisfy the convention.
> Would this additional attribute then control "add usage domains to the
> list of control domains automatically", or "don't allow to add a usage
> domain if it has not already been added as a control domain"?

It was just a proposal that wasn't really discussed at all, but this
attribute would add usage domains to the list of control domains
automatically if set to one. That would be the default behavior which
would be turned off by manually setting it to zero.

>
> One thing I'm still unsure about is how libvirt comes into the picture
> here. Will it consume the setting, or actively manipulate it?

libvirt is not in the picture. The only thing libvirt does is identify
which mediated matrix device (i.e., UUID) is to be used by the guest.

>
> [In general, I'm not very clear about how libvirt will interact with the
> whole infrastructure...]

The libvirt domain XML will only specify the UUID of the matrix device
to be used by the guest. Consequently, libvirt will insert the device
statement for the vfio-ap device into the QEMU command line:

-device vfio-ap,syfsdev={path-to-mdev}

All configuration of the mediated device, including the proposed attribute
above, is handled via sysfs attributes.

>

2018-08-23 16:29:31

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

On 23/08/2018 15:38, David Hildenbrand wrote:
> On 23.08.2018 15:22, Halil Pasic wrote:
>>
>>
>> On 08/23/2018 02:47 PM, Pierre Morel wrote:
>>> On 23/08/2018 13:12, David Hildenbrand wrote:
>> [..]
>>>>>>>
>>>>>>> I'm confused, which 128 bit?
>>>>>>
>>>>>>
>>>>>> Me too :) , I was assuming this block to be 128bit, but the qci block
>>>>>> has 128 bytes....
>>>>>>
>>>>>> And looking at arch/s390/include/asm/ap.h, there is a lot of information
>>>>>> contained that is definitely not of interest for CPU models...
>>>>>>
>>>>>> I wonder if there is somewhere defined which bits are reserved for
>>>>>> future features/facilities, compared to ap masks and such.
>>>>>>
>>>>>> This is really hard to understand/plan without access to documentation.
>>>>>>
>>>>>> You (Halil, Tony, Pier, ...) should have a look if what I described
>>>>>> related to PQAP(QCI) containing features that should get part of the CPU
>>>>>> model makes sense or not. For now I was thinking that there is some part
>>>>>> inside of QCI that is strictly reserved for facilities/features that we
>>>>>> can use.
>>
>> No there is no such part. The architecture documentation is quite confusing
>> with some aspects (e.g. persistence) of how exactly some of these features
>> work and are indicated. I'm having a hard time finding my opinion. I may
>> end up asking some questions later, but for now i have to think first.
>>
>> Just one hint. There is a programming note stating that if bit 2 of the
>> QCI block is one there is at least one AP card in the machine that actually
>> has APXA installed.
>>
>> I read the architecture so that the APXA has a 'cpu part' (if we are
>> doing APXA the cpu can't spec exception on certain bits not being zor9)
>> and a 'card(s) part'.
>>
>> Since the stuff seems quite difficult to sort out properly, I ask myself
>> are there real problems we must solve?
>>
>> This ultimately seems to be about the migration, right? You say 'This helps
>> to catch nasty migration bugs (e.g. APXA suddenly disappearing).' at the very
>> beginning of the discussion. Yes, we don't have to have an vfio_ap device,
>> he guest can and will start looking for AP resources if
>> only the cpu model features installed. So the guest could observe
>> a disappearing APXA, but I don't think that would lead to problems (with
>> Linux at least).
>>
>> And there ain't much AP a guest can sanely do without if no AP resources
>> are there.
>>
>> I would really prefer not rushing a solution if we don't have to.
>>
>>>
>>>
>>>>
>>>> What is apsc, qact, rc8a in the qci blocks? are the facility bits?
>>>
>>> Yes, facility bits concerning the AP instructions
>>>
>>
>> According to the current AR document rc8a ain't a facility but bits
>> 0-2 and 4-7 kind of are.
>>
>
> Easy ( :) ) answer. Everything that is the CPU part should get into the
> CPU model. Everything that is AP specific not. If APXA is not a CPU
> facility, fine with me to leave it out.
>
> Ack to not rushing, but also ack to not leaving out important things.
> Ack that this stuff is hard to ficure out.

APXA is not a CPU part, it is a machine part (SIE) and a AP part (QCI,TAPQ),
it has no influence on CPU instructions but on the AP instructions.
Consequently, if I understood the definition correctly, it should not go
in the CPU model.

Regards,
Pierre

--
Pierre Morel
Linux/KVM/QEMU in Böblingen - Germany

2018-08-23 17:37:18

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

On 08/23/2018 10:59 AM, Pierre Morel wrote:
> On 23/08/2018 15:38, David Hildenbrand wrote:
>> On 23.08.2018 15:22, Halil Pasic wrote:
>>>
>>>
>>> On 08/23/2018 02:47 PM, Pierre Morel wrote:
>>>> On 23/08/2018 13:12, David Hildenbrand wrote:
>>> [..]
>>>>>>>>
>>>>>>>> I'm confused, which 128 bit?
>>>>>>>
>>>>>>>
>>>>>>> Me too :) , I was assuming this block to be 128bit, but the qci
>>>>>>> block
>>>>>>> has 128 bytes....
>>>>>>>
>>>>>>> And looking at arch/s390/include/asm/ap.h, there is a lot of
>>>>>>> information
>>>>>>> contained that is definitely not of interest for CPU models...
>>>>>>>
>>>>>>> I wonder if there is somewhere defined which bits are reserved for
>>>>>>> future features/facilities, compared to ap masks and such.
>>>>>>>
>>>>>>> This is really hard to understand/plan without access to
>>>>>>> documentation.
>>>>>>>
>>>>>>> You (Halil, Tony, Pier, ...) should have a look if what I described
>>>>>>> related to PQAP(QCI) containing features that should get part of
>>>>>>> the CPU
>>>>>>> model makes sense or not. For now I was thinking that there is
>>>>>>> some part
>>>>>>> inside of QCI that is strictly reserved for facilities/features
>>>>>>> that we
>>>>>>> can use.
>>>
>>> No there is no such part. The architecture documentation is quite
>>> confusing
>>> with some aspects (e.g. persistence) of how exactly some of these
>>> features
>>> work and are indicated. I'm having a hard time finding my opinion. I
>>> may
>>> end up asking some questions later, but for now i have to think first.
>>>
>>> Just one hint. There is a programming note stating that if bit 2 of the
>>> QCI block is one there is at least one AP card in the machine that
>>> actually
>>> has APXA installed.
>>>
>>> I read the architecture so that the APXA has a 'cpu part' (if we are
>>> doing APXA the cpu can't spec exception on certain bits not being zor9)
>>> and a 'card(s) part'.
>>>
>>> Since the stuff seems quite difficult to sort out properly, I ask
>>> myself
>>> are there real problems we must solve?
>>>
>>> This ultimately seems to be about the migration, right? You say
>>> 'This helps
>>> to catch nasty migration bugs (e.g. APXA suddenly disappearing).' at
>>> the very
>>> beginning of the discussion. Yes, we don't have to have an vfio_ap
>>> device,
>>> he guest can and will start looking for AP resources if
>>> only the cpu model features installed. So the guest could observe
>>> a disappearing APXA, but I don't think that would lead to problems
>>> (with
>>> Linux at least).
>>>
>>> And there ain't much AP a guest can sanely do without if no AP
>>> resources
>>> are there.
>>>
>>> I would really prefer not rushing a solution if we don't have to.
>>>
>>>>
>>>>
>>>>>
>>>>> What is apsc, qact, rc8a in the qci blocks? are the facility bits?
>>>>
>>>> Yes, facility bits concerning the AP instructions
>>>>
>>>
>>> According to the current AR document rc8a ain't a facility but bits
>>> 0-2 and 4-7 kind of are.
>>>
>>
>> Easy ( :) ) answer. Everything that is the CPU part should get into the
>> CPU model. Everything that is AP specific not. If APXA is not a CPU
>> facility, fine with me to leave it out.
>>
>> Ack to not rushing, but also ack to not leaving out important things.
>> Ack that this stuff is hard to ficure out.
>
> APXA is not a CPU part, it is a machine part (SIE) and a AP part
> (QCI,TAPQ),
> it has no influence on CPU instructions but on the AP instructions.
> Consequently, if I understood the definition correctly, it should not
> go in the CPU model.

The APXA bit returned via the PQAP(QCI) instruction indicates the APXA
facility is
installed in the CPUs of the configuration. This means that the facility is
installed in one or more adjunct processors but not necessarily all.
Given that
it indicates a CPU property, maybe it does belong in the CPU model?

>
> Regards,
> Pierre
>
>
>
>

2018-08-23 17:42:36

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

On 23.08.2018 19:35, Tony Krowiak wrote:
> On 08/23/2018 10:59 AM, Pierre Morel wrote:
>> On 23/08/2018 15:38, David Hildenbrand wrote:
>>> On 23.08.2018 15:22, Halil Pasic wrote:
>>>>
>>>>
>>>> On 08/23/2018 02:47 PM, Pierre Morel wrote:
>>>>> On 23/08/2018 13:12, David Hildenbrand wrote:
>>>> [..]
>>>>>>>>>
>>>>>>>>> I'm confused, which 128 bit?
>>>>>>>>
>>>>>>>>
>>>>>>>> Me too :) , I was assuming this block to be 128bit, but the qci
>>>>>>>> block
>>>>>>>> has 128 bytes....
>>>>>>>>
>>>>>>>> And looking at arch/s390/include/asm/ap.h, there is a lot of
>>>>>>>> information
>>>>>>>> contained that is definitely not of interest for CPU models...
>>>>>>>>
>>>>>>>> I wonder if there is somewhere defined which bits are reserved for
>>>>>>>> future features/facilities, compared to ap masks and such.
>>>>>>>>
>>>>>>>> This is really hard to understand/plan without access to
>>>>>>>> documentation.
>>>>>>>>
>>>>>>>> You (Halil, Tony, Pier, ...) should have a look if what I described
>>>>>>>> related to PQAP(QCI) containing features that should get part of
>>>>>>>> the CPU
>>>>>>>> model makes sense or not. For now I was thinking that there is
>>>>>>>> some part
>>>>>>>> inside of QCI that is strictly reserved for facilities/features
>>>>>>>> that we
>>>>>>>> can use.
>>>>
>>>> No there is no such part. The architecture documentation is quite
>>>> confusing
>>>> with some aspects (e.g. persistence) of how exactly some of these
>>>> features
>>>> work and are indicated. I'm having a hard time finding my opinion. I
>>>> may
>>>> end up asking some questions later, but for now i have to think first.
>>>>
>>>> Just one hint. There is a programming note stating that if bit 2 of the
>>>> QCI block is one there is at least one AP card in the machine that
>>>> actually
>>>> has APXA installed.
>>>>
>>>> I read the architecture so that the APXA has a 'cpu part' (if we are
>>>> doing APXA the cpu can't spec exception on certain bits not being zor9)
>>>> and a 'card(s) part'.
>>>>
>>>> Since the stuff seems quite difficult to sort out properly, I ask
>>>> myself
>>>> are there real problems we must solve?
>>>>
>>>> This ultimately seems to be about the migration, right? You say
>>>> 'This helps
>>>> to catch nasty migration bugs (e.g. APXA suddenly disappearing).' at
>>>> the very
>>>> beginning of the discussion. Yes, we don't have to have an vfio_ap
>>>> device,
>>>> he guest can and will start looking for AP resources if
>>>> only the cpu model features installed. So the guest could observe
>>>> a disappearing APXA, but I don't think that would lead to problems
>>>> (with
>>>> Linux at least).
>>>>
>>>> And there ain't much AP a guest can sanely do without if no AP
>>>> resources
>>>> are there.
>>>>
>>>> I would really prefer not rushing a solution if we don't have to.
>>>>
>>>>>
>>>>>
>>>>>>
>>>>>> What is apsc, qact, rc8a in the qci blocks? are the facility bits?
>>>>>
>>>>> Yes, facility bits concerning the AP instructions
>>>>>
>>>>
>>>> According to the current AR document rc8a ain't a facility but bits
>>>> 0-2 and 4-7 kind of are.
>>>>
>>>
>>> Easy ( :) ) answer. Everything that is the CPU part should get into the
>>> CPU model. Everything that is AP specific not. If APXA is not a CPU
>>> facility, fine with me to leave it out.
>>>
>>> Ack to not rushing, but also ack to not leaving out important things.
>>> Ack that this stuff is hard to ficure out.
>>
>> APXA is not a CPU part, it is a machine part (SIE) and a AP part
>> (QCI,TAPQ),
>> it has no influence on CPU instructions but on the AP instructions.
>> Consequently, if I understood the definition correctly, it should not
>> go in the CPU model.
>
> The APXA bit returned via the PQAP(QCI) instruction indicates the APXA
> facility is
> installed in the CPUs of the configuration. This means that the facility is
> installed in one or more adjunct processors but not necessarily all.
> Given that
> it indicates a CPU property, maybe it does belong in the CPU model?
>

Hmmm, I tend to agree - especially as it affects SIE behavior. But as
this is not a feature block (compared to what I thought), this clould be
model as a CPU feature like AP.

What about the other facilities? Do they smell more like AP card
specific stuff?

--

Thanks,

David / dhildenb

2018-08-24 10:31:44

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

On 08/23/2018 07:40 PM, David Hildenbrand wrote:
> On 23.08.2018 19:35, Tony Krowiak wrote:
>> On 08/23/2018 10:59 AM, Pierre Morel wrote:
>>> On 23/08/2018 15:38, David Hildenbrand wrote:
>>>> On 23.08.2018 15:22, Halil Pasic wrote:
>>>>>
>>>>>
>>>>> On 08/23/2018 02:47 PM, Pierre Morel wrote:
>>>>>> On 23/08/2018 13:12, David Hildenbrand wrote:
>>>>> [..]
>>>>>>>>>>
>>>>>>>>>> I'm confused, which 128 bit?
>>>>>>>>>
>>>>>>>>>
>>>>>>>>> Me too :) , I was assuming this block to be 128bit, but the qci
>>>>>>>>> block
>>>>>>>>> has 128 bytes....
>>>>>>>>>
>>>>>>>>> And looking at arch/s390/include/asm/ap.h, there is a lot of
>>>>>>>>> information
>>>>>>>>> contained that is definitely not of interest for CPU models...
>>>>>>>>>
>>>>>>>>> I wonder if there is somewhere defined which bits are reserved for
>>>>>>>>> future features/facilities, compared to ap masks and such.
>>>>>>>>>
>>>>>>>>> This is really hard to understand/plan without access to
>>>>>>>>> documentation.
>>>>>>>>>
>>>>>>>>> You (Halil, Tony, Pier, ...) should have a look if what I described
>>>>>>>>> related to PQAP(QCI) containing features that should get part of
>>>>>>>>> the CPU
>>>>>>>>> model makes sense or not. For now I was thinking that there is
>>>>>>>>> some part
>>>>>>>>> inside of QCI that is strictly reserved for facilities/features
>>>>>>>>> that we
>>>>>>>>> can use.
>>>>>
>>>>> No there is no such part. The architecture documentation is quite
>>>>> confusing
>>>>> with some aspects (e.g. persistence) of how exactly some of these
>>>>> features
>>>>> work and are indicated. I'm having a hard time finding my opinion. I
>>>>> may
>>>>> end up asking some questions later, but for now i have to think first.
>>>>>
>>>>> Just one hint. There is a programming note stating that if bit 2 of the
>>>>> QCI block is one there is at least one AP card in the machine that
>>>>> actually
>>>>> has APXA installed.
>>>>>
>>>>> I read the architecture so that the APXA has a 'cpu part' (if we are
>>>>> doing APXA the cpu can't spec exception on certain bits not being zor9)
>>>>> and a 'card(s) part'.
>>>>>
>>>>> Since the stuff seems quite difficult to sort out properly, I ask
>>>>> myself
>>>>> are there real problems we must solve?
>>>>>
>>>>> This ultimately seems to be about the migration, right? You say
>>>>> 'This helps
>>>>> to catch nasty migration bugs (e.g. APXA suddenly disappearing).' at
>>>>> the very
>>>>> beginning of the discussion. Yes, we don't have to have an vfio_ap
>>>>> device,
>>>>> he guest can and will start looking for AP resources if
>>>>> only the cpu model features installed. So the guest could observe
>>>>> a disappearing APXA, but I don't think that would lead to problems
>>>>> (with
>>>>> Linux at least).
>>>>>
>>>>> And there ain't much AP a guest can sanely do without if no AP
>>>>> resources
>>>>> are there.
>>>>>
>>>>> I would really prefer not rushing a solution if we don't have to.
>>>>>
>>>>>>
>>>>>>
>>>>>>>
>>>>>>> What is apsc, qact, rc8a in the qci blocks? are the facility bits?
>>>>>>
>>>>>> Yes, facility bits concerning the AP instructions
>>>>>>
>>>>>
>>>>> According to the current AR document rc8a ain't a facility but bits
>>>>> 0-2 and 4-7 kind of are.
>>>>>
>>>>
>>>> Easy ( :) ) answer. Everything that is the CPU part should get into the
>>>> CPU model. Everything that is AP specific not. If APXA is not a CPU
>>>> facility, fine with me to leave it out.
>>>>
>>>> Ack to not rushing, but also ack to not leaving out important things.
>>>> Ack that this stuff is hard to ficure out.
>>>
>>> APXA is not a CPU part, it is a machine part (SIE) and a AP part
>>> (QCI,TAPQ),
>>> it has no influence on CPU instructions but on the AP instructions.
>>> Consequently, if I understood the definition correctly, it should not
>>> go in the CPU model.
>>
>> The APXA bit returned via the PQAP(QCI) instruction indicates the APXA
>> facility is
>> installed in the CPUs of the configuration. This means that the facility is
>> installed in one or more adjunct processors but not necessarily all.
>> Given that
>> it indicates a CPU property, maybe it does belong in the CPU model?
>>
>
> Hmmm, I tend to agree - especially as it affects SIE behavior. But as
> this is not a feature block (compared to what I thought), this clould be
> model as a CPU feature like AP.
>

There is certainly a CPU aspect to APXA: before APXA the APQN had to
have zeros in certain bits (otherwise specification exception). When
running with APXA we have a guarantee that there won't be any
specification exception flying because such an bit is set. The interesting
question is, is APXA constant let's say as long as an LPAR partition is
activated?

Regards,
Halil

2018-08-27 08:36:03

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On Thu, 23 Aug 2018 10:16:59 -0400
Tony Krowiak <[email protected]> wrote:

> On 08/23/2018 06:25 AM, Cornelia Huck wrote:
> > On Wed, 22 Aug 2018 15:16:19 -0400
> > Tony Krowiak <[email protected]> wrote:
> >
> >> One of the things I suggested in a private conversation with Christian
> >> earlier
> >> today was to provide an additional rw sysfs attribute - a boolean - that
> >> indicates
> >> whether all usage domains should also be control domains. The default
> >> could be
> >> true. This would allow one to configure guests with usage-only domains
> >> as well
> >> as satisfy the convention.
> > Would this additional attribute then control "add usage domains to the
> > list of control domains automatically", or "don't allow to add a usage
> > domain if it has not already been added as a control domain"?
>
> It was just a proposal that wasn't really discussed at all, but this
> attribute would add usage domains to the list of control domains
> automatically if set to one. That would be the default behavior which
> would be turned off by manually setting it to zero.

If we want to do something like that, having it add the usage domains
automatically sounds like the more workable alternative. What I like
about this is that we make it explicit that we change the masks beyond
what the admin explicitly configured, and provide a knob to turn off
that behaviour.

>
> >
> > One thing I'm still unsure about is how libvirt comes into the picture
> > here. Will it consume the setting, or actively manipulate it?
>
> libvirt is not in the picture. The only thing libvirt does is identify
> which mediated matrix device (i.e., UUID) is to be used by the guest.
>
> >
> > [In general, I'm not very clear about how libvirt will interact with the
> > whole infrastructure...]
>
> The libvirt domain XML will only specify the UUID of the matrix device
> to be used by the guest. Consequently, libvirt will insert the device
> statement for the vfio-ap device into the QEMU command line:
>
> -device vfio-ap,syfsdev={path-to-mdev}
>
> All configuration of the mediated device, including the proposed attribute
> above, is handled via sysfs attributes.

So, basically, libvirt will only consume what the admin has already
configured?

2018-08-27 13:50:10

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On 08/27/2018 04:33 AM, Cornelia Huck wrote:
> On Thu, 23 Aug 2018 10:16:59 -0400
> Tony Krowiak <[email protected]> wrote:
>
>> On 08/23/2018 06:25 AM, Cornelia Huck wrote:
>>> On Wed, 22 Aug 2018 15:16:19 -0400
>>> Tony Krowiak <[email protected]> wrote:
>>>
>>>> One of the things I suggested in a private conversation with Christian
>>>> earlier
>>>> today was to provide an additional rw sysfs attribute - a boolean - that
>>>> indicates
>>>> whether all usage domains should also be control domains. The default
>>>> could be
>>>> true. This would allow one to configure guests with usage-only domains
>>>> as well
>>>> as satisfy the convention.
>>> Would this additional attribute then control "add usage domains to the
>>> list of control domains automatically", or "don't allow to add a usage
>>> domain if it has not already been added as a control domain"?
>> It was just a proposal that wasn't really discussed at all, but this
>> attribute would add usage domains to the list of control domains
>> automatically if set to one. That would be the default behavior which
>> would be turned off by manually setting it to zero.
> If we want to do something like that, having it add the usage domains
> automatically sounds like the more workable alternative. What I like
> about this is that we make it explicit that we change the masks beyond
> what the admin explicitly configured, and provide a knob to turn off
> that behaviour.

So, are you saying I should go ahead and implement this?

>
>>> One thing I'm still unsure about is how libvirt comes into the picture
>>> here. Will it consume the setting, or actively manipulate it?
>> libvirt is not in the picture. The only thing libvirt does is identify
>> which mediated matrix device (i.e., UUID) is to be used by the guest.
>>
>>> [In general, I'm not very clear about how libvirt will interact with the
>>> whole infrastructure...]
>> The libvirt domain XML will only specify the UUID of the matrix device
>> to be used by the guest. Consequently, libvirt will insert the device
>> statement for the vfio-ap device into the QEMU command line:
>>
>> -device vfio-ap,syfsdev={path-to-mdev}
>>
>> All configuration of the mediated device, including the proposed attribute
>> above, is handled via sysfs attributes.
> So, basically, libvirt will only consume what the admin has already
> configured?

Yes.

>

2018-08-27 13:54:12

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On Mon, 27 Aug 2018 09:47:58 -0400
Tony Krowiak <[email protected]> wrote:

> On 08/27/2018 04:33 AM, Cornelia Huck wrote:
> > On Thu, 23 Aug 2018 10:16:59 -0400
> > Tony Krowiak <[email protected]> wrote:
> >
> >> On 08/23/2018 06:25 AM, Cornelia Huck wrote:
> >>> On Wed, 22 Aug 2018 15:16:19 -0400
> >>> Tony Krowiak <[email protected]> wrote:
> >>>
> >>>> One of the things I suggested in a private conversation with Christian
> >>>> earlier
> >>>> today was to provide an additional rw sysfs attribute - a boolean - that
> >>>> indicates
> >>>> whether all usage domains should also be control domains. The default
> >>>> could be
> >>>> true. This would allow one to configure guests with usage-only domains
> >>>> as well
> >>>> as satisfy the convention.
> >>> Would this additional attribute then control "add usage domains to the
> >>> list of control domains automatically", or "don't allow to add a usage
> >>> domain if it has not already been added as a control domain"?
> >> It was just a proposal that wasn't really discussed at all, but this
> >> attribute would add usage domains to the list of control domains
> >> automatically if set to one. That would be the default behavior which
> >> would be turned off by manually setting it to zero.
> > If we want to do something like that, having it add the usage domains
> > automatically sounds like the more workable alternative. What I like
> > about this is that we make it explicit that we change the masks beyond
> > what the admin explicitly configured, and provide a knob to turn off
> > that behaviour.
>
> So, are you saying I should go ahead and implement this?

I'm just saying that it does not sound like a bad idea :)

If you agree that it's a good idea and if others also like it... I'd
certainly not mind you going ahead :)

2018-08-27 15:41:11

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On 08/27/2018 03:51 PM, Cornelia Huck wrote:
> On Mon, 27 Aug 2018 09:47:58 -0400
> Tony Krowiak <[email protected]> wrote:
>
>> On 08/27/2018 04:33 AM, Cornelia Huck wrote:
>>> On Thu, 23 Aug 2018 10:16:59 -0400
>>> Tony Krowiak <[email protected]> wrote:
>>>
>>>> On 08/23/2018 06:25 AM, Cornelia Huck wrote:
>>>>> On Wed, 22 Aug 2018 15:16:19 -0400
>>>>> Tony Krowiak <[email protected]> wrote:
>>>>>
>>>>>> One of the things I suggested in a private conversation with Christian
>>>>>> earlier
>>>>>> today was to provide an additional rw sysfs attribute - a boolean - that
>>>>>> indicates
>>>>>> whether all usage domains should also be control domains. The default
>>>>>> could be
>>>>>> true. This would allow one to configure guests with usage-only domains
>>>>>> as well
>>>>>> as satisfy the convention.
>>>>> Would this additional attribute then control "add usage domains to the
>>>>> list of control domains automatically", or "don't allow to add a usage
>>>>> domain if it has not already been added as a control domain"?
>>>> It was just a proposal that wasn't really discussed at all, but this
>>>> attribute would add usage domains to the list of control domains
>>>> automatically if set to one. That would be the default behavior which
>>>> would be turned off by manually setting it to zero.
>>> If we want to do something like that, having it add the usage domains
>>> automatically sounds like the more workable alternative. What I like
>>> about this is that we make it explicit that we change the masks beyond
>>> what the admin explicitly configured, and provide a knob to turn off
>>> that behaviour.
>>
>> So, are you saying I should go ahead and implement this?
>
> I'm just saying that it does not sound like a bad idea :)
>
> If you agree that it's a good idea and if others also like it... I'd
> certainly not mind you going ahead :)
>

I can live with it. What I don't like about it is that it adds
more context dependent semantics. The same sequence of actions
results in a different result (depending on the mode of operation).

Regards,
Halil

2018-09-06 09:48:43

[permalink] [raw]

Subject: Re: [PATCH v9 09/22] s390: vfio-ap: register matrix device with VFIO mdev framework

On 13/08/2018 23:48, Tony Krowiak wrote:
> From: Tony Krowiak <[email protected]>
>
> Registers the matrix device created by the VFIO AP device
> driver with the VFIO mediated device framework.
> Registering the matrix device will create the sysfs
> structures needed to create mediated matrix devices
> each of which will be used to configure the AP matrix
> for a guest and connect it to the VFIO AP device driver.
>
> Registering the matrix device with the VFIO mediated device
> framework will create the following sysfs structures:
>
> /sys/devices/vfio_ap/matrix/
> ...... [mdev_supported_types]
> ......... [vfio_ap-passthrough]
> ............ create
>
> To create a mediated device for the AP matrix device, write a UUID
> to the create file:
>
> uuidgen > create
>
> A symbolic link to the mediated device's directory will be created in the
> devices subdirectory named after the generated $uuid:
>
> /sys/devices/vfio_ap/matrix/
> ...... [mdev_supported_types]
> ......... [vfio_ap-passthrough]
> ............ [devices]
> ............... [$uuid]
>
> A symbolic link to the mediated device will also be created
> in the vfio_ap matrix's directory:
>
> /sys/devices/vfio_ap/matrix/[$uuid]
>
> Signed-off-by: Tony Krowiak <[email protected]>
> Reviewed-by: Halil Pasic <[email protected]>
> Tested-by: Michael Mueller <[email protected]>
> Tested-by: Farhan Ali <[email protected]>
> Signed-off-by: Christian Borntraeger <[email protected]>
> ---
> MAINTAINERS | 1 +
> drivers/s390/crypto/Makefile | 2 +-
> drivers/s390/crypto/vfio_ap_drv.c | 23 ++++++
> drivers/s390/crypto/vfio_ap_ops.c | 124 +++++++++++++++++++++++++++++++++
> drivers/s390/crypto/vfio_ap_private.h | 45 ++++++++++++
> include/uapi/linux/vfio.h | 1 +
> 6 files changed, 195 insertions(+), 1 deletions(-)
> create mode 100644 drivers/s390/crypto/vfio_ap_ops.c
>
> diff --git a/MAINTAINERS b/MAINTAINERS
> index e84c559..f60dd56 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -12427,6 +12427,7 @@ W: http://www.ibm.com/developerworks/linux/linux390/
> S: Supported
> F: drivers/s390/crypto/vfio_ap_drv.c
> F: drivers/s390/crypto/vfio_ap_private.h
> +F: drivers/s390/crypto/vfio_ap_ops.c
>
> S390 ZFCP DRIVER
> M: Steffen Maier <[email protected]>
> diff --git a/drivers/s390/crypto/Makefile b/drivers/s390/crypto/Makefile
> index 48e466e..8d36b05 100644
> --- a/drivers/s390/crypto/Makefile
> +++ b/drivers/s390/crypto/Makefile
> @@ -17,5 +17,5 @@ pkey-objs := pkey_api.o
> obj-$(CONFIG_PKEY) += pkey.o
>
> # adjunct processor matrix
> -vfio_ap-objs := vfio_ap_drv.o
> +vfio_ap-objs := vfio_ap_drv.o vfio_ap_ops.o
> obj-$(CONFIG_VFIO_AP) += vfio_ap.o
> diff --git a/drivers/s390/crypto/vfio_ap_drv.c b/drivers/s390/crypto/vfio_ap_drv.c
> index 5069580..fa04c5a 100644
> --- a/drivers/s390/crypto/vfio_ap_drv.c
> +++ b/drivers/s390/crypto/vfio_ap_drv.c
> @@ -11,6 +11,7 @@
> #include <linux/mod_devicetable.h>
> #include <linux/slab.h>
> #include <linux/string.h>
> +#include <asm/zcrypt.h>
> #include "vfio_ap_private.h"
>
> #define VFIO_AP_ROOT_NAME "vfio_ap"
> @@ -65,6 +66,19 @@ static int vfio_ap_matrix_dev_init(void)
> return ret;
> }
>
> + mutex_init(&matrix_dev.lock);
> + INIT_LIST_HEAD(&matrix_dev.mdev_list);
> +
> + /* Test if PQAP(QCI) instruction is available */
> + if (test_facility(12)) {
> + ret = ap_qci(&matrix_dev.info);
> + if (ret) {
> + root_device_unregister(root_device);
> + return ret;
> + }
> + }
> +
> + atomic_set(&matrix_dev.available_instances, MAX_ZDEV_ENTRIES_EXT);
> matrix_dev.device.type = &vfio_ap_dev_type;
> dev_set_name(&matrix_dev.device, "%s", VFIO_AP_DEV_NAME);
> matrix_dev.device.type = &vfio_ap_dev_type;
> @@ -105,11 +119,20 @@ int __init vfio_ap_init(void)
> return ret;
> }
>
> + ret = vfio_ap_mdev_register();
> + if (ret) {
> + ap_driver_unregister(&vfio_ap_drv);
> + vfio_ap_matrix_dev_destroy();
> +
> + return ret;
> + }
> +
> return 0;
> }
>
> void __exit vfio_ap_exit(void)
> {
> + vfio_ap_mdev_unregister();
> ap_driver_unregister(&vfio_ap_drv);
> vfio_ap_matrix_dev_destroy();
> }
> diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c
> new file mode 100644
> index 0000000..8018c2d
> --- /dev/null
> +++ b/drivers/s390/crypto/vfio_ap_ops.c
> @@ -0,0 +1,124 @@
> +// SPDX-License-Identifier: GPL-2.0+
> +/*
> + * Adjunct processor matrix VFIO device driver callbacks.
> + *
> + * Copyright IBM Corp. 2018
> + *
> + * Author(s): Tony Krowiak <[email protected]>
> + * Halil Pasic <[email protected]>
> + * Pierre Morel <[email protected]>
> + */
> +#include <linux/string.h>
> +#include <linux/vfio.h>
> +#include <linux/device.h>
> +#include <linux/list.h>
> +#include <linux/ctype.h>
> +
> +#include "vfio_ap_private.h"
> +
> +#define VFIO_AP_MDEV_TYPE_HWVIRT "passthrough"
> +#define VFIO_AP_MDEV_NAME_HWVIRT "VFIO AP Passthrough Device"
> +
> +static void vfio_ap_matrix_init(struct ap_config_info *info,
> + struct ap_matrix *matrix)
> +{
> + matrix->apm_max = info->apxa ? info->Na : 63;
> + matrix->aqm_max = info->apxa ? info->Nd : 15;
> + matrix->adm_max = info->apxa ? info->Nd : 15;
> +}
> +
> +static int vfio_ap_mdev_create(struct kobject *kobj, struct mdev_device *mdev)
> +{
> + struct ap_matrix_mdev *matrix_mdev;
> +
> + matrix_mdev = kzalloc(sizeof(*matrix_mdev), GFP_KERNEL);
> + if (!matrix_mdev)
> + return -ENOMEM;
> +
> + matrix_mdev->name = dev_name(mdev_dev(mdev));
> + vfio_ap_matrix_init(&matrix_dev.info, &matrix_mdev->matrix);
> + mdev_set_drvdata(mdev, matrix_mdev);
> +
> + if (atomic_dec_if_positive(&matrix_dev.available_instances) < 0) {
> + kfree(matrix_mdev);
> + return -EPERM;
> + }
> +
> + mutex_lock(&matrix_dev.lock);
> + list_add(&matrix_mdev->list, &matrix_dev.mdev_list);
> + mutex_unlock(&matrix_dev.lock);
> +
Hi Tony,

You need to initialize the matrix_mdev->list before
using it.

Regards,
Pierre

--
Pierre Morel
Linux/KVM/QEMU in Böblingen - Germany

2018-09-10 13:31:31

[permalink] [raw]

Subject: Re: [PATCH v9 12/22] s390: vfio-ap: sysfs interfaces to configure control domains

On 08/27/2018 09:51 AM, Cornelia Huck wrote:
> On Mon, 27 Aug 2018 09:47:58 -0400
> Tony Krowiak <[email protected]> wrote:
>
>> On 08/27/2018 04:33 AM, Cornelia Huck wrote:
>>> On Thu, 23 Aug 2018 10:16:59 -0400
>>> Tony Krowiak <[email protected]> wrote:
>>>
>>>> On 08/23/2018 06:25 AM, Cornelia Huck wrote:
>>>>> On Wed, 22 Aug 2018 15:16:19 -0400
>>>>> Tony Krowiak <[email protected]> wrote:
>>>>>
>>>>>> One of the things I suggested in a private conversation with Christian
>>>>>> earlier
>>>>>> today was to provide an additional rw sysfs attribute - a boolean - that
>>>>>> indicates
>>>>>> whether all usage domains should also be control domains. The default
>>>>>> could be
>>>>>> true. This would allow one to configure guests with usage-only domains
>>>>>> as well
>>>>>> as satisfy the convention.
>>>>> Would this additional attribute then control "add usage domains to the
>>>>> list of control domains automatically", or "don't allow to add a usage
>>>>> domain if it has not already been added as a control domain"?
>>>> It was just a proposal that wasn't really discussed at all, but this
>>>> attribute would add usage domains to the list of control domains
>>>> automatically if set to one. That would be the default behavior which
>>>> would be turned off by manually setting it to zero.
>>> If we want to do something like that, having it add the usage domains
>>> automatically sounds like the more workable alternative. What I like
>>> about this is that we make it explicit that we change the masks beyond
>>> what the admin explicitly configured, and provide a knob to turn off
>>> that behaviour.
>> So, are you saying I should go ahead and implement this?
> I'm just saying that it does not sound like a bad idea :)
>
> If you agree that it's a good idea and if others also like it... I'd
> certainly not mind you going ahead :)

This was discussed with out crypto team and hardware architects and it
was decided that configuring all usage domains as control domains also
is not the right thing to do, so only domains assigned as control
domains will be set in the ADM field of the the guest's CRYCB.

>

2018-09-10 13:41:25

[permalink] [raw]

Subject: Re: [PATCH v9 09/22] s390: vfio-ap: register matrix device with VFIO mdev framework

On 09/06/2018 04:49 AM, Pierre Morel wrote:
> On 13/08/2018 23:48, Tony Krowiak wrote:
>> From: Tony Krowiak <[email protected]>
>>
>> Registers the matrix device created by the VFIO AP device
>> driver with the VFIO mediated device framework.
>> Registering the matrix device will create the sysfs
>> structures needed to create mediated matrix devices
>> each of which will be used to configure the AP matrix
>> for a guest and connect it to the VFIO AP device driver.
>>
>> Registering the matrix device with the VFIO mediated device
>> framework will create the following sysfs structures:
>>
>> /sys/devices/vfio_ap/matrix/
>> ...... [mdev_supported_types]
>> ......... [vfio_ap-passthrough]
>> ............ create
>>
>> To create a mediated device for the AP matrix device, write a UUID
>> to the create file:
>>
>> uuidgen > create
>>
>> A symbolic link to the mediated device's directory will be created in
>> the
>> devices subdirectory named after the generated $uuid:
>>
>> /sys/devices/vfio_ap/matrix/
>> ...... [mdev_supported_types]
>> ......... [vfio_ap-passthrough]
>> ............ [devices]
>> ............... [$uuid]
>>
>> A symbolic link to the mediated device will also be created
>> in the vfio_ap matrix's directory:
>>
>> /sys/devices/vfio_ap/matrix/[$uuid]
>>
>> Signed-off-by: Tony Krowiak <[email protected]>
>> Reviewed-by: Halil Pasic <[email protected]>
>> Tested-by: Michael Mueller <[email protected]>
>> Tested-by: Farhan Ali <[email protected]>
>> Signed-off-by: Christian Borntraeger <[email protected]>
>> ---
>> MAINTAINERS | 1 +
>> drivers/s390/crypto/Makefile | 2 +-
>> drivers/s390/crypto/vfio_ap_drv.c | 23 ++++++
>> drivers/s390/crypto/vfio_ap_ops.c | 124
>> +++++++++++++++++++++++++++++++++
>> drivers/s390/crypto/vfio_ap_private.h | 45 ++++++++++++
>> include/uapi/linux/vfio.h | 1 +
>> 6 files changed, 195 insertions(+), 1 deletions(-)
>> create mode 100644 drivers/s390/crypto/vfio_ap_ops.c
>>
>> diff --git a/MAINTAINERS b/MAINTAINERS
>> index e84c559..f60dd56 100644
>> --- a/MAINTAINERS
>> +++ b/MAINTAINERS
>> @@ -12427,6 +12427,7 @@ W:
>> http://www.ibm.com/developerworks/linux/linux390/
>> S: Supported
>> F: drivers/s390/crypto/vfio_ap_drv.c
>> F: drivers/s390/crypto/vfio_ap_private.h
>> +F: drivers/s390/crypto/vfio_ap_ops.c
>>
>> S390 ZFCP DRIVER
>> M: Steffen Maier <[email protected]>
>> diff --git a/drivers/s390/crypto/Makefile b/drivers/s390/crypto/Makefile
>> index 48e466e..8d36b05 100644
>> --- a/drivers/s390/crypto/Makefile
>> +++ b/drivers/s390/crypto/Makefile
>> @@ -17,5 +17,5 @@ pkey-objs := pkey_api.o
>> obj-$(CONFIG_PKEY) += pkey.o
>>
>> # adjunct processor matrix
>> -vfio_ap-objs := vfio_ap_drv.o
>> +vfio_ap-objs := vfio_ap_drv.o vfio_ap_ops.o
>> obj-$(CONFIG_VFIO_AP) += vfio_ap.o
>> diff --git a/drivers/s390/crypto/vfio_ap_drv.c
>> b/drivers/s390/crypto/vfio_ap_drv.c
>> index 5069580..fa04c5a 100644
>> --- a/drivers/s390/crypto/vfio_ap_drv.c
>> +++ b/drivers/s390/crypto/vfio_ap_drv.c
>> @@ -11,6 +11,7 @@
>> #include <linux/mod_devicetable.h>
>> #include <linux/slab.h>
>> #include <linux/string.h>
>> +#include <asm/zcrypt.h>
>> #include "vfio_ap_private.h"
>>
>> #define VFIO_AP_ROOT_NAME "vfio_ap"
>> @@ -65,6 +66,19 @@ static int vfio_ap_matrix_dev_init(void)
>> return ret;
>> }
>>
>> + mutex_init(&matrix_dev.lock);
>> + INIT_LIST_HEAD(&matrix_dev.mdev_list);
>> +
>> + /* Test if PQAP(QCI) instruction is available */
>> + if (test_facility(12)) {
>> + ret = ap_qci(&matrix_dev.info);
>> + if (ret) {
>> + root_device_unregister(root_device);
>> + return ret;
>> + }
>> + }
>> +
>> + atomic_set(&matrix_dev.available_instances, MAX_ZDEV_ENTRIES_EXT);
>> matrix_dev.device.type = &vfio_ap_dev_type;
>> dev_set_name(&matrix_dev.device, "%s", VFIO_AP_DEV_NAME);
>> matrix_dev.device.type = &vfio_ap_dev_type;
>> @@ -105,11 +119,20 @@ int __init vfio_ap_init(void)
>> return ret;
>> }
>>
>> + ret = vfio_ap_mdev_register();
>> + if (ret) {
>> + ap_driver_unregister(&vfio_ap_drv);
>> + vfio_ap_matrix_dev_destroy();
>> +
>> + return ret;
>> + }
>> +
>> return 0;
>> }
>>
>> void __exit vfio_ap_exit(void)
>> {
>> + vfio_ap_mdev_unregister();
>> ap_driver_unregister(&vfio_ap_drv);
>> vfio_ap_matrix_dev_destroy();
>> }
>> diff --git a/drivers/s390/crypto/vfio_ap_ops.c
>> b/drivers/s390/crypto/vfio_ap_ops.c
>> new file mode 100644
>> index 0000000..8018c2d
>> --- /dev/null
>> +++ b/drivers/s390/crypto/vfio_ap_ops.c
>> @@ -0,0 +1,124 @@
>> +// SPDX-License-Identifier: GPL-2.0+
>> +/*
>> + * Adjunct processor matrix VFIO device driver callbacks.
>> + *
>> + * Copyright IBM Corp. 2018
>> + *
>> + * Author(s): Tony Krowiak <[email protected]>
>> + * Halil Pasic <[email protected]>
>> + * Pierre Morel <[email protected]>
>> + */
>> +#include <linux/string.h>
>> +#include <linux/vfio.h>
>> +#include <linux/device.h>
>> +#include <linux/list.h>
>> +#include <linux/ctype.h>
>> +
>> +#include "vfio_ap_private.h"
>> +
>> +#define VFIO_AP_MDEV_TYPE_HWVIRT "passthrough"
>> +#define VFIO_AP_MDEV_NAME_HWVIRT "VFIO AP Passthrough Device"
>> +
>> +static void vfio_ap_matrix_init(struct ap_config_info *info,
>> + struct ap_matrix *matrix)
>> +{
>> + matrix->apm_max = info->apxa ? info->Na : 63;
>> + matrix->aqm_max = info->apxa ? info->Nd : 15;
>> + matrix->adm_max = info->apxa ? info->Nd : 15;
>> +}
>> +
>> +static int vfio_ap_mdev_create(struct kobject *kobj, struct
>> mdev_device *mdev)
>> +{
>> + struct ap_matrix_mdev *matrix_mdev;
>> +
>> + matrix_mdev = kzalloc(sizeof(*matrix_mdev), GFP_KERNEL);
>> + if (!matrix_mdev)
>> + return -ENOMEM;
>> +
>> + matrix_mdev->name = dev_name(mdev_dev(mdev));
>> + vfio_ap_matrix_init(&matrix_dev.info, &matrix_mdev->matrix);
>> + mdev_set_drvdata(mdev, matrix_mdev);
>> +
>> + if (atomic_dec_if_positive(&matrix_dev.available_instances) < 0) {
>> + kfree(matrix_mdev);
>> + return -EPERM;
>> + }
>> +
>> + mutex_lock(&matrix_dev.lock);
>> + list_add(&matrix_mdev->list, &matrix_dev.mdev_list);
>> + mutex_unlock(&matrix_dev.lock);
>> +
> Hi Tony,
>
> You need to initialize the matrix_mdev->list before
> using it.

Why? Initialization only sets its prev and next pointers to point to
itself. The
list_add immediately sets those pointers to insert it into the
matrix_dev.mdev_list,
so what would be the purose?

>
>
> Regards,
> Pierre
>
>

2018-09-10 22:00:29

[permalink] [raw]

Subject: Re: [PATCH v9 09/22] s390: vfio-ap: register matrix device with VFIO mdev framework

On 09/10/2018 03:38 PM, Tony Krowiak wrote:
> On 09/06/2018 04:49 AM, Pierre Morel wrote:
>> On 13/08/2018 23:48, Tony Krowiak wrote:
>>> From: Tony Krowiak <[email protected]>
>>>
>>> Registers the matrix device created by the VFIO AP device
>>> driver with the VFIO mediated device framework.
>>> Registering the matrix device will create the sysfs
>>> structures needed to create mediated matrix devices
>>> each of which will be used to configure the AP matrix
>>> for a guest and connect it to the VFIO AP device driver.
>>>
>>> Registering the matrix device with the VFIO mediated device
>>> framework will create the following sysfs structures:
>>>
>>> /sys/devices/vfio_ap/matrix/
>>> ...... [mdev_supported_types]
>>> ......... [vfio_ap-passthrough]
>>> ............ create
>>>
>>> To create a mediated device for the AP matrix device, write a UUID
>>> to the create file:
>>>
>>>     uuidgen > create
>>>
>>> A symbolic link to the mediated device's directory will be created in the
>>> devices subdirectory named after the generated $uuid:
>>>
>>> /sys/devices/vfio_ap/matrix/
>>> ...... [mdev_supported_types]
>>> ......... [vfio_ap-passthrough]
>>> ............ [devices]
>>> ............... [$uuid]
>>>
>>> A symbolic link to the mediated device will also be created
>>> in the vfio_ap matrix's directory:
>>>
>>> /sys/devices/vfio_ap/matrix/[$uuid]
>>>
>>> Signed-off-by: Tony Krowiak <[email protected]>
>>> Reviewed-by: Halil Pasic <[email protected]>
>>> Tested-by: Michael Mueller <[email protected]>
>>> Tested-by: Farhan Ali <[email protected]>
>>> Signed-off-by: Christian Borntraeger <[email protected]>
>>> ---
>>> MAINTAINERS                           |    1 +
>>> drivers/s390/crypto/Makefile          |    2 +-
>>> drivers/s390/crypto/vfio_ap_drv.c     |   23 ++++++
>>> drivers/s390/crypto/vfio_ap_ops.c     | 124 +++++++++++++++++++++++++++++++++
>>> drivers/s390/crypto/vfio_ap_private.h |   45 ++++++++++++
>>> include/uapi/linux/vfio.h             |    1 +
>>> 6 files changed, 195 insertions(+), 1 deletions(-)
>>> create mode 100644 drivers/s390/crypto/vfio_ap_ops.c
>>>
>>> diff --git a/MAINTAINERS b/MAINTAINERS
>>> index e84c559..f60dd56 100644
>>> --- a/MAINTAINERS
>>> +++ b/MAINTAINERS
>>> @@ -12427,6 +12427,7 @@ W: http://www.ibm.com/developerworks/linux/linux390/
>>> S:    Supported
>>> F:    drivers/s390/crypto/vfio_ap_drv.c
>>> F:    drivers/s390/crypto/vfio_ap_private.h
>>> +F:    drivers/s390/crypto/vfio_ap_ops.c
>>>
>>> S390 ZFCP DRIVER
>>> M:    Steffen Maier <[email protected]>
>>> diff --git a/drivers/s390/crypto/Makefile b/drivers/s390/crypto/Makefile
>>> index 48e466e..8d36b05 100644
>>> --- a/drivers/s390/crypto/Makefile
>>> +++ b/drivers/s390/crypto/Makefile
>>> @@ -17,5 +17,5 @@ pkey-objs := pkey_api.o
>>> obj-$(CONFIG_PKEY) += pkey.o
>>>
>>> # adjunct processor matrix
>>> -vfio_ap-objs := vfio_ap_drv.o
>>> +vfio_ap-objs := vfio_ap_drv.o vfio_ap_ops.o
>>> obj-$(CONFIG_VFIO_AP) += vfio_ap.o
>>> diff --git a/drivers/s390/crypto/vfio_ap_drv.c b/drivers/s390/crypto/vfio_ap_drv.c
>>> index 5069580..fa04c5a 100644
>>> --- a/drivers/s390/crypto/vfio_ap_drv.c
>>> +++ b/drivers/s390/crypto/vfio_ap_drv.c
>>> @@ -11,6 +11,7 @@
>>> #include <linux/mod_devicetable.h>
>>> #include <linux/slab.h>
>>> #include <linux/string.h>
>>> +#include <asm/zcrypt.h>
>>> #include "vfio_ap_private.h"
>>>
>>> #define VFIO_AP_ROOT_NAME "vfio_ap"
>>> @@ -65,6 +66,19 @@ static int vfio_ap_matrix_dev_init(void)
>>>           return ret;
>>>       }
>>>
>>> +    mutex_init(&matrix_dev.lock);
>>> +    INIT_LIST_HEAD(&matrix_dev.mdev_list);
>>> +
>>> +    /* Test if PQAP(QCI) instruction is available */
>>> +    if (test_facility(12)) {
>>> +        ret = ap_qci(&matrix_dev.info);
>>> +        if (ret) {
>>> +            root_device_unregister(root_device);
>>> +            return ret;
>>> +        }
>>> +    }
>>> +
>>> +    atomic_set(&matrix_dev.available_instances, MAX_ZDEV_ENTRIES_EXT);
>>>       matrix_dev.device.type = &vfio_ap_dev_type;
>>>       dev_set_name(&matrix_dev.device, "%s", VFIO_AP_DEV_NAME);
>>>       matrix_dev.device.type = &vfio_ap_dev_type;
>>> @@ -105,11 +119,20 @@ int __init vfio_ap_init(void)
>>>           return ret;
>>>       }
>>>
>>> +    ret = vfio_ap_mdev_register();
>>> +    if (ret) {
>>> +        ap_driver_unregister(&vfio_ap_drv);
>>> +        vfio_ap_matrix_dev_destroy();
>>> +
>>> +        return ret;
>>> +    }
>>> +
>>>       return 0;
>>> }
>>>
>>> void __exit vfio_ap_exit(void)
>>> {
>>> +    vfio_ap_mdev_unregister();
>>>       ap_driver_unregister(&vfio_ap_drv);
>>>       vfio_ap_matrix_dev_destroy();
>>> }
>>> diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c
>>> new file mode 100644
>>> index 0000000..8018c2d
>>> --- /dev/null
>>> +++ b/drivers/s390/crypto/vfio_ap_ops.c
>>> @@ -0,0 +1,124 @@
>>> +// SPDX-License-Identifier: GPL-2.0+
>>> +/*
>>> + * Adjunct processor matrix VFIO device driver callbacks.
>>> + *
>>> + * Copyright IBM Corp. 2018
>>> + *
>>> + * Author(s): Tony Krowiak <[email protected]>
>>> + *          Halil Pasic <[email protected]>
>>> + *          Pierre Morel <[email protected]>
>>> + */
>>> +#include <linux/string.h>
>>> +#include <linux/vfio.h>
>>> +#include <linux/device.h>
>>> +#include <linux/list.h>
>>> +#include <linux/ctype.h>
>>> +
>>> +#include "vfio_ap_private.h"
>>> +
>>> +#define VFIO_AP_MDEV_TYPE_HWVIRT "passthrough"
>>> +#define VFIO_AP_MDEV_NAME_HWVIRT "VFIO AP Passthrough Device"
>>> +
>>> +static void vfio_ap_matrix_init(struct ap_config_info *info,
>>> +                struct ap_matrix *matrix)
>>> +{
>>> +    matrix->apm_max = info->apxa ? info->Na : 63;
>>> +    matrix->aqm_max = info->apxa ? info->Nd : 15;
>>> +    matrix->adm_max = info->apxa ? info->Nd : 15;
>>> +}
>>> +
>>> +static int vfio_ap_mdev_create(struct kobject *kobj, struct mdev_device *mdev)
>>> +{
>>> +    struct ap_matrix_mdev *matrix_mdev;
>>> +
>>> +    matrix_mdev = kzalloc(sizeof(*matrix_mdev), GFP_KERNEL);
>>> +    if (!matrix_mdev)
>>> +        return -ENOMEM;
>>> +
>>> +    matrix_mdev->name = dev_name(mdev_dev(mdev));
>>> +    vfio_ap_matrix_init(&matrix_dev.info, &matrix_mdev->matrix);
>>> +    mdev_set_drvdata(mdev, matrix_mdev);
>>> +
>>> +    if (atomic_dec_if_positive(&matrix_dev.available_instances) < 0) {
>>> +        kfree(matrix_mdev);
>>> +        return -EPERM;
>>> +    }
>>> +
>>> +    mutex_lock(&matrix_dev.lock);
>>> +    list_add(&matrix_mdev->list, &matrix_dev.mdev_list);
>>> +    mutex_unlock(&matrix_dev.lock);
>>> +
>> Hi Tony,
>>
>> You need to initialize the matrix_mdev->list before
>> using it.
>
> Why? Initialization only sets its prev and next pointers to point to itself. The
> list_add immediately sets those pointers to insert it into the matrix_dev.mdev_list,
> so what would be the purose?
>

I think the source of the confusion is that matrix_mdev->list a new element to be
added to the matrix_dev.mdev_list and not a sentinel node of some list (which in
a sense stands for the list as a whole).

Regards,
Halil

2018-09-12 17:03:48

[permalink] [raw]

Subject: Re: [PATCH v9 13/22] s390: vfio-ap: sysfs interface to view matrix mdev matrix

On 08/20/2018 10:08 AM, Cornelia Huck wrote:
> I find this output to be a bit confusing; but OTOH, I'm probably not
> the person to parse it :) Still, some comments.
>
> From previous discussions, ISTR that this is mainly supposed to be a
> debugging/administration aid. Of course, this generates some questions:
> - Should this be in sysfs (sysfs attributes are supposed to follow the
> "one value per file" rule, at least for the most part), or would
> debugfs be a better fit?
> - Should userspace code be able to introspect the current
> configuration? If yes, it might be better to have some
> not-so-nice-but-easily-parsable output, possibly one attribute for
> the assigned adapters and one for the assigned domains, and a tool
> which distills that into a nice "matrix" with labeled rows and
> columns for human consumption.
>
> That said, I don't really have major objections to that interface.

For now, I will leave it in, but we could conceivably simplify configuration
of the matrix and adhere to sysfs rules by modeling this after the AP bus
apmask and aqmask sysfs interfaces. If we did that, there would need to
be only
three RW sysfs interfaces:

apm or apmask or ap_mask or adapter_mask or apid_mask
aqm or aqmask or aq_mask or domains or usage_domain_mask or apqi_mask
adm or admask or ad_mask or control_domains or control_domain_mask or
domain_mask

To assign a device, either a bitmask or a comma separated list of IDs
prepended
with a '+' could be passed in. To unassign a device, either a bitmask in
or a
comma-separated list of IDs prepended with a '-' could be passed in.

Reading an attribute would return the mask. As you suggested, tools could be
provided to parse the output and display it in a human-readable format.

>
>> Signed-off-by: Tony Krowiak <[email protected]>
>> Reviewed-by: Halil Pasic <[email protected]>
>> Tested-by: Michael Mueller <[email protected]>
>> Tested-by: Farhan Ali <[email protected]>
>> Tested-by: Pierre Morel <[email protected]>
>> Signed-off-by: Christian Borntraeger <[email protected]>
>> ---
>> drivers/s390/crypto/vfio_ap_ops.c | 52 +++++++++++++++++++++++++++++++++++++
>> 1 files changed, 52 insertions(+), 0 deletions(-)

2018-09-12 17:44:13

[permalink] [raw]

Subject: Re: [PATCH v9 21/22] KVM: s390: CPU model support for AP virtualization

On 08/23/2018 04:24 AM, Cornelia Huck wrote:
> On Thu, 23 Aug 2018 09:48:48 +0200
> David Hildenbrand <[email protected]> wrote:
>
>>> Migration of AP devices is not supported by this patch series, so this
>>> should
>>> not be an issue.
>> Might not be a problem now, but could be later. As I said in a different
>> reply, the CPU model in QEMU does not care about KVM.
>>
>> I want the QEMU CPU model and the KVM interfaces to be clean and future
>> proof. That's why my opinion is to handle PQAP(QCI) just like all the
>> other "feature blocks" we already have.
> +1 to that sentiment.
>
> It's better to try to get this correct now than having to hack around
> should we want to implement things in the future.

Just so we're on the same page here as far as what to expect for v10 of
this patch series, let me summarize the the very long series of private
exchanges as well as this thread:

* The APXA facility indicated by a bit returned in the response to the
PQAP(QCI) function indicates only whether the APXA facility is available
on one or more APs installed on the system.
* The only way to change the bit returned from PQAP(QCI) is to intercept the
instruction and emulate it, so it makes no sense for passthrough devices.
* The AP(s) with APXA installed may not necessarily even be in the
configuration.
* The only way to determine whether APXA is installed in a given AP is to
query it using the PQAP(TAPQ) instruction.

It was decided that APXA is better modeled as device configuration. If
and when
emulation is implemented, APXA can be configured for any AP devices assigned
to a guest. Since AP instructions will be intercepted when emulating AP,
the PQAP(QCI) instruction can return the APXA bit according to whether any
AP is configured with APXA installed. That matches the real architecture
much
more closely. So, the bottom line is that we will not introduce a new
CPU model
feature for APXA in v10 of this series.

>

2018-09-13 09:12:40

[permalink] [raw]

Subject: Re: [PATCH v9 13/22] s390: vfio-ap: sysfs interface to view matrix mdev matrix

On 09/12/2018 07:01 PM, Tony Krowiak wrote:
> On 08/20/2018 10:08 AM, Cornelia Huck wrote:
>> I find this output to be a bit confusing; but OTOH, I'm probably not
>> the person to parse it :) Still, some comments.
>>
>> From previous discussions, ISTR that this is mainly supposed to be a
>> debugging/administration aid. Of course, this generates some questions:
>> - Should this be in sysfs (sysfs attributes are supposed to follow the
>>    "one value per file" rule, at least for the most part), or would
>>    debugfs be a better fit?
>> - Should userspace code be able to introspect the current
>>    configuration? If yes, it might be better to have some
>>    not-so-nice-but-easily-parsable output, possibly one attribute for
>>    the assigned adapters and one for the assigned domains, and a tool
>>    which distills that into a nice "matrix" with labeled rows and
>>    columns for human consumption.
>>
>> That said, I don't really have major objections to that interface.
>
> For now, I will leave it in, but we could conceivably simplify configuration
> of the matrix and adhere to sysfs rules by modeling this after the AP bus
> apmask and aqmask sysfs interfaces. If we did that, there would need to be only
> three RW sysfs interfaces:
>
> apm or apmask or ap_mask or adapter_mask or apid_mask
> aqm or aqmask or aq_mask or domains or usage_domain_mask or apqi_mask
> adm or admask or ad_mask or control_domains or control_domain_mask or domain_mask
>
> To assign a device, either a bitmask or a comma separated list of IDs prepended
> with a '+' could be passed in. To unassign a device, either a bitmask in or a
> comma-separated list of IDs prepended with a '-' could be passed in.
>
> Reading an attribute would return the mask. As you suggested, tools could be
> provided to parse the output and display it in a human-readable format.
>

I would prefer an interface where one can set at least one whole mask in one
go over the current one that let us manipulate just one bit at a time.

Hallil

>>
>>> Signed-off-by: Tony Krowiak <[email protected]>
>>> Reviewed-by: Halil Pasic <[email protected]>
>>> Tested-by: Michael Mueller <[email protected]>
>>> Tested-by: Farhan Ali <[email protected]>
>>> Tested-by: Pierre Morel <[email protected]>
>>> Signed-off-by: Christian Borntraeger <[email protected]>
>>> ---
>>> drivers/s390/crypto/vfio_ap_ops.c |   52 +++++++++++++++++++++++++++++++++++++
>>> 1 files changed, 52 insertions(+), 0 deletions(-)
>
>

2018-09-17 07:57:36