LinuxLists.cc - [PATCH v2] KVM: x86: fix backward migration with async

2018-02-01 21:17:59

Subject: [PATCH v2] KVM: x86: fix backward migration with async_PF

Guests on new hypersiors might set KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT
bit when enabling async_PF, but this bit is reserved on old hypervisors,
which results in a failure upon migration.

To avoid breaking different cases, we are checking for CPUID feature bit
before enabling the feature and nothing else.

Fixes: 52a5c155cf79 ("KVM: async_pf: Let guest support delivery of async_pf from guest mode")
Cc: <[email protected]>
Signed-off-by: Radim Krčmář <[email protected]>
---
v2:
* added documentation [Paolo]
* retained compatibility recent kernels [Paolo]
---
Documentation/virtual/kvm/cpuid.txt | 4 ++++
Documentation/virtual/kvm/msr.txt | 3 ++-
arch/x86/include/uapi/asm/kvm_para.h | 1 +
arch/x86/kernel/kvm.c | 8 ++++----
arch/x86/kvm/cpuid.c | 3 ++-
5 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/Documentation/virtual/kvm/cpuid.txt b/Documentation/virtual/kvm/cpuid.txt
index dcab6dc11e3b..87a7506f31c2 100644
--- a/Documentation/virtual/kvm/cpuid.txt
+++ b/Documentation/virtual/kvm/cpuid.txt
@@ -58,6 +58,10 @@ KVM_FEATURE_PV_TLB_FLUSH || 9 || guest checks this feature bit
|| || before enabling paravirtualized
|| || tlb flush.
------------------------------------------------------------------------------
+KVM_FEATURE_ASYNC_PF_VMEXIT || 10 || paravirtualized async PF VM exit
+ || || can be enabled by setting bit 2
+ || || when writing to msr 0x4b564d02
+------------------------------------------------------------------------------
KVM_FEATURE_CLOCKSOURCE_STABLE_BIT || 24 || host will warn if no guest-side
|| || per-cpu warps are expected in
|| || kvmclock.
diff --git a/Documentation/virtual/kvm/msr.txt b/Documentation/virtual/kvm/msr.txt
index 1ebecc115dc6..f3f0d57ced8e 100644
--- a/Documentation/virtual/kvm/msr.txt
+++ b/Documentation/virtual/kvm/msr.txt
@@ -170,7 +170,8 @@ MSR_KVM_ASYNC_PF_EN: 0x4b564d02
when asynchronous page faults are enabled on the vcpu 0 when
disabled. Bit 1 is 1 if asynchronous page faults can be injected
when vcpu is in cpl == 0. Bit 2 is 1 if asynchronous page faults
- are delivered to L1 as #PF vmexits.
+ are delivered to L1 as #PF vmexits. Bit 2 can be set only if
+ KVM_FEATURE_ASYNC_PF_VMEXIT is present in CPUID.

First 4 byte of 64 byte memory location will be written to by
the hypervisor at the time of asynchronous page fault (APF)
diff --git a/arch/x86/include/uapi/asm/kvm_para.h b/arch/x86/include/uapi/asm/kvm_para.h
index 7a2ade4aa235..6cfa9c8cb7d6 100644
--- a/arch/x86/include/uapi/asm/kvm_para.h
+++ b/arch/x86/include/uapi/asm/kvm_para.h
@@ -26,6 +26,7 @@
#define KVM_FEATURE_PV_EOI 6
#define KVM_FEATURE_PV_UNHALT 7
#define KVM_FEATURE_PV_TLB_FLUSH 9
+#define KVM_FEATURE_ASYNC_PF_VMEXIT 10

/* The last 8 bits are used to indicate how to interpret the flags field
* in pvclock structure. If no bits are set, all flags are ignored.
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index 4e37d1a851a6..971babe964d2 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -341,10 +341,10 @@ static void kvm_guest_cpu_init(void)
#endif
pa |= KVM_ASYNC_PF_ENABLED;

- /* Async page fault support for L1 hypervisor is optional */
- if (wrmsr_safe(MSR_KVM_ASYNC_PF_EN,
- (pa | KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT) & 0xffffffff, pa >> 32) < 0)
- wrmsrl(MSR_KVM_ASYNC_PF_EN, pa);
+ if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF_VMEXIT))
+ pa |= KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;
+
+ wrmsrl(MSR_KVM_ASYNC_PF_EN, pa);
__this_cpu_write(apf_reason.enabled, 1);
printk(KERN_INFO"KVM setup async PF for cpu %d\n",
smp_processor_id());
diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c
index 20e491b94f44..7fc04a176c57 100644
--- a/arch/x86/kvm/cpuid.c
+++ b/arch/x86/kvm/cpuid.c
@@ -604,7 +604,8 @@ static inline int __do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
(1 << KVM_FEATURE_PV_EOI) |
(1 << KVM_FEATURE_CLOCKSOURCE_STABLE_BIT) |
(1 << KVM_FEATURE_PV_UNHALT) |
- (1 << KVM_FEATURE_PV_TLB_FLUSH);
+ (1 << KVM_FEATURE_PV_TLB_FLUSH) |
+ (1 << KVM_FEATURE_ASYNC_PF_VMEXIT);

if (sched_info_on())
entry->eax |= (1 << KVM_FEATURE_STEAL_TIME);
--
2.15.1

2018-02-02 01:46:31

by Wanpeng Li

[permalink] [raw]

Subject: Re: [PATCH v2] KVM: x86: fix backward migration with async_PF

2018-02-02 5:16 GMT+08:00 Radim Krčmář <[email protected]>:
> Guests on new hypersiors might set KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT
> bit when enabling async_PF, but this bit is reserved on old hypervisors,
> which results in a failure upon migration.
>
> To avoid breaking different cases, we are checking for CPUID feature bit
> before enabling the feature and nothing else.
>
> Fixes: 52a5c155cf79 ("KVM: async_pf: Let guest support delivery of async_pf from guest mode")
> Cc: <[email protected]>
> Signed-off-by: Radim Krčmář <[email protected]>

Reviewed-by: Wanpeng Li <[email protected]>

> ---
> v2:
> * added documentation [Paolo]
> * retained compatibility recent kernels [Paolo]
> ---
> Documentation/virtual/kvm/cpuid.txt | 4 ++++
> Documentation/virtual/kvm/msr.txt | 3 ++-
> arch/x86/include/uapi/asm/kvm_para.h | 1 +
> arch/x86/kernel/kvm.c | 8 ++++----
> arch/x86/kvm/cpuid.c | 3 ++-
> 5 files changed, 13 insertions(+), 6 deletions(-)
>
> diff --git a/Documentation/virtual/kvm/cpuid.txt b/Documentation/virtual/kvm/cpuid.txt
> index dcab6dc11e3b..87a7506f31c2 100644
> --- a/Documentation/virtual/kvm/cpuid.txt
> +++ b/Documentation/virtual/kvm/cpuid.txt
> @@ -58,6 +58,10 @@ KVM_FEATURE_PV_TLB_FLUSH || 9 || guest checks this feature bit
> || || before enabling paravirtualized
> || || tlb flush.
> ------------------------------------------------------------------------------
> +KVM_FEATURE_ASYNC_PF_VMEXIT || 10 || paravirtualized async PF VM exit
> + || || can be enabled by setting bit 2
> + || || when writing to msr 0x4b564d02
> +------------------------------------------------------------------------------
> KVM_FEATURE_CLOCKSOURCE_STABLE_BIT || 24 || host will warn if no guest-side
> || || per-cpu warps are expected in
> || || kvmclock.
> diff --git a/Documentation/virtual/kvm/msr.txt b/Documentation/virtual/kvm/msr.txt
> index 1ebecc115dc6..f3f0d57ced8e 100644
> --- a/Documentation/virtual/kvm/msr.txt
> +++ b/Documentation/virtual/kvm/msr.txt
> @@ -170,7 +170,8 @@ MSR_KVM_ASYNC_PF_EN: 0x4b564d02
> when asynchronous page faults are enabled on the vcpu 0 when
> disabled. Bit 1 is 1 if asynchronous page faults can be injected
> when vcpu is in cpl == 0. Bit 2 is 1 if asynchronous page faults
> - are delivered to L1 as #PF vmexits.
> + are delivered to L1 as #PF vmexits. Bit 2 can be set only if
> + KVM_FEATURE_ASYNC_PF_VMEXIT is present in CPUID.
>
> First 4 byte of 64 byte memory location will be written to by
> the hypervisor at the time of asynchronous page fault (APF)
> diff --git a/arch/x86/include/uapi/asm/kvm_para.h b/arch/x86/include/uapi/asm/kvm_para.h
> index 7a2ade4aa235..6cfa9c8cb7d6 100644
> --- a/arch/x86/include/uapi/asm/kvm_para.h
> +++ b/arch/x86/include/uapi/asm/kvm_para.h
> @@ -26,6 +26,7 @@
> #define KVM_FEATURE_PV_EOI 6
> #define KVM_FEATURE_PV_UNHALT 7
> #define KVM_FEATURE_PV_TLB_FLUSH 9
> +#define KVM_FEATURE_ASYNC_PF_VMEXIT 10
>
> /* The last 8 bits are used to indicate how to interpret the flags field
> * in pvclock structure. If no bits are set, all flags are ignored.
> diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
> index 4e37d1a851a6..971babe964d2 100644
> --- a/arch/x86/kernel/kvm.c
> +++ b/arch/x86/kernel/kvm.c
> @@ -341,10 +341,10 @@ static void kvm_guest_cpu_init(void)
> #endif
> pa |= KVM_ASYNC_PF_ENABLED;
>
> - /* Async page fault support for L1 hypervisor is optional */
> - if (wrmsr_safe(MSR_KVM_ASYNC_PF_EN,
> - (pa | KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT) & 0xffffffff, pa >> 32) < 0)
> - wrmsrl(MSR_KVM_ASYNC_PF_EN, pa);
> + if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF_VMEXIT))
> + pa |= KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;
> +
> + wrmsrl(MSR_KVM_ASYNC_PF_EN, pa);
> __this_cpu_write(apf_reason.enabled, 1);
> printk(KERN_INFO"KVM setup async PF for cpu %d\n",
> smp_processor_id());
> diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c
> index 20e491b94f44..7fc04a176c57 100644
> --- a/arch/x86/kvm/cpuid.c
> +++ b/arch/x86/kvm/cpuid.c
> @@ -604,7 +604,8 @@ static inline int __do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
> (1 << KVM_FEATURE_PV_EOI) |
> (1 << KVM_FEATURE_CLOCKSOURCE_STABLE_BIT) |
> (1 << KVM_FEATURE_PV_UNHALT) |
> - (1 << KVM_FEATURE_PV_TLB_FLUSH);
> + (1 << KVM_FEATURE_PV_TLB_FLUSH) |
> + (1 << KVM_FEATURE_ASYNC_PF_VMEXIT);
>
> if (sched_info_on())
> entry->eax |= (1 << KVM_FEATURE_STEAL_TIME);
> --
> 2.15.1
>

2018-02-02 10:04:57

by David Hildenbrand

[permalink] [raw]

Subject: Re: [PATCH v2] KVM: x86: fix backward migration with async_PF

On 01.02.2018 22:16, Radim Krčmář wrote:
> Guests on new hypersiors might set KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT
> bit when enabling async_PF, but this bit is reserved on old hypervisors,
> which results in a failure upon migration.
>
> To avoid breaking different cases, we are checking for CPUID feature bit
> before enabling the feature and nothing else.
>
> Fixes: 52a5c155cf79 ("KVM: async_pf: Let guest support delivery of async_pf from guest mode")
> Cc: <[email protected]>
> Signed-off-by: Radim Krčmář <[email protected]>
> ---
> v2:
> * added documentation [Paolo]
> * retained compatibility recent kernels [Paolo]
> ---
> Documentation/virtual/kvm/cpuid.txt | 4 ++++
> Documentation/virtual/kvm/msr.txt | 3 ++-
> arch/x86/include/uapi/asm/kvm_para.h | 1 +
> arch/x86/kernel/kvm.c | 8 ++++----
> arch/x86/kvm/cpuid.c | 3 ++-
> 5 files changed, 13 insertions(+), 6 deletions(-)
>
> diff --git a/Documentation/virtual/kvm/cpuid.txt b/Documentation/virtual/kvm/cpuid.txt
> index dcab6dc11e3b..87a7506f31c2 100644
> --- a/Documentation/virtual/kvm/cpuid.txt
> +++ b/Documentation/virtual/kvm/cpuid.txt
> @@ -58,6 +58,10 @@ KVM_FEATURE_PV_TLB_FLUSH || 9 || guest checks this feature bit
> || || before enabling paravirtualized
> || || tlb flush.
> ------------------------------------------------------------------------------
> +KVM_FEATURE_ASYNC_PF_VMEXIT || 10 || paravirtualized async PF VM exit
> + || || can be enabled by setting bit 2
> + || || when writing to msr 0x4b564d02
> +------------------------------------------------------------------------------
> KVM_FEATURE_CLOCKSOURCE_STABLE_BIT || 24 || host will warn if no guest-side
> || || per-cpu warps are expected in
> || || kvmclock.
> diff --git a/Documentation/virtual/kvm/msr.txt b/Documentation/virtual/kvm/msr.txt
> index 1ebecc115dc6..f3f0d57ced8e 100644
> --- a/Documentation/virtual/kvm/msr.txt
> +++ b/Documentation/virtual/kvm/msr.txt
> @@ -170,7 +170,8 @@ MSR_KVM_ASYNC_PF_EN: 0x4b564d02
> when asynchronous page faults are enabled on the vcpu 0 when
> disabled. Bit 1 is 1 if asynchronous page faults can be injected
> when vcpu is in cpl == 0. Bit 2 is 1 if asynchronous page faults
> - are delivered to L1 as #PF vmexits.
> + are delivered to L1 as #PF vmexits. Bit 2 can be set only if
> + KVM_FEATURE_ASYNC_PF_VMEXIT is present in CPUID.
>
> First 4 byte of 64 byte memory location will be written to by
> the hypervisor at the time of asynchronous page fault (APF)
> diff --git a/arch/x86/include/uapi/asm/kvm_para.h b/arch/x86/include/uapi/asm/kvm_para.h
> index 7a2ade4aa235..6cfa9c8cb7d6 100644
> --- a/arch/x86/include/uapi/asm/kvm_para.h
> +++ b/arch/x86/include/uapi/asm/kvm_para.h
> @@ -26,6 +26,7 @@
> #define KVM_FEATURE_PV_EOI 6
> #define KVM_FEATURE_PV_UNHALT 7
> #define KVM_FEATURE_PV_TLB_FLUSH 9
> +#define KVM_FEATURE_ASYNC_PF_VMEXIT 10
>
> /* The last 8 bits are used to indicate how to interpret the flags field
> * in pvclock structure. If no bits are set, all flags are ignored.
> diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
> index 4e37d1a851a6..971babe964d2 100644
> --- a/arch/x86/kernel/kvm.c
> +++ b/arch/x86/kernel/kvm.c
> @@ -341,10 +341,10 @@ static void kvm_guest_cpu_init(void)
> #endif
> pa |= KVM_ASYNC_PF_ENABLED;
>
> - /* Async page fault support for L1 hypervisor is optional */
> - if (wrmsr_safe(MSR_KVM_ASYNC_PF_EN,
> - (pa | KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT) & 0xffffffff, pa >> 32) < 0)
> - wrmsrl(MSR_KVM_ASYNC_PF_EN, pa);
> + if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF_VMEXIT))
> + pa |= KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;
> +
> + wrmsrl(MSR_KVM_ASYNC_PF_EN, pa);
> __this_cpu_write(apf_reason.enabled, 1);
> printk(KERN_INFO"KVM setup async PF for cpu %d\n",
> smp_processor_id());
> diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c
> index 20e491b94f44..7fc04a176c57 100644
> --- a/arch/x86/kvm/cpuid.c
> +++ b/arch/x86/kvm/cpuid.c
> @@ -604,7 +604,8 @@ static inline int __do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
> (1 << KVM_FEATURE_PV_EOI) |
> (1 << KVM_FEATURE_CLOCKSOURCE_STABLE_BIT) |
> (1 << KVM_FEATURE_PV_UNHALT) |
> - (1 << KVM_FEATURE_PV_TLB_FLUSH);
> + (1 << KVM_FEATURE_PV_TLB_FLUSH) |
> + (1 << KVM_FEATURE_ASYNC_PF_VMEXIT);
>
> if (sched_info_on())
> entry->eax |= (1 << KVM_FEATURE_STEAL_TIME);
>

Reviewed-by: David Hildenbrand <[email protected]>

--

Thanks,

David / dhildenb

2018-02-23 22:47:23

by Paolo Bonzini

[permalink] [raw]

Subject: Re: [PATCH v2] KVM: x86: fix backward migration with async_PF

On 01/02/2018 22:16, Radim Krčmář wrote:
> Guests on new hypersiors might set KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT
> bit when enabling async_PF, but this bit is reserved on old hypervisors,
> which results in a failure upon migration.
>
> To avoid breaking different cases, we are checking for CPUID feature bit
> before enabling the feature and nothing else.
>
> Fixes: 52a5c155cf79 ("KVM: async_pf: Let guest support delivery of async_pf from guest mode")
> Cc: <[email protected]>
> Signed-off-by: Radim Krčmář <[email protected]>
> ---
> v2:
> * added documentation [Paolo]
> * retained compatibility recent kernels [Paolo]
> ---
> Documentation/virtual/kvm/cpuid.txt | 4 ++++
> Documentation/virtual/kvm/msr.txt | 3 ++-
> arch/x86/include/uapi/asm/kvm_para.h | 1 +
> arch/x86/kernel/kvm.c | 8 ++++----
> arch/x86/kvm/cpuid.c | 3 ++-
> 5 files changed, 13 insertions(+), 6 deletions(-)
>
> diff --git a/Documentation/virtual/kvm/cpuid.txt b/Documentation/virtual/kvm/cpuid.txt
> index dcab6dc11e3b..87a7506f31c2 100644
> --- a/Documentation/virtual/kvm/cpuid.txt
> +++ b/Documentation/virtual/kvm/cpuid.txt
> @@ -58,6 +58,10 @@ KVM_FEATURE_PV_TLB_FLUSH || 9 || guest checks this feature bit
> || || before enabling paravirtualized
> || || tlb flush.
> ------------------------------------------------------------------------------
> +KVM_FEATURE_ASYNC_PF_VMEXIT || 10 || paravirtualized async PF VM exit
> + || || can be enabled by setting bit 2
> + || || when writing to msr 0x4b564d02
> +------------------------------------------------------------------------------
> KVM_FEATURE_CLOCKSOURCE_STABLE_BIT || 24 || host will warn if no guest-side
> || || per-cpu warps are expected in
> || || kvmclock.
> diff --git a/Documentation/virtual/kvm/msr.txt b/Documentation/virtual/kvm/msr.txt
> index 1ebecc115dc6..f3f0d57ced8e 100644
> --- a/Documentation/virtual/kvm/msr.txt
> +++ b/Documentation/virtual/kvm/msr.txt
> @@ -170,7 +170,8 @@ MSR_KVM_ASYNC_PF_EN: 0x4b564d02
> when asynchronous page faults are enabled on the vcpu 0 when
> disabled. Bit 1 is 1 if asynchronous page faults can be injected
> when vcpu is in cpl == 0. Bit 2 is 1 if asynchronous page faults
> - are delivered to L1 as #PF vmexits.
> + are delivered to L1 as #PF vmexits. Bit 2 can be set only if
> + KVM_FEATURE_ASYNC_PF_VMEXIT is present in CPUID.
>
> First 4 byte of 64 byte memory location will be written to by
> the hypervisor at the time of asynchronous page fault (APF)
> diff --git a/arch/x86/include/uapi/asm/kvm_para.h b/arch/x86/include/uapi/asm/kvm_para.h
> index 7a2ade4aa235..6cfa9c8cb7d6 100644
> --- a/arch/x86/include/uapi/asm/kvm_para.h
> +++ b/arch/x86/include/uapi/asm/kvm_para.h
> @@ -26,6 +26,7 @@
> #define KVM_FEATURE_PV_EOI 6
> #define KVM_FEATURE_PV_UNHALT 7
> #define KVM_FEATURE_PV_TLB_FLUSH 9
> +#define KVM_FEATURE_ASYNC_PF_VMEXIT 10
>
> /* The last 8 bits are used to indicate how to interpret the flags field
> * in pvclock structure. If no bits are set, all flags are ignored.
> diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
> index 4e37d1a851a6..971babe964d2 100644
> --- a/arch/x86/kernel/kvm.c
> +++ b/arch/x86/kernel/kvm.c
> @@ -341,10 +341,10 @@ static void kvm_guest_cpu_init(void)
> #endif
> pa |= KVM_ASYNC_PF_ENABLED;
>
> - /* Async page fault support for L1 hypervisor is optional */
> - if (wrmsr_safe(MSR_KVM_ASYNC_PF_EN,
> - (pa | KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT) & 0xffffffff, pa >> 32) < 0)
> - wrmsrl(MSR_KVM_ASYNC_PF_EN, pa);
> + if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF_VMEXIT))
> + pa |= KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;
> +
> + wrmsrl(MSR_KVM_ASYNC_PF_EN, pa);
> __this_cpu_write(apf_reason.enabled, 1);
> printk(KERN_INFO"KVM setup async PF for cpu %d\n",
> smp_processor_id());
> diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c
> index 20e491b94f44..7fc04a176c57 100644
> --- a/arch/x86/kvm/cpuid.c
> +++ b/arch/x86/kvm/cpuid.c
> @@ -604,7 +604,8 @@ static inline int __do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
> (1 << KVM_FEATURE_PV_EOI) |
> (1 << KVM_FEATURE_CLOCKSOURCE_STABLE_BIT) |
> (1 << KVM_FEATURE_PV_UNHALT) |
> - (1 << KVM_FEATURE_PV_TLB_FLUSH);
> + (1 << KVM_FEATURE_PV_TLB_FLUSH) |
> + (1 << KVM_FEATURE_ASYNC_PF_VMEXIT);
>
> if (sched_info_on())
> entry->eax |= (1 << KVM_FEATURE_STEAL_TIME);
>

Queued, thanks.

Paolo