Until v2.6.37, Linux used NMI watchdog that utilized IO-APIC and LVT0.
This series fixes some problems with APICv, restore, and concurrency
while keeping the monster asleep.
Radim Krčmář (3):
KVM: x86: keep track of LVT0 changes under APICv
KVM: x86: properly restore LVT0
KVM: x86: make vapics_in_nmi_mode atomic
arch/x86/include/asm/kvm_host.h | 2 +-
arch/x86/kvm/i8254.c | 2 +-
arch/x86/kvm/lapic.c | 17 ++++++++++-------
arch/x86/kvm/lapic.h | 1 +
4 files changed, 13 insertions(+), 9 deletions(-)
--
2.4.5
Memory-mapped LVT0 register already contains the new value when APICv
traps so we can't directly detect a change.
Memorize a bit we are interested in to enable legacy NMI watchdog.
Suggested-by: Yoshida Nobuo <[email protected]>
Signed-off-by: Radim Krčmář <[email protected]>
---
arch/x86/kvm/lapic.c | 14 ++++++++------
arch/x86/kvm/lapic.h | 1 +
2 files changed, 9 insertions(+), 6 deletions(-)
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index 36e9de1b4127..f49c7cca1de6 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -1257,16 +1257,17 @@ static void start_apic_timer(struct kvm_lapic *apic)
static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val)
{
- int nmi_wd_enabled = apic_lvt_nmi_mode(kvm_apic_get_reg(apic, APIC_LVT0));
+ bool lvt0_in_nmi_mode = apic_lvt_nmi_mode(lvt0_val);
- if (apic_lvt_nmi_mode(lvt0_val)) {
- if (!nmi_wd_enabled) {
+ if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) {
+ apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode;
+ if (lvt0_in_nmi_mode) {
apic_debug("Receive NMI setting on APIC_LVT0 "
"for cpu %d\n", apic->vcpu->vcpu_id);
apic->vcpu->kvm->arch.vapics_in_nmi_mode++;
- }
- } else if (nmi_wd_enabled)
- apic->vcpu->kvm->arch.vapics_in_nmi_mode--;
+ } else
+ apic->vcpu->kvm->arch.vapics_in_nmi_mode--;
+ }
}
static int apic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
@@ -1597,6 +1598,7 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event)
if (!(vcpu->kvm->arch.disabled_quirks & KVM_QUIRK_LINT0_REENABLED))
apic_set_reg(apic, APIC_LVT0,
SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT));
+ apic_manage_nmi_watchdog(apic, kvm_apic_get_reg(apic, APIC_LVT0));
apic_set_reg(apic, APIC_DFR, 0xffffffffU);
apic_set_spiv(apic, 0xff);
diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h
index f2f4e10ab772..71952748222a 100644
--- a/arch/x86/kvm/lapic.h
+++ b/arch/x86/kvm/lapic.h
@@ -26,6 +26,7 @@ struct kvm_lapic {
struct kvm_vcpu *vcpu;
bool sw_enabled;
bool irr_pending;
+ bool lvt0_in_nmi_mode;
/* Number of bits set in ISR. */
s16 isr_count;
/* The highest vector set in ISR; if -1 - invalid, must scan ISR. */
--
2.4.5
Legacy NMI watchdog didn't work after migration/resume, because
vapics_in_nmi_mode was left at 0.
Signed-off-by: Radim Krčmář <[email protected]>
---
arch/x86/kvm/lapic.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index f49c7cca1de6..8dc32b5a4e0d 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -1824,6 +1824,7 @@ void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu,
apic_update_ppr(apic);
hrtimer_cancel(&apic->lapic_timer.timer);
apic_update_lvtt(apic);
+ apic_manage_nmi_watchdog(apic, kvm_apic_get_reg(apic, APIC_LVT0));
update_divide_count(apic);
start_apic_timer(apic);
apic->irr_pending = true;
--
2.4.5
Writes were a bit racy, but hard to turn into a bug at the same time.
(Particularly because modern Linux doesn't use this feature anymore.)
Signed-off-by: Radim Krčmář <[email protected]>
---
arch/x86/include/asm/kvm_host.h | 2 +-
arch/x86/kvm/i8254.c | 2 +-
arch/x86/kvm/lapic.c | 4 ++--
3 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index c7fa57b529d2..2a7f5d782c33 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -607,7 +607,7 @@ struct kvm_arch {
struct kvm_pic *vpic;
struct kvm_ioapic *vioapic;
struct kvm_pit *vpit;
- int vapics_in_nmi_mode;
+ atomic_t vapics_in_nmi_mode;
struct mutex apic_map_lock;
struct kvm_apic_map *apic_map;
diff --git a/arch/x86/kvm/i8254.c b/arch/x86/kvm/i8254.c
index 4dce6f8b6129..f90952f64e79 100644
--- a/arch/x86/kvm/i8254.c
+++ b/arch/x86/kvm/i8254.c
@@ -305,7 +305,7 @@ static void pit_do_work(struct kthread_work *work)
* LVT0 to NMI delivery. Other PIC interrupts are just sent to
* VCPU0, and only if its LVT0 is in EXTINT mode.
*/
- if (kvm->arch.vapics_in_nmi_mode > 0)
+ if (atomic_read(&kvm->arch.vapics_in_nmi_mode) > 0)
kvm_for_each_vcpu(i, vcpu, kvm)
kvm_apic_nmi_wd_deliver(vcpu);
}
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index 8dc32b5a4e0d..954e98a8c2e3 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -1264,9 +1264,9 @@ static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val)
if (lvt0_in_nmi_mode) {
apic_debug("Receive NMI setting on APIC_LVT0 "
"for cpu %d\n", apic->vcpu->vcpu_id);
- apic->vcpu->kvm->arch.vapics_in_nmi_mode++;
+ atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
} else
- apic->vcpu->kvm->arch.vapics_in_nmi_mode--;
+ atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
}
}
--
2.4.5
On 30/06/2015 22:19, Radim Krčmář wrote:
> Legacy NMI watchdog didn't work after migration/resume, because
> vapics_in_nmi_mode was left at 0.
>
> Signed-off-by: Radim Krčmář <[email protected]>
> ---
> arch/x86/kvm/lapic.c | 1 +
> 1 file changed, 1 insertion(+)
>
> diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
> index f49c7cca1de6..8dc32b5a4e0d 100644
> --- a/arch/x86/kvm/lapic.c
> +++ b/arch/x86/kvm/lapic.c
> @@ -1824,6 +1824,7 @@ void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu,
> apic_update_ppr(apic);
> hrtimer_cancel(&apic->lapic_timer.timer);
> apic_update_lvtt(apic);
> + apic_manage_nmi_watchdog(apic, kvm_apic_get_reg(apic, APIC_LVT0));
> update_divide_count(apic);
> start_apic_timer(apic);
> apic->irr_pending = true;
>
Applied already, with Cc: stable, as it is not related to APICv.
Paolo
On 30/06/2015 22:19, Radim Krčmář wrote:
> Writes were a bit racy, but hard to turn into a bug at the same time.
> (Particularly because modern Linux doesn't use this feature anymore.)
I suspect patch 2 makes this race much easier to trigger, so it deserves
Cc: stable@ as well.
Paolo
> Signed-off-by: Radim Krčmář <[email protected]>
> ---
> arch/x86/include/asm/kvm_host.h | 2 +-
> arch/x86/kvm/i8254.c | 2 +-
> arch/x86/kvm/lapic.c | 4 ++--
> 3 files changed, 4 insertions(+), 4 deletions(-)
>
> diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
> index c7fa57b529d2..2a7f5d782c33 100644
> --- a/arch/x86/include/asm/kvm_host.h
> +++ b/arch/x86/include/asm/kvm_host.h
> @@ -607,7 +607,7 @@ struct kvm_arch {
> struct kvm_pic *vpic;
> struct kvm_ioapic *vioapic;
> struct kvm_pit *vpit;
> - int vapics_in_nmi_mode;
> + atomic_t vapics_in_nmi_mode;
> struct mutex apic_map_lock;
> struct kvm_apic_map *apic_map;
>
> diff --git a/arch/x86/kvm/i8254.c b/arch/x86/kvm/i8254.c
> index 4dce6f8b6129..f90952f64e79 100644
> --- a/arch/x86/kvm/i8254.c
> +++ b/arch/x86/kvm/i8254.c
> @@ -305,7 +305,7 @@ static void pit_do_work(struct kthread_work *work)
> * LVT0 to NMI delivery. Other PIC interrupts are just sent to
> * VCPU0, and only if its LVT0 is in EXTINT mode.
> */
> - if (kvm->arch.vapics_in_nmi_mode > 0)
> + if (atomic_read(&kvm->arch.vapics_in_nmi_mode) > 0)
> kvm_for_each_vcpu(i, vcpu, kvm)
> kvm_apic_nmi_wd_deliver(vcpu);
> }
> diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
> index 8dc32b5a4e0d..954e98a8c2e3 100644
> --- a/arch/x86/kvm/lapic.c
> +++ b/arch/x86/kvm/lapic.c
> @@ -1264,9 +1264,9 @@ static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val)
> if (lvt0_in_nmi_mode) {
> apic_debug("Receive NMI setting on APIC_LVT0 "
> "for cpu %d\n", apic->vcpu->vcpu_id);
> - apic->vcpu->kvm->arch.vapics_in_nmi_mode++;
> + atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
> } else
> - apic->vcpu->kvm->arch.vapics_in_nmi_mode--;
> + atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
> }
> }
>
>
On 30/06/2015 22:19, Radim Krčmář wrote:
> Until v2.6.37, Linux used NMI watchdog that utilized IO-APIC and LVT0.
> This series fixes some problems with APICv, restore, and concurrency
> while keeping the monster asleep.
Queued for 4.2.
Paolo