This patches removes SW timer switch during vcpu block stage. VM uses HW
timer rather than SW PV timer on LoongArch system, it can check pending
HW timer interrupt status directly, rather than switch to SW timer and
check injected SW timer interrupt.
When SW timer is not used in vcpu halt-polling mode, the relative
SW timer handling before entering guest can be removed also. Timer
emulation is simpler than before, SW timer emuation is only used in vcpu
thread context switch.
---
Changes in v4:
If vcpu is scheduled out since there is no pending event, and timer is
fired during sched-out period. SW hrtimer is used to wake up vcpu thread
in time, rather than inject pending timer irq.
Changes in v3:
Add kvm_arch_vcpu_runnable checking before kvm_vcpu_halt.
Changes in v2:
Add halt polling support for idle instruction emulation, using api
kvm_vcpu_halt rather than kvm_vcpu_block in function kvm_emu_idle.
---
Bibo Mao (3):
LoongArch: KVM: Remove SW timer switch when vcpu is halt polling
LoongArch: KVM: Allow to access HW timer CSR registers always
LoongArch: KVM: Remove kvm_acquire_timer before entering guest
arch/loongarch/include/asm/kvm_vcpu.h | 1 -
arch/loongarch/kvm/exit.c | 13 +-----
arch/loongarch/kvm/main.c | 1 -
arch/loongarch/kvm/timer.c | 62 ++++++++++-----------------
arch/loongarch/kvm/vcpu.c | 38 ++++------------
5 files changed, 32 insertions(+), 83 deletions(-)
base-commit: c42d9eeef8e5ba9292eda36fd8e3c11f35ee065c
--
2.39.3
Timer emulation method in VM is switch to SW timer, there are two
places where timer emulation is needed. One is during vcpu thread
context switch, the other is halt-polling with idle instruction
emulation. SW timer switching is remove during halt-polling mode,
so it is not necessary to disable SW timer before entering to guest.
This patch removes SW timer handling before entering guest mode, and
put it in HW timer restoring flow when vcpu thread is sched-in. With
this patch, vm timer emulation is simpler, there is SW/HW timer
switch only in vcpu thread context switch scenario.
Signed-off-by: Bibo Mao <[email protected]>
---
arch/loongarch/include/asm/kvm_vcpu.h | 1 -
arch/loongarch/kvm/timer.c | 22 ++++++--------------
arch/loongarch/kvm/vcpu.c | 29 ---------------------------
3 files changed, 6 insertions(+), 46 deletions(-)
diff --git a/arch/loongarch/include/asm/kvm_vcpu.h b/arch/loongarch/include/asm/kvm_vcpu.h
index 553cfa2b2b1c..0e87652f780a 100644
--- a/arch/loongarch/include/asm/kvm_vcpu.h
+++ b/arch/loongarch/include/asm/kvm_vcpu.h
@@ -55,7 +55,6 @@ void kvm_save_fpu(struct loongarch_fpu *fpu);
void kvm_restore_fpu(struct loongarch_fpu *fpu);
void kvm_restore_fcsr(struct loongarch_fpu *fpu);
-void kvm_acquire_timer(struct kvm_vcpu *vcpu);
void kvm_init_timer(struct kvm_vcpu *vcpu, unsigned long hz);
void kvm_reset_timer(struct kvm_vcpu *vcpu);
void kvm_save_timer(struct kvm_vcpu *vcpu);
diff --git a/arch/loongarch/kvm/timer.c b/arch/loongarch/kvm/timer.c
index e37c0ebffabd..711982f9eeb5 100644
--- a/arch/loongarch/kvm/timer.c
+++ b/arch/loongarch/kvm/timer.c
@@ -64,19 +64,6 @@ void kvm_init_timer(struct kvm_vcpu *vcpu, unsigned long timer_hz)
kvm_write_sw_gcsr(vcpu->arch.csr, LOONGARCH_CSR_TVAL, 0);
}
-/*
- * Restore hard timer state and enable guest to access timer registers
- * without trap, should be called with irq disabled
- */
-void kvm_acquire_timer(struct kvm_vcpu *vcpu)
-{
- /*
- * Freeze the soft-timer and sync the guest stable timer with it. We do
- * this with interrupts disabled to avoid latency.
- */
- hrtimer_cancel(&vcpu->arch.swtimer);
-}
-
/*
* Restore soft timer state from saved context.
*/
@@ -98,6 +85,11 @@ void kvm_restore_timer(struct kvm_vcpu *vcpu)
return;
}
+ /*
+ * Freeze the soft-timer and sync the guest stable timer with it.
+ */
+ hrtimer_cancel(&vcpu->arch.swtimer);
+
/*
* Set remainder tick value if not expired
*/
@@ -115,7 +107,7 @@ void kvm_restore_timer(struct kvm_vcpu *vcpu)
/*
* Inject timer here though sw timer should inject timer
* interrupt async already, since sw timer may be cancelled
- * during injecting intr async in function kvm_acquire_timer
+ * during injecting intr async
*/
kvm_queue_irq(vcpu, INT_TI);
}
@@ -140,11 +132,9 @@ static void _kvm_save_timer(struct kvm_vcpu *vcpu)
vcpu->arch.expire = expire;
if (ticks) {
/*
- * Update hrtimer to use new timeout
* HRTIMER_MODE_PINNED is suggested since vcpu may run in
* the same physical cpu in next time
*/
- hrtimer_cancel(&vcpu->arch.swtimer);
hrtimer_start(&vcpu->arch.swtimer, expire, HRTIMER_MODE_ABS_PINNED);
} else if (vcpu->stat.generic.blocking) {
/*
diff --git a/arch/loongarch/kvm/vcpu.c b/arch/loongarch/kvm/vcpu.c
index 42663a345bd1..cf1c4d64c1b7 100644
--- a/arch/loongarch/kvm/vcpu.c
+++ b/arch/loongarch/kvm/vcpu.c
@@ -95,7 +95,6 @@ static int kvm_pre_enter_guest(struct kvm_vcpu *vcpu)
* check vmid before vcpu enter guest
*/
local_irq_disable();
- kvm_acquire_timer(vcpu);
kvm_deliver_intr(vcpu);
kvm_deliver_exception(vcpu);
/* Make sure the vcpu mode has been written */
@@ -251,23 +250,6 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
return -EINVAL;
}
-/**
- * kvm_migrate_count() - Migrate timer.
- * @vcpu: Virtual CPU.
- *
- * Migrate hrtimer to the current CPU by cancelling and restarting it
- * if the hrtimer is active.
- *
- * Must be called when the vCPU is migrated to a different CPU, so that
- * the timer can interrupt the guest at the new CPU, and the timer irq can
- * be delivered to the vCPU.
- */
-static void kvm_migrate_count(struct kvm_vcpu *vcpu)
-{
- if (hrtimer_cancel(&vcpu->arch.swtimer))
- hrtimer_restart(&vcpu->arch.swtimer);
-}
-
static int _kvm_getcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 *val)
{
unsigned long gintc;
@@ -796,17 +778,6 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
unsigned long flags;
local_irq_save(flags);
- if (vcpu->arch.last_sched_cpu != cpu) {
- kvm_debug("[%d->%d]KVM vCPU[%d] switch\n",
- vcpu->arch.last_sched_cpu, cpu, vcpu->vcpu_id);
- /*
- * Migrate the timer interrupt to the current CPU so that it
- * always interrupts the guest and synchronously triggers a
- * guest timer interrupt.
- */
- kvm_migrate_count(vcpu);
- }
-
/* Restore guest state to registers */
_kvm_vcpu_load(vcpu, cpu);
local_irq_restore(flags);
--
2.39.3
Hi, Bibo,
I suggest submitting this series to the internal repo, too. Because we
don't have enough resources to test the stability for the upstream
version, while this is a fundamental change. On the other hand, the
patch "LoongArch:LSVZ: set timer offset at first time once" can be
submitted first because it is already in the internal repo.
Huacai
On Thu, Nov 16, 2023 at 10:33 AM Bibo Mao <[email protected]> wrote:
>
> This patches removes SW timer switch during vcpu block stage. VM uses HW
> timer rather than SW PV timer on LoongArch system, it can check pending
> HW timer interrupt status directly, rather than switch to SW timer and
> check injected SW timer interrupt.
>
> When SW timer is not used in vcpu halt-polling mode, the relative
> SW timer handling before entering guest can be removed also. Timer
> emulation is simpler than before, SW timer emuation is only used in vcpu
> thread context switch.
> ---
> Changes in v4:
> If vcpu is scheduled out since there is no pending event, and timer is
> fired during sched-out period. SW hrtimer is used to wake up vcpu thread
> in time, rather than inject pending timer irq.
>
> Changes in v3:
> Add kvm_arch_vcpu_runnable checking before kvm_vcpu_halt.
>
> Changes in v2:
> Add halt polling support for idle instruction emulation, using api
> kvm_vcpu_halt rather than kvm_vcpu_block in function kvm_emu_idle.
>
> ---
> Bibo Mao (3):
> LoongArch: KVM: Remove SW timer switch when vcpu is halt polling
> LoongArch: KVM: Allow to access HW timer CSR registers always
> LoongArch: KVM: Remove kvm_acquire_timer before entering guest
>
> arch/loongarch/include/asm/kvm_vcpu.h | 1 -
> arch/loongarch/kvm/exit.c | 13 +-----
> arch/loongarch/kvm/main.c | 1 -
> arch/loongarch/kvm/timer.c | 62 ++++++++++-----------------
> arch/loongarch/kvm/vcpu.c | 38 ++++------------
> 5 files changed, 32 insertions(+), 83 deletions(-)
>
>
> base-commit: c42d9eeef8e5ba9292eda36fd8e3c11f35ee065c
> --
> 2.39.3
>
On 2023/11/16 上午10:54, Huacai Chen wrote:
> Hi, Bibo,
>
> I suggest submitting this series to the internal repo, too. Because we
> don't have enough resources to test the stability for the upstream
> version, while this is a fundamental change. On the other hand, the
> patch "LoongArch:LSVZ: set timer offset at first time once" can be
> submitted first because it is already in the internal repo.
We ever consider to keep the same between internal repo and upstream,
only that there are many differences since kernel version is different.
We are preparing to setup test environment for upstream kvm module, the
stability for the upstream is very important for us -:)
Regards
Bibo Mao
>
> Huacai
>
> On Thu, Nov 16, 2023 at 10:33 AM Bibo Mao <[email protected]> wrote:
>>
>> This patches removes SW timer switch during vcpu block stage. VM uses HW
>> timer rather than SW PV timer on LoongArch system, it can check pending
>> HW timer interrupt status directly, rather than switch to SW timer and
>> check injected SW timer interrupt.
>>
>> When SW timer is not used in vcpu halt-polling mode, the relative
>> SW timer handling before entering guest can be removed also. Timer
>> emulation is simpler than before, SW timer emuation is only used in vcpu
>> thread context switch.
>> ---
>> Changes in v4:
>> If vcpu is scheduled out since there is no pending event, and timer is
>> fired during sched-out period. SW hrtimer is used to wake up vcpu thread
>> in time, rather than inject pending timer irq.
>>
>> Changes in v3:
>> Add kvm_arch_vcpu_runnable checking before kvm_vcpu_halt.
>>
>> Changes in v2:
>> Add halt polling support for idle instruction emulation, using api
>> kvm_vcpu_halt rather than kvm_vcpu_block in function kvm_emu_idle.
>>
>> ---
>> Bibo Mao (3):
>> LoongArch: KVM: Remove SW timer switch when vcpu is halt polling
>> LoongArch: KVM: Allow to access HW timer CSR registers always
>> LoongArch: KVM: Remove kvm_acquire_timer before entering guest
>>
>> arch/loongarch/include/asm/kvm_vcpu.h | 1 -
>> arch/loongarch/kvm/exit.c | 13 +-----
>> arch/loongarch/kvm/main.c | 1 -
>> arch/loongarch/kvm/timer.c | 62 ++++++++++-----------------
>> arch/loongarch/kvm/vcpu.c | 38 ++++------------
>> 5 files changed, 32 insertions(+), 83 deletions(-)
>>
>>
>> base-commit: c42d9eeef8e5ba9292eda36fd8e3c11f35ee065c
>> --
>> 2.39.3
>>
slightly ping...
On 2023/11/16 上午10:30, Bibo Mao wrote:
> This patches removes SW timer switch during vcpu block stage. VM uses HW
> timer rather than SW PV timer on LoongArch system, it can check pending
> HW timer interrupt status directly, rather than switch to SW timer and
> check injected SW timer interrupt.
>
> When SW timer is not used in vcpu halt-polling mode, the relative
> SW timer handling before entering guest can be removed also. Timer
> emulation is simpler than before, SW timer emuation is only used in vcpu
> thread context switch.
> ---
> Changes in v4:
> If vcpu is scheduled out since there is no pending event, and timer is
> fired during sched-out period. SW hrtimer is used to wake up vcpu thread
> in time, rather than inject pending timer irq.
>
> Changes in v3:
> Add kvm_arch_vcpu_runnable checking before kvm_vcpu_halt.
>
> Changes in v2:
> Add halt polling support for idle instruction emulation, using api
> kvm_vcpu_halt rather than kvm_vcpu_block in function kvm_emu_idle.
>
> ---
> Bibo Mao (3):
> LoongArch: KVM: Remove SW timer switch when vcpu is halt polling
> LoongArch: KVM: Allow to access HW timer CSR registers always
> LoongArch: KVM: Remove kvm_acquire_timer before entering guest
>
> arch/loongarch/include/asm/kvm_vcpu.h | 1 -
> arch/loongarch/kvm/exit.c | 13 +-----
> arch/loongarch/kvm/main.c | 1 -
> arch/loongarch/kvm/timer.c | 62 ++++++++++-----------------
> arch/loongarch/kvm/vcpu.c | 38 ++++------------
> 5 files changed, 32 insertions(+), 83 deletions(-)
>
>
> base-commit: c42d9eeef8e5ba9292eda36fd8e3c11f35ee065c
>
Reviewed-by: Tianrui Zhao <[email protected]>
?? 2023/11/16 ????10:30, Bibo Mao д??:
> Timer emulation method in VM is switch to SW timer, there are two
> places where timer emulation is needed. One is during vcpu thread
> context switch, the other is halt-polling with idle instruction
> emulation. SW timer switching is remove during halt-polling mode,
> so it is not necessary to disable SW timer before entering to guest.
>
> This patch removes SW timer handling before entering guest mode, and
> put it in HW timer restoring flow when vcpu thread is sched-in. With
> this patch, vm timer emulation is simpler, there is SW/HW timer
> switch only in vcpu thread context switch scenario.
>
> Signed-off-by: Bibo Mao <[email protected]>
> ---
> arch/loongarch/include/asm/kvm_vcpu.h | 1 -
> arch/loongarch/kvm/timer.c | 22 ++++++--------------
> arch/loongarch/kvm/vcpu.c | 29 ---------------------------
> 3 files changed, 6 insertions(+), 46 deletions(-)
>
> diff --git a/arch/loongarch/include/asm/kvm_vcpu.h b/arch/loongarch/include/asm/kvm_vcpu.h
> index 553cfa2b2b1c..0e87652f780a 100644
> --- a/arch/loongarch/include/asm/kvm_vcpu.h
> +++ b/arch/loongarch/include/asm/kvm_vcpu.h
> @@ -55,7 +55,6 @@ void kvm_save_fpu(struct loongarch_fpu *fpu);
> void kvm_restore_fpu(struct loongarch_fpu *fpu);
> void kvm_restore_fcsr(struct loongarch_fpu *fpu);
>
> -void kvm_acquire_timer(struct kvm_vcpu *vcpu);
> void kvm_init_timer(struct kvm_vcpu *vcpu, unsigned long hz);
> void kvm_reset_timer(struct kvm_vcpu *vcpu);
> void kvm_save_timer(struct kvm_vcpu *vcpu);
> diff --git a/arch/loongarch/kvm/timer.c b/arch/loongarch/kvm/timer.c
> index e37c0ebffabd..711982f9eeb5 100644
> --- a/arch/loongarch/kvm/timer.c
> +++ b/arch/loongarch/kvm/timer.c
> @@ -64,19 +64,6 @@ void kvm_init_timer(struct kvm_vcpu *vcpu, unsigned long timer_hz)
> kvm_write_sw_gcsr(vcpu->arch.csr, LOONGARCH_CSR_TVAL, 0);
> }
>
> -/*
> - * Restore hard timer state and enable guest to access timer registers
> - * without trap, should be called with irq disabled
> - */
> -void kvm_acquire_timer(struct kvm_vcpu *vcpu)
> -{
> - /*
> - * Freeze the soft-timer and sync the guest stable timer with it. We do
> - * this with interrupts disabled to avoid latency.
> - */
> - hrtimer_cancel(&vcpu->arch.swtimer);
> -}
> -
> /*
> * Restore soft timer state from saved context.
> */
> @@ -98,6 +85,11 @@ void kvm_restore_timer(struct kvm_vcpu *vcpu)
> return;
> }
>
> + /*
> + * Freeze the soft-timer and sync the guest stable timer with it.
> + */
> + hrtimer_cancel(&vcpu->arch.swtimer);
> +
> /*
> * Set remainder tick value if not expired
> */
> @@ -115,7 +107,7 @@ void kvm_restore_timer(struct kvm_vcpu *vcpu)
> /*
> * Inject timer here though sw timer should inject timer
> * interrupt async already, since sw timer may be cancelled
> - * during injecting intr async in function kvm_acquire_timer
> + * during injecting intr async
> */
> kvm_queue_irq(vcpu, INT_TI);
> }
> @@ -140,11 +132,9 @@ static void _kvm_save_timer(struct kvm_vcpu *vcpu)
> vcpu->arch.expire = expire;
> if (ticks) {
> /*
> - * Update hrtimer to use new timeout
> * HRTIMER_MODE_PINNED is suggested since vcpu may run in
> * the same physical cpu in next time
> */
> - hrtimer_cancel(&vcpu->arch.swtimer);
> hrtimer_start(&vcpu->arch.swtimer, expire, HRTIMER_MODE_ABS_PINNED);
> } else if (vcpu->stat.generic.blocking) {
> /*
> diff --git a/arch/loongarch/kvm/vcpu.c b/arch/loongarch/kvm/vcpu.c
> index 42663a345bd1..cf1c4d64c1b7 100644
> --- a/arch/loongarch/kvm/vcpu.c
> +++ b/arch/loongarch/kvm/vcpu.c
> @@ -95,7 +95,6 @@ static int kvm_pre_enter_guest(struct kvm_vcpu *vcpu)
> * check vmid before vcpu enter guest
> */
> local_irq_disable();
> - kvm_acquire_timer(vcpu);
> kvm_deliver_intr(vcpu);
> kvm_deliver_exception(vcpu);
> /* Make sure the vcpu mode has been written */
> @@ -251,23 +250,6 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
> return -EINVAL;
> }
>
> -/**
> - * kvm_migrate_count() - Migrate timer.
> - * @vcpu: Virtual CPU.
> - *
> - * Migrate hrtimer to the current CPU by cancelling and restarting it
> - * if the hrtimer is active.
> - *
> - * Must be called when the vCPU is migrated to a different CPU, so that
> - * the timer can interrupt the guest at the new CPU, and the timer irq can
> - * be delivered to the vCPU.
> - */
> -static void kvm_migrate_count(struct kvm_vcpu *vcpu)
> -{
> - if (hrtimer_cancel(&vcpu->arch.swtimer))
> - hrtimer_restart(&vcpu->arch.swtimer);
> -}
> -
> static int _kvm_getcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 *val)
> {
> unsigned long gintc;
> @@ -796,17 +778,6 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
> unsigned long flags;
>
> local_irq_save(flags);
> - if (vcpu->arch.last_sched_cpu != cpu) {
> - kvm_debug("[%d->%d]KVM vCPU[%d] switch\n",
> - vcpu->arch.last_sched_cpu, cpu, vcpu->vcpu_id);
> - /*
> - * Migrate the timer interrupt to the current CPU so that it
> - * always interrupts the guest and synchronously triggers a
> - * guest timer interrupt.
> - */
> - kvm_migrate_count(vcpu);
> - }
> -
> /* Restore guest state to registers */
> _kvm_vcpu_load(vcpu, cpu);
> local_irq_restore(flags);