Add a selftest to get/set triple fault event by:
- exiting to userspace via I/O from L2;
- using KVM_GET_VCPU_EVENTS to verify the TRIPEL_FAULT field is valid
in flags.
- using KVM_SET_VCPU_EVENTS to inject a triple fault event so that L1
can see the appropriate exit.
Suggested-by: Sean Christopherson <[email protected]>
Signed-off-by: Chenyi Qiang <[email protected]>
---
tools/testing/selftests/kvm/.gitignore | 1 +
tools/testing/selftests/kvm/Makefile | 1 +
.../kvm/x86_64/triple_fault_event_test.c | 96 +++++++++++++++++++
3 files changed, 98 insertions(+)
create mode 100644 tools/testing/selftests/kvm/x86_64/triple_fault_event_test.c
diff --git a/tools/testing/selftests/kvm/.gitignore b/tools/testing/selftests/kvm/.gitignore
index 56140068b763..989fd4672f61 100644
--- a/tools/testing/selftests/kvm/.gitignore
+++ b/tools/testing/selftests/kvm/.gitignore
@@ -55,6 +55,7 @@
/x86_64/xen_vmcall_test
/x86_64/xss_msr_test
/x86_64/vmx_pmu_msrs_test
+/x86_64/triple_fault_event_test
/access_tracking_perf_test
/demand_paging_test
/dirty_log_test
diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile
index af582d168621..ce147b50b9be 100644
--- a/tools/testing/selftests/kvm/Makefile
+++ b/tools/testing/selftests/kvm/Makefile
@@ -88,6 +88,7 @@ TEST_GEN_PROGS_x86_64 += x86_64/xen_shinfo_test
TEST_GEN_PROGS_x86_64 += x86_64/xen_vmcall_test
TEST_GEN_PROGS_x86_64 += x86_64/sev_migrate_tests
TEST_GEN_PROGS_x86_64 += x86_64/amx_test
+TEST_GEN_PROGS_x86_64 += x86_64/triple_fault_event_test
TEST_GEN_PROGS_x86_64 += access_tracking_perf_test
TEST_GEN_PROGS_x86_64 += demand_paging_test
TEST_GEN_PROGS_x86_64 += dirty_log_test
diff --git a/tools/testing/selftests/kvm/x86_64/triple_fault_event_test.c b/tools/testing/selftests/kvm/x86_64/triple_fault_event_test.c
new file mode 100644
index 000000000000..20bf8f9a61d7
--- /dev/null
+++ b/tools/testing/selftests/kvm/x86_64/triple_fault_event_test.c
@@ -0,0 +1,96 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include "test_util.h"
+#include "kvm_util.h"
+#include "processor.h"
+#include "vmx.h"
+
+#include <string.h>
+#include <sys/ioctl.h>
+
+#include "kselftest.h"
+
+#ifndef __x86_64__
+# error This test is 64-bit only
+#endif
+
+#define VCPU_ID 0
+#define ARBITRARY_IO_PORT 0x2000
+
+/* The virtual machine object. */
+static struct kvm_vm *vm;
+
+static void l2_guest_code(void)
+{
+ /*
+ * Generate an exit to L0 userspace, i.e. main(), via I/O to an
+ * arbitrary port.
+ */
+ asm volatile("inb %%dx, %%al"
+ : : [port] "d" (ARBITRARY_IO_PORT) : "rax");
+}
+
+void l1_guest_code(struct vmx_pages *vmx)
+{
+#define L2_GUEST_STACK_SIZE 64
+ unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
+
+ GUEST_ASSERT(vmx->vmcs_gpa);
+ GUEST_ASSERT(prepare_for_vmx_operation(vmx));
+ GUEST_ASSERT(load_vmcs(vmx));
+
+ prepare_vmcs(vmx, l2_guest_code,
+ &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+
+ GUEST_ASSERT(!vmlaunch());
+ /* L2 should triple fault after a triple fault event injected. */
+ GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_TRIPLE_FAULT);
+ GUEST_DONE();
+}
+
+int main(void)
+{
+ struct kvm_run *run;
+ struct kvm_vcpu_events events;
+ vm_vaddr_t vmx_pages_gva;
+ struct ucall uc;
+
+ if (!nested_vmx_supported()) {
+ print_skip("Nested VMX not supported");
+ exit(KSFT_SKIP);
+ }
+
+ vm = vm_create_default(VCPU_ID, 0, (void *) l1_guest_code);
+
+ vcpu_alloc_vmx(vm, &vmx_pages_gva);
+ vcpu_args_set(vm, VCPU_ID, 1, vmx_pages_gva);
+
+ vcpu_run(vm, VCPU_ID);
+
+ run = vcpu_state(vm, VCPU_ID);
+
+ TEST_ASSERT(run->exit_reason == KVM_EXIT_IO,
+ "Expected KVM_EXIT_IO, got: %u (%s)\n",
+ run->exit_reason, exit_reason_str(run->exit_reason));
+
+ TEST_ASSERT(run->io.port == ARBITRARY_IO_PORT,
+ "Expected IN from port %d from L2, got port %d",
+ ARBITRARY_IO_PORT, run->io.port);
+
+ vcpu_events_get(vm, VCPU_ID, &events);
+ TEST_ASSERT(events.flags & KVM_VCPUEVENT_VALID_TRIPLE_FAULT,
+ "Triple fault event invalid");
+ events.triple_fault_pending = true;
+ vcpu_events_set(vm, VCPU_ID, &events);
+
+ vcpu_run(vm, VCPU_ID);
+
+ switch (get_ucall(vm, VCPU_ID, &uc)) {
+ case UCALL_DONE:
+ break;
+ case UCALL_ABORT:
+ TEST_FAIL("%s", (const char *)uc.args[0]);
+ default:
+ TEST_FAIL("Unexpected ucall: %lu", uc.cmd);
+ }
+
+}
--
2.17.1
On Thu, Apr 21, 2022, Chenyi Qiang wrote:
> +#ifndef __x86_64__
> +# error This test is 64-bit only
No need, all of KVM selftests are 64-bit only.
> +#endif
> +
> +#define VCPU_ID 0
> +#define ARBITRARY_IO_PORT 0x2000
> +
> +/* The virtual machine object. */
> +static struct kvm_vm *vm;
> +
> +static void l2_guest_code(void)
> +{
> + /*
> + * Generate an exit to L0 userspace, i.e. main(), via I/O to an
> + * arbitrary port.
> + */
I think we can test a "real" triple fault without too much effort by abusing
vcpu->run->request_interrupt_window. E.g. map the run struct into L2, clear
PIN_BASED_EXT_INTR_MASK in vmcs12, and then in l2_guest_code() do:
asm volatile("cli");
run->request_interrupt_window = true;
asm volatile("sti; ud2");
asm volatile("inb %%dx, %%al"
: : [port] "d" (ARBITRARY_IO_PORT) : "rax");
The STI shadow will block the IRQ-window VM-Exit until after the ud2, i.e. until
after the triple fault is pending.
And then main() can
1. verify it got KVM_EXIT_IRQ_WINDOW_OPEN with a pending triple fault
2. clear the triple fault and re-enter L1+l2
3. continue with the existing code, e.g. verify it got KVM_EXIT_IO, stuff a
pending triple fault, etc...
That said, typing that out makes me think we should technically handle/prevent this
in KVM since triple fault / shutdown is kinda sorta just a special exception.
Heh, and a potentially bad/crazy idea would be to use a reserved/magic vector in
kvm_vcpu_events.exception to save/restore triple fault, e.g. we could steal
NMI_VECTOR.
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index f2d0ee9296b9..d58c0cfd3cd3 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -4743,7 +4743,8 @@ static int kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu *vcpu)
return (kvm_arch_interrupt_allowed(vcpu) &&
kvm_cpu_accept_dm_intr(vcpu) &&
!kvm_event_needs_reinjection(vcpu) &&
- !vcpu->arch.exception.pending);
+ !vcpu->arch.exception.pending &&
+ !kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu));
}
static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
On 5/19/2022 3:20 AM, Sean Christopherson wrote:
> On Thu, Apr 21, 2022, Chenyi Qiang wrote:
>> +#ifndef __x86_64__
>> +# error This test is 64-bit only
>
> No need, all of KVM selftests are 64-bit only.
>
>> +#endif
>> +
>> +#define VCPU_ID 0
>> +#define ARBITRARY_IO_PORT 0x2000
>> +
>> +/* The virtual machine object. */
>> +static struct kvm_vm *vm;
>> +
>> +static void l2_guest_code(void)
>> +{
>> + /*
>> + * Generate an exit to L0 userspace, i.e. main(), via I/O to an
>> + * arbitrary port.
>> + */
>
> I think we can test a "real" triple fault without too much effort by abusing
> vcpu->run->request_interrupt_window. E.g. map the run struct into L2, clear
> PIN_BASED_EXT_INTR_MASK in vmcs12, and then in l2_guest_code() do:
>
> asm volatile("cli");
>
> run->request_interrupt_window = true;
>
Maybe, A GUEST_SYNC to main() to change the request_interrupt_window
also works.
> asm volatile("sti; ud2");
>
How does the "real" triple fault occur here? Through UD2?
> asm volatile("inb %%dx, %%al"
> : : [port] "d" (ARBITRARY_IO_PORT) : "rax");
>
> The STI shadow will block the IRQ-window VM-Exit until after the ud2, i.e. until
> after the triple fault is pending.
>
> And then main() can
>
> 1. verify it got KVM_EXIT_IRQ_WINDOW_OPEN with a pending triple fault
> 2. clear the triple fault and re-enter L1+l2
> 3. continue with the existing code, e.g. verify it got KVM_EXIT_IO, stuff a
> pending triple fault, etc...
>
> That said, typing that out makes me think we should technically handle/prevent this
> in KVM since triple fault / shutdown is kinda sorta just a special exception.
>
> Heh, and a potentially bad/crazy idea would be to use a reserved/magic vector in
> kvm_vcpu_events.exception to save/restore triple fault, e.g. we could steal
> NMI_VECTOR.
>
> diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
> index f2d0ee9296b9..d58c0cfd3cd3 100644
> --- a/arch/x86/kvm/x86.c
> +++ b/arch/x86/kvm/x86.c
> @@ -4743,7 +4743,8 @@ static int kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu *vcpu)
> return (kvm_arch_interrupt_allowed(vcpu) &&
> kvm_cpu_accept_dm_intr(vcpu) &&
> !kvm_event_needs_reinjection(vcpu) &&
> - !vcpu->arch.exception.pending);
> + !vcpu->arch.exception.pending &&
> + !kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu));
> }
>
> static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
>
>
>
On Mon, May 23, 2022, Chenyi Qiang wrote:
>
>
> On 5/19/2022 3:20 AM, Sean Christopherson wrote:
> > On Thu, Apr 21, 2022, Chenyi Qiang wrote:
> > > +#ifndef __x86_64__
> > > +# error This test is 64-bit only
> >
> > No need, all of KVM selftests are 64-bit only.
> >
> > > +#endif
> > > +
> > > +#define VCPU_ID 0
> > > +#define ARBITRARY_IO_PORT 0x2000
> > > +
> > > +/* The virtual machine object. */
> > > +static struct kvm_vm *vm;
> > > +
> > > +static void l2_guest_code(void)
> > > +{
> > > + /*
> > > + * Generate an exit to L0 userspace, i.e. main(), via I/O to an
> > > + * arbitrary port.
> > > + */
> >
> > I think we can test a "real" triple fault without too much effort by abusing
> > vcpu->run->request_interrupt_window. E.g. map the run struct into L2, clear
> > PIN_BASED_EXT_INTR_MASK in vmcs12, and then in l2_guest_code() do:
> >
> > asm volatile("cli");
> >
> > run->request_interrupt_window = true;
> >
>
> Maybe, A GUEST_SYNC to main() to change the request_interrupt_window also
> works.
Hmm, yes, that should work too. Feel free to punt on writing this sub-test. As
mentioned below, KVM should treat a pending triple fault as a pending "exception",
i.e. userspace shouldn't see KVM_EXIT_IRQ_WINDOW_OPEN with a pending triple fault,
and so the test should actually assert that the triple fault occurs in L2. But I
can roll that test into the KVM fix if you'd like.
> > asm volatile("sti; ud2");
> >
>
> How does the "real" triple fault occur here? Through UD2?
Yeah. IIRC, by default L2 doesn't have a valid IDT, so any fault will escalate to
a triple fault.
> > asm volatile("inb %%dx, %%al"
> > : : [port] "d" (ARBITRARY_IO_PORT) : "rax");
> >
> > The STI shadow will block the IRQ-window VM-Exit until after the ud2, i.e. until
> > after the triple fault is pending.
> >
> > And then main() can
> >
> > 1. verify it got KVM_EXIT_IRQ_WINDOW_OPEN with a pending triple fault
> > 2. clear the triple fault and re-enter L1+l2
> > 3. continue with the existing code, e.g. verify it got KVM_EXIT_IO, stuff a
> > pending triple fault, etc...
> >
> > That said, typing that out makes me think we should technically handle/prevent this
> > in KVM since triple fault / shutdown is kinda sorta just a special exception.
On 5/24/2022 12:23 AM, Sean Christopherson wrote:
> On Mon, May 23, 2022, Chenyi Qiang wrote:
>>
>>
>> On 5/19/2022 3:20 AM, Sean Christopherson wrote:
>>> On Thu, Apr 21, 2022, Chenyi Qiang wrote:
>>>> +#ifndef __x86_64__
>>>> +# error This test is 64-bit only
>>>
>>> No need, all of KVM selftests are 64-bit only.
>>>
>>>> +#endif
>>>> +
>>>> +#define VCPU_ID 0
>>>> +#define ARBITRARY_IO_PORT 0x2000
>>>> +
>>>> +/* The virtual machine object. */
>>>> +static struct kvm_vm *vm;
>>>> +
>>>> +static void l2_guest_code(void)
>>>> +{
>>>> + /*
>>>> + * Generate an exit to L0 userspace, i.e. main(), via I/O to an
>>>> + * arbitrary port.
>>>> + */
>>>
>>> I think we can test a "real" triple fault without too much effort by abusing
>>> vcpu->run->request_interrupt_window. E.g. map the run struct into L2, clear
>>> PIN_BASED_EXT_INTR_MASK in vmcs12, and then in l2_guest_code() do:
>>>
>>> asm volatile("cli");
>>>
>>> run->request_interrupt_window = true;
>>>
>>
>> Maybe, A GUEST_SYNC to main() to change the request_interrupt_window also
>> works.
>
> Hmm, yes, that should work too. Feel free to punt on writing this sub-test. As
> mentioned below, KVM should treat a pending triple fault as a pending "exception",
> i.e. userspace shouldn't see KVM_EXIT_IRQ_WINDOW_OPEN with a pending triple fault,
> and so the test should actually assert that the triple fault occurs in L2. But I
> can roll that test into the KVM fix if you'd like.
If the KVM_EXIT_IRQ_WINDOW_OPEN can't see the pending triple fault, only
asserting the triple fault may not have meaning in this selftest.
Anyway, feel free to add the pending triple fault test in your KVM fix
patches.
>
>>> asm volatile("sti; ud2");
>>>
>>
>> How does the "real" triple fault occur here? Through UD2?
>
> Yeah. IIRC, by default L2 doesn't have a valid IDT, so any fault will escalate to
> a triple fault.
>
>>> asm volatile("inb %%dx, %%al"
>>> : : [port] "d" (ARBITRARY_IO_PORT) : "rax");
>>>
>>> The STI shadow will block the IRQ-window VM-Exit until after the ud2, i.e. until
>>> after the triple fault is pending.
>>>
>>> And then main() can
>>>
>>> 1. verify it got KVM_EXIT_IRQ_WINDOW_OPEN with a pending triple fault
>>> 2. clear the triple fault and re-enter L1+l2
>>> 3. continue with the existing code, e.g. verify it got KVM_EXIT_IO, stuff a
>>> pending triple fault, etc...
>>>
>>> That said, typing that out makes me think we should technically handle/prevent this
>>> in KVM since triple fault / shutdown is kinda sorta just a special exception.