Do some minor housekeeping on the ioctl() macros, and then teach them
to detect and report when an ioctl() unexpectedly fails because KVM has
killed and/or bugged the VM.
Note, I'm 50/50 on whether or not the ARM patch is worthwhile, though I
spent a stupid amount of time on it (don't ask), so darn it I'm at least
posting it.
Oh, and the To: will probably show up funky, but I'd like to take this
through kvm-x86/selftests, not the ARM tree.
Thanks!
Sean Christopherson (4):
KVM: selftests: Drop the single-underscore ioctl() helpers
KVM: selftests: Add helper macros for ioctl()s that return file
descriptors
KVM: selftests: Use asserting kvm_ioctl() macros when getting ARM page
sizes
KVM: selftests: Add logic to detect if ioctl() failed because VM was
killed
.../selftests/kvm/include/kvm_util_base.h | 101 ++++++++++++------
.../selftests/kvm/lib/aarch64/processor.c | 18 ++--
tools/testing/selftests/kvm/lib/kvm_util.c | 17 +--
3 files changed, 84 insertions(+), 52 deletions(-)
base-commit: 240f736891887939571854bd6d734b6c9291f22e
--
2.41.0.585.gd2178a4bd4-goog
Drop _kvm_ioctl(), _vm_ioctl(), and _vcpu_ioctl(), as they are no longer
used by anything other than the no-underscores variants (and may have
never been used directly). The single-underscore variants were never
intended to be a "feature", they were a stopgap of sorts to ease the
conversion to pretty printing ioctl() names when reporting errors.
Opportunistically add a comment explaining when to use __KVM_IOCTL_ERROR()
versus KVM_IOCTL_ERROR(). The single-underscore macros were subtly
ensuring that the name of the ioctl() was printed on error, i.e. it's all
too easy to overlook the fact that using __KVM_IOCTL_ERROR() is
intentional.
Signed-off-by: Sean Christopherson <[email protected]>
---
.../selftests/kvm/include/kvm_util_base.h | 42 +++++++++----------
1 file changed, 19 insertions(+), 23 deletions(-)
diff --git a/tools/testing/selftests/kvm/include/kvm_util_base.h b/tools/testing/selftests/kvm/include/kvm_util_base.h
index 07732a157ccd..90b7739ca204 100644
--- a/tools/testing/selftests/kvm/include/kvm_util_base.h
+++ b/tools/testing/selftests/kvm/include/kvm_util_base.h
@@ -227,6 +227,13 @@ static inline bool kvm_has_cap(long cap)
#define __KVM_SYSCALL_ERROR(_name, _ret) \
"%s failed, rc: %i errno: %i (%s)", (_name), (_ret), errno, strerror(errno)
+/*
+ * Use the "inner", double-underscore macro when reporting errors from within
+ * other macros so that the name of ioctl() and not its literal numeric value
+ * is printed on error. The "outer" macro is strongly preferred when reporting
+ * errors "directly", i.e. without an additional layer of macros, as it reduces
+ * the probability of passing in the wrong string.
+ */
#define __KVM_IOCTL_ERROR(_name, _ret) __KVM_SYSCALL_ERROR(_name, _ret)
#define KVM_IOCTL_ERROR(_ioctl, _ret) __KVM_IOCTL_ERROR(#_ioctl, _ret)
@@ -239,17 +246,13 @@ static inline bool kvm_has_cap(long cap)
#define __kvm_ioctl(kvm_fd, cmd, arg) \
kvm_do_ioctl(kvm_fd, cmd, arg)
-
-#define _kvm_ioctl(kvm_fd, cmd, name, arg) \
+#define kvm_ioctl(kvm_fd, cmd, arg) \
({ \
int ret = __kvm_ioctl(kvm_fd, cmd, arg); \
\
- TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(name, ret)); \
+ TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(#cmd, ret)); \
})
-#define kvm_ioctl(kvm_fd, cmd, arg) \
- _kvm_ioctl(kvm_fd, cmd, #cmd, arg)
-
static __always_inline void static_assert_is_vm(struct kvm_vm *vm) { }
#define __vm_ioctl(vm, cmd, arg) \
@@ -258,16 +261,12 @@ static __always_inline void static_assert_is_vm(struct kvm_vm *vm) { }
kvm_do_ioctl((vm)->fd, cmd, arg); \
})
-#define _vm_ioctl(vm, cmd, name, arg) \
-({ \
- int ret = __vm_ioctl(vm, cmd, arg); \
- \
- TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(name, ret)); \
-})
-
#define vm_ioctl(vm, cmd, arg) \
- _vm_ioctl(vm, cmd, #cmd, arg)
-
+({ \
+ int ret = __vm_ioctl(vm, cmd, arg); \
+ \
+ TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(#cmd, ret)); \
+})
static __always_inline void static_assert_is_vcpu(struct kvm_vcpu *vcpu) { }
@@ -277,15 +276,12 @@ static __always_inline void static_assert_is_vcpu(struct kvm_vcpu *vcpu) { }
kvm_do_ioctl((vcpu)->fd, cmd, arg); \
})
-#define _vcpu_ioctl(vcpu, cmd, name, arg) \
-({ \
- int ret = __vcpu_ioctl(vcpu, cmd, arg); \
- \
- TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(name, ret)); \
-})
-
#define vcpu_ioctl(vcpu, cmd, arg) \
- _vcpu_ioctl(vcpu, cmd, #cmd, arg)
+({ \
+ int ret = __vcpu_ioctl(vcpu, cmd, arg); \
+ \
+ TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(#cmd, ret)); \
+})
/*
* Looks up and returns the value corresponding to the capability
--
2.41.0.585.gd2178a4bd4-goog
Use kvm_ioctl() instead of open coding equivalent ioctl()+TEST_ASSERT()
calls when getting the support page sizes on ARM. The macro usage is a
little funky since the "kvm_fd" parameter implies an actual /dev/kvm fd,
but on the other hand the code is invoking KVM ioctl()s.
Alternatively, the core utilities could expose a vm_open()+vm_close()
pair so that the ARM code could create a dummy, on-stack VM+vCPU pair and
use {vm,vcpu}_ioctl() as appropriate. But the odds of something breaking
due to oddball, partial usage of kvm_vm and kvm_vcpu structures is much
higher than realizing meaningful benefit from using {vm,vcpu}_ioctl().
Signed-off-by: Sean Christopherson <[email protected]>
---
.../selftests/kvm/lib/aarch64/processor.c | 18 ++++++------------
1 file changed, 6 insertions(+), 12 deletions(-)
diff --git a/tools/testing/selftests/kvm/lib/aarch64/processor.c b/tools/testing/selftests/kvm/lib/aarch64/processor.c
index 3a0259e25335..afec1a30916f 100644
--- a/tools/testing/selftests/kvm/lib/aarch64/processor.c
+++ b/tools/testing/selftests/kvm/lib/aarch64/processor.c
@@ -496,7 +496,7 @@ void aarch64_get_supported_page_sizes(uint32_t ipa,
bool *ps4k, bool *ps16k, bool *ps64k)
{
struct kvm_vcpu_init preferred_init;
- int kvm_fd, vm_fd, vcpu_fd, err;
+ int kvm_fd, vm_fd, vcpu_fd;
uint64_t val;
struct kvm_one_reg reg = {
.id = KVM_ARM64_SYS_REG(SYS_ID_AA64MMFR0_EL1),
@@ -504,19 +504,13 @@ void aarch64_get_supported_page_sizes(uint32_t ipa,
};
kvm_fd = open_kvm_dev_path_or_exit();
- vm_fd = __kvm_ioctl(kvm_fd, KVM_CREATE_VM, (void *)(unsigned long)ipa);
- TEST_ASSERT(vm_fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_VM, vm_fd));
+ vm_fd = kvm_fd_ioctl(kvm_fd, KVM_CREATE_VM, (void *)(unsigned long)ipa);
+ vcpu_fd = kvm_fd_ioctl(vm_fd, KVM_CREATE_VCPU, (void *)0ul);
- vcpu_fd = ioctl(vm_fd, KVM_CREATE_VCPU, 0);
- TEST_ASSERT(vcpu_fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_VCPU, vcpu_fd));
+ kvm_ioctl(vm_fd, KVM_ARM_PREFERRED_TARGET, &preferred_init);
+ kvm_ioctl(vcpu_fd, KVM_ARM_VCPU_INIT, &preferred_init);
- err = ioctl(vm_fd, KVM_ARM_PREFERRED_TARGET, &preferred_init);
- TEST_ASSERT(err == 0, KVM_IOCTL_ERROR(KVM_ARM_PREFERRED_TARGET, err));
- err = ioctl(vcpu_fd, KVM_ARM_VCPU_INIT, &preferred_init);
- TEST_ASSERT(err == 0, KVM_IOCTL_ERROR(KVM_ARM_VCPU_INIT, err));
-
- err = ioctl(vcpu_fd, KVM_GET_ONE_REG, ®);
- TEST_ASSERT(err == 0, KVM_IOCTL_ERROR(KVM_GET_ONE_REG, vcpu_fd));
+ kvm_ioctl(vcpu_fd, KVM_GET_ONE_REG, ®);
*ps4k = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR0_TGRAN4), val) != 0xf;
*ps64k = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR0_TGRAN64), val) == 0;
--
2.41.0.585.gd2178a4bd4-goog
Add yet another macro to the VM/vCPU ioctl() framework to detect when an
ioctl() failed because KVM killed/bugged the VM, i.e. when there was
nothing wrong with the ioctl() itself. If KVM kills a VM, e.g. by way of
a failed KVM_BUG_ON(), all subsequent VM and vCPU ioctl()s will fail with
-EIO, which can be quite misleading and ultimately waste user/developer
time.
Use KVM_CHECK_EXTENSION on KVM_CAP_USER_MEMORY to detect if the VM is
dead and/or bug, as KVM doesn't provide a dedicated ioctl(). Using a
heuristic is obviously less than ideal, but practically speaking the logic
is bulletproof barring a KVM change, and any such change would arguably
break userspace, e.g. if KVM returns something other than -EIO.
Without the detection, tearing down a bugged VM yields a cryptic failure
when deleting memslots:
==== Test Assertion Failure ====
lib/kvm_util.c:689: !ret
pid=45131 tid=45131 errno=5 - Input/output error
1 0x00000000004036c3: __vm_mem_region_delete at kvm_util.c:689
2 0x00000000004042f0: kvm_vm_free at kvm_util.c:724 (discriminator 12)
3 0x0000000000402929: race_sync_regs at sync_regs_test.c:193
4 0x0000000000401cab: main at sync_regs_test.c:334 (discriminator 6)
5 0x0000000000416f13: __libc_start_call_main at libc-start.o:?
6 0x000000000041855f: __libc_start_main_impl at ??:?
7 0x0000000000401d40: _start at ??:?
KVM_SET_USER_MEMORY_REGION failed, rc: -1 errno: 5 (Input/output error)
Which morphs into a more pointed error message with the detection:
==== Test Assertion Failure ====
lib/kvm_util.c:689: false
pid=80347 tid=80347 errno=5 - Input/output error
1 0x00000000004039ab: __vm_mem_region_delete at kvm_util.c:689 (discriminator 5)
2 0x0000000000404660: kvm_vm_free at kvm_util.c:724 (discriminator 12)
3 0x0000000000402ac9: race_sync_regs at sync_regs_test.c:193
4 0x0000000000401cb7: main at sync_regs_test.c:334 (discriminator 6)
5 0x0000000000418263: __libc_start_call_main at libc-start.o:?
6 0x00000000004198af: __libc_start_main_impl at ??:?
7 0x0000000000401d90: _start at ??:?
KVM killed/bugged the VM, check the kernel log for clues
Suggested-by: Michal Luczaj <[email protected]>
Signed-off-by: Sean Christopherson <[email protected]>
---
.../selftests/kvm/include/kvm_util_base.h | 34 ++++++++++++++++---
1 file changed, 30 insertions(+), 4 deletions(-)
diff --git a/tools/testing/selftests/kvm/include/kvm_util_base.h b/tools/testing/selftests/kvm/include/kvm_util_base.h
index b35b0bd23683..49ad681d2161 100644
--- a/tools/testing/selftests/kvm/include/kvm_util_base.h
+++ b/tools/testing/selftests/kvm/include/kvm_util_base.h
@@ -269,18 +269,44 @@ static __always_inline void static_assert_is_vm(struct kvm_vm *vm) { }
kvm_do_ioctl((vm)->fd, cmd, arg); \
})
+/*
+ * Assert that a VM or vCPU ioctl() succeeded, with extra magic to detect if
+ * the ioctl() failed because KVM killed/bugged the VM. To detect a dead VM,
+ * probe KVM_CAP_USER_MEMORY, which (a) has been supported by KVM since before
+ * selftests existed and (b) should never outright fail, i.e. is supposed to
+ * return 0 or 1. If KVM kills a VM, KVM returns -EIO for all ioctl()s for the
+ * VM and its vCPUs, including KVM_CHECK_EXTENSION.
+ */
+#define TEST_ASSERT_VM_VCPU_IOCTL(cond, name, ret, vm) \
+do { \
+ int __errno = errno; \
+ \
+ static_assert_is_vm(vm); \
+ \
+ if (cond) \
+ break; \
+ \
+ if (errno == EIO && \
+ __vm_ioctl(vm, KVM_CHECK_EXTENSION, (void *)KVM_CAP_USER_MEMORY) < 0) { \
+ TEST_ASSERT(errno == EIO, "KVM killed the VM, should return -EIO"); \
+ TEST_FAIL("KVM killed/bugged the VM, check the kernel log for clues"); \
+ } \
+ errno = __errno; \
+ TEST_ASSERT(cond, __KVM_IOCTL_ERROR(name, ret)); \
+} while (0)
+
#define vm_ioctl(vm, cmd, arg) \
({ \
int ret = __vm_ioctl(vm, cmd, arg); \
\
- TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(#cmd, ret)); \
+ TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, vm); \
})
#define vm_fd_ioctl(vm, cmd, arg) \
({ \
int fd = __vm_ioctl(vm, cmd, arg); \
\
- TEST_ASSERT(fd >= 0, __KVM_IOCTL_ERROR(#cmd, fd)); \
+ TEST_ASSERT_VM_VCPU_IOCTL(fd >= 0, #cmd, fd, vm); \
fd; \
})
@@ -296,14 +322,14 @@ static __always_inline void static_assert_is_vcpu(struct kvm_vcpu *vcpu) { }
({ \
int ret = __vcpu_ioctl(vcpu, cmd, arg); \
\
- TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(#cmd, ret)); \
+ TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, (vcpu)->vm); \
})
#define vcpu_fd_ioctl(vcpu, cmd, arg) \
({ \
int fd = __vcpu_ioctl(vcpu, cmd, arg); \
\
- TEST_ASSERT(fd >= 0, __KVM_IOCTL_ERROR(#cmd, fd)); \
+ TEST_ASSERT_VM_VCPU_IOCTL(fd >= 0, #cmd, fd, (vcpu)->vm);\
fd; \
})
--
2.41.0.585.gd2178a4bd4-goog
On Fri, Aug 04, 2023, Michal Luczaj wrote:
> On 8/4/23 02:42, Sean Christopherson wrote:
> > Use kvm_ioctl() instead of open coding equivalent ioctl()+TEST_ASSERT()
> > calls when getting the support page sizes on ARM. The macro usage is a
> > little funky since the "kvm_fd" parameter implies an actual /dev/kvm fd,
> > but on the other hand the code is invoking KVM ioctl()s.
> >
> > Alternatively, the core utilities could expose a vm_open()+vm_close()
> > pair so that the ARM code could create a dummy, on-stack VM+vCPU pair and
> > use {vm,vcpu}_ioctl() as appropriate. But the odds of something breaking
> > due to oddball, partial usage of kvm_vm and kvm_vcpu structures is much
> > higher than realizing meaningful benefit from using {vm,vcpu}_ioctl().
>
> Since you're doing the cleanup, does mmio_warning_test qualify for the
> same (funky usage ahead)?
Hmm, I'm heavily leaning towards deleting that test entirely. It's almost
literally a copy+paste of the most basic syzkaller test, e.g. spawn a vCPU with
no backing memory and watch it die a horrible death. Unless I'm missing something,
the test is complete overkill too, e.g. I highly doubt the original KVM bug required
userspace to fork() and whatnot, but syzkaller spawns threads for everything and
so that got copied into the selftest.
And this stuff is just silly:
TEST_REQUIRE(host_cpu_is_intel);
TEST_REQUIRE(!vm_is_unrestricted_guest(NULL));
because crashing the VM doesn't require Intel, nor does it require !URG, those
just happened to be the conditions for the bug.
As much as I like having explicit testcases, adding a new selftest for every bug
that syzkaller finds is neither realistic nor productive. In other words, I think
we should treat syzkaller as being part of KVM's test infrastructure.
I'll send a patch to nuke the test.
> - kvm = open("/dev/kvm", O_RDWR);
> - TEST_ASSERT(kvm != -1, "failed to open /dev/kvm");
> - kvmvm = __kvm_ioctl(kvm, KVM_CREATE_VM, NULL);
> - TEST_ASSERT(kvmvm > 0, KVM_IOCTL_ERROR(KVM_CREATE_VM, kvmvm));
> - kvmcpu = ioctl(kvmvm, KVM_CREATE_VCPU, 0);
> - TEST_ASSERT(kvmcpu != -1, KVM_IOCTL_ERROR(KVM_CREATE_VCPU, kvmcpu));
> + kvm = open_path_or_exit(KVM_DEV_PATH, O_RDWR);
> + kvmvm = kvm_fd_ioctl(kvm, KVM_CREATE_VM, NULL);
> + kvmcpu = kvm_fd_ioctl(kvmvm, KVM_CREATE_VCPU, NULL);
>
> Side note, just in case this wasn't your intention: no kvm@ in cc.
Wasn't intentional, I was moving too fast at the end of the day and missed that
KVM wasn't Cc'd. Grr.
On 8/4/23 02:42, Sean Christopherson wrote:
> Use kvm_ioctl() instead of open coding equivalent ioctl()+TEST_ASSERT()
> calls when getting the support page sizes on ARM. The macro usage is a
> little funky since the "kvm_fd" parameter implies an actual /dev/kvm fd,
> but on the other hand the code is invoking KVM ioctl()s.
>
> Alternatively, the core utilities could expose a vm_open()+vm_close()
> pair so that the ARM code could create a dummy, on-stack VM+vCPU pair and
> use {vm,vcpu}_ioctl() as appropriate. But the odds of something breaking
> due to oddball, partial usage of kvm_vm and kvm_vcpu structures is much
> higher than realizing meaningful benefit from using {vm,vcpu}_ioctl().
Since you're doing the cleanup, does mmio_warning_test qualify for the
same (funky usage ahead)?
- kvm = open("/dev/kvm", O_RDWR);
- TEST_ASSERT(kvm != -1, "failed to open /dev/kvm");
- kvmvm = __kvm_ioctl(kvm, KVM_CREATE_VM, NULL);
- TEST_ASSERT(kvmvm > 0, KVM_IOCTL_ERROR(KVM_CREATE_VM, kvmvm));
- kvmcpu = ioctl(kvmvm, KVM_CREATE_VCPU, 0);
- TEST_ASSERT(kvmcpu != -1, KVM_IOCTL_ERROR(KVM_CREATE_VCPU, kvmcpu));
+ kvm = open_path_or_exit(KVM_DEV_PATH, O_RDWR);
+ kvmvm = kvm_fd_ioctl(kvm, KVM_CREATE_VM, NULL);
+ kvmcpu = kvm_fd_ioctl(kvmvm, KVM_CREATE_VCPU, NULL);
Side note, just in case this wasn't your intention: no kvm@ in cc.
thanks,
Michal
+KVM (good job me)
On Thu, Aug 03, 2023, Sean Christopherson wrote:
> Do some minor housekeeping on the ioctl() macros, and then teach them
> to detect and report when an ioctl() unexpectedly fails because KVM has
> killed and/or bugged the VM.
>
> Note, I'm 50/50 on whether or not the ARM patch is worthwhile, though I
> spent a stupid amount of time on it (don't ask), so darn it I'm at least
> posting it.
>
> Oh, and the To: will probably show up funky, but I'd like to take this
> through kvm-x86/selftests, not the ARM tree.
>
> Thanks!
>
> Sean Christopherson (4):
> KVM: selftests: Drop the single-underscore ioctl() helpers
> KVM: selftests: Add helper macros for ioctl()s that return file
> descriptors
> KVM: selftests: Use asserting kvm_ioctl() macros when getting ARM page
> sizes
> KVM: selftests: Add logic to detect if ioctl() failed because VM was
> killed
>
> .../selftests/kvm/include/kvm_util_base.h | 101 ++++++++++++------
> .../selftests/kvm/lib/aarch64/processor.c | 18 ++--
> tools/testing/selftests/kvm/lib/kvm_util.c | 17 +--
> 3 files changed, 84 insertions(+), 52 deletions(-)
>
>
> base-commit: 240f736891887939571854bd6d734b6c9291f22e
> --
> 2.41.0.585.gd2178a4bd4-goog
>