Hi,
The KVM selftests show advantages over KUT in terms of finding defects through
flexible and varied guest settings from the KVM user space.
This patchset tests whether the Intel vPMU works properly with different Intel
CPUID.0xA configurations. It also provides test scaffolding and a sufficient
number of PMU test cases to subsequently offer adequate code coverage of AMD
vPMU or Intel complex features, such as LBR or PEBS, in selftests.
All patches have been tested on both Intel and AMD machines, with one exception
patch 11 "KVM: selftests: Test AMD Guest PerfMonV2" has not been tested on my
AMD machine, as does not support PerfMonV2.
Any feedback or suggestions are greatly appreciated.
Sincerely,
Jinrong Liang
Please note that following patch should be applied before this patch series:
https://lore.kernel.org/kvm/[email protected]/
Changelog:
v3:
- Rebased to 74c2185c5b74(tag: kvm-x86-next-2023.08.02)
- Add a new patch to test AMD PMU legacy four performance counters.
- Add a new patch to test AMD Guest PerfMonV2.
- Refactor code to simplify logic and improve readability.
- Use TEST_ASSERT_EQ() instead of ASSERT_EQ() when checking return values.
- Add vcpu_set_cpuid_property() helper for setting properties. (Sean)
- Add arch_event_is_supported() helper to check if an event is supported. (Sean)
- Add fixed_counter_is_supported() helper to check if a fixed counter is supported. (Sean)
- Drop macros that hides important details. (Sean)
- Use enumerations to avoid performance events magic numbers. (Sean)
- TEST_FAIL() instead of TEST_ASSERT() in run_vcpu() wrapper. (Sean)
- Update variable names for better readability and consistency. (Sean)
- Rename functions to better reflect their purpose. (Sean)
- Improve comments for better clarity and understanding of the code. (Sean, Jim)
v2:
https://lore.kernel.org/kvm/[email protected]/T/
Jinrong Liang (11):
KVM: selftests: Add vcpu_set_cpuid_property() to set properties
KVM: selftests: Add pmu.h for PMU events and common masks
KVM: selftests: Test Intel PMU architectural events on gp counters
KVM: selftests: Test Intel PMU architectural events on fixed counters
KVM: selftests: Test consistency of CPUID with num of gp counters
KVM: selftests: Test consistency of CPUID with num of fixed counters
KVM: selftests: Test Intel supported fixed counters bit mask
KVM: selftests: Test consistency of PMU MSRs with Intel PMU version
KVM: selftests: Add x86 feature and properties for AMD PMU in
processor.h
KVM: selftests: Test AMD PMU events on legacy four performance
counters
KVM: selftests: Test AMD Guest PerfMonV2
tools/testing/selftests/kvm/Makefile | 1 +
.../selftests/kvm/include/x86_64/pmu.h | 124 +++++
.../selftests/kvm/include/x86_64/processor.h | 11 +
.../selftests/kvm/lib/x86_64/processor.c | 14 +
.../kvm/x86_64/pmu_basic_functionality_test.c | 505 ++++++++++++++++++
5 files changed, 655 insertions(+)
create mode 100644 tools/testing/selftests/kvm/include/x86_64/pmu.h
create mode 100644 tools/testing/selftests/kvm/x86_64/pmu_basic_functionality_test.c
base-commit: 74c2185c5b74fd0ae91133ad5afe8684f6a02b91
prerequisite-patch-id: 8718ffb8c05e453db9aae9896787cb6650d3cd52
--
2.39.3
From: Jinrong Liang <[email protected]>
Add a test to check that fixed counters enabled via guest
CPUID.0xA.ECX (instead of EDX[04:00]) work as normal as usual.
Co-developed-by: Like Xu <[email protected]>
Signed-off-by: Like Xu <[email protected]>
Signed-off-by: Jinrong Liang <[email protected]>
---
.../kvm/x86_64/pmu_basic_functionality_test.c | 60 +++++++++++++++++++
1 file changed, 60 insertions(+)
diff --git a/tools/testing/selftests/kvm/x86_64/pmu_basic_functionality_test.c b/tools/testing/selftests/kvm/x86_64/pmu_basic_functionality_test.c
index db1c1230700a..3bbf3bd2846b 100644
--- a/tools/testing/selftests/kvm/x86_64/pmu_basic_functionality_test.c
+++ b/tools/testing/selftests/kvm/x86_64/pmu_basic_functionality_test.c
@@ -282,6 +282,65 @@ static void intel_test_counters_num(void)
}
}
+static void intel_guest_run_fixed_counters(void)
+{
+ uint64_t supported_bitmask = this_cpu_property(X86_PROPERTY_PMU_FIXED_COUNTERS_BITMASK);
+ uint32_t nr_fixed_counter = this_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS);
+ uint64_t msr_val;
+ unsigned int i;
+ bool expected;
+
+ for (i = 0; i < nr_fixed_counter; i++) {
+ expected = supported_bitmask & BIT_ULL(i) || i < nr_fixed_counter;
+
+ wrmsr_safe(MSR_CORE_PERF_FIXED_CTR0 + i, 0);
+ wrmsr_safe(MSR_CORE_PERF_FIXED_CTR_CTRL, BIT_ULL(4 * i));
+ wrmsr_safe(MSR_CORE_PERF_GLOBAL_CTRL, BIT_ULL(INTEL_PMC_IDX_FIXED + i));
+ __asm__ __volatile__("loop ." : "+c"((int){NUM_BRANCHES}));
+ wrmsr_safe(MSR_CORE_PERF_GLOBAL_CTRL, 0);
+ rdmsr_safe(MSR_CORE_PERF_FIXED_CTR0 + i, &msr_val);
+
+ GUEST_ASSERT(expected == !!msr_val);
+ }
+
+ GUEST_DONE();
+}
+
+static void test_fixed_counters_setup(struct kvm_vcpu *vcpu,
+ uint32_t fixed_bitmask,
+ uint8_t edx_fixed_num)
+{
+ int ret;
+
+ vcpu_set_cpuid_property(vcpu, X86_PROPERTY_PMU_FIXED_COUNTERS_BITMASK,
+ fixed_bitmask);
+ vcpu_set_cpuid_property(vcpu, X86_PROPERTY_PMU_NR_FIXED_COUNTERS,
+ edx_fixed_num);
+
+ do {
+ ret = run_vcpu(vcpu, NULL);
+ } while (ret != UCALL_DONE);
+}
+
+static void intel_test_fixed_counters(void)
+{
+ uint8_t nr_fixed_counters = kvm_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS);
+ struct kvm_vcpu *vcpu;
+ struct kvm_vm *vm;
+ uint32_t ecx;
+ uint8_t edx;
+
+ for (edx = 0; edx <= nr_fixed_counters; edx++) {
+ /* KVM doesn't emulate more fixed counters than it can support. */
+ for (ecx = 0; ecx <= (BIT_ULL(nr_fixed_counters) - 1); ecx++) {
+ vm = pmu_vm_create_with_one_vcpu(&vcpu,
+ intel_guest_run_fixed_counters);
+ test_fixed_counters_setup(vcpu, ecx, edx);
+ kvm_vm_free(vm);
+ }
+ }
+}
+
int main(int argc, char *argv[])
{
TEST_REQUIRE(get_kvm_param_bool("enable_pmu"));
@@ -293,6 +352,7 @@ int main(int argc, char *argv[])
intel_test_arch_events();
intel_test_counters_num();
+ intel_test_fixed_counters();
return 0;
}
--
2.39.3
From: Jinrong Liang <[email protected]>
Add test cases to check if different Architectural events are available
after it's marked as unavailable via CPUID. It covers vPMU event filtering
logic based on Intel CPUID, which is a complement to pmu_event_filter.
According to Intel SDM, the number of architectural events is reported
through CPUID.0AH:EAX[31:24] and the architectural event x is supported
if EBX[x]=0 && EAX[31:24]>x.
Co-developed-by: Like Xu <[email protected]>
Signed-off-by: Like Xu <[email protected]>
Signed-off-by: Jinrong Liang <[email protected]>
---
tools/testing/selftests/kvm/Makefile | 1 +
.../kvm/x86_64/pmu_basic_functionality_test.c | 158 ++++++++++++++++++
2 files changed, 159 insertions(+)
create mode 100644 tools/testing/selftests/kvm/x86_64/pmu_basic_functionality_test.c
diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile
index 77026907968f..965a36562ef8 100644
--- a/tools/testing/selftests/kvm/Makefile
+++ b/tools/testing/selftests/kvm/Makefile
@@ -80,6 +80,7 @@ TEST_GEN_PROGS_x86_64 += x86_64/mmio_warning_test
TEST_GEN_PROGS_x86_64 += x86_64/monitor_mwait_test
TEST_GEN_PROGS_x86_64 += x86_64/nested_exceptions_test
TEST_GEN_PROGS_x86_64 += x86_64/platform_info_test
+TEST_GEN_PROGS_x86_64 += x86_64/pmu_basic_functionality_test
TEST_GEN_PROGS_x86_64 += x86_64/pmu_event_filter_test
TEST_GEN_PROGS_x86_64 += x86_64/set_boot_cpu_id
TEST_GEN_PROGS_x86_64 += x86_64/set_sregs_test
diff --git a/tools/testing/selftests/kvm/x86_64/pmu_basic_functionality_test.c b/tools/testing/selftests/kvm/x86_64/pmu_basic_functionality_test.c
new file mode 100644
index 000000000000..c04eb0bdf69f
--- /dev/null
+++ b/tools/testing/selftests/kvm/x86_64/pmu_basic_functionality_test.c
@@ -0,0 +1,158 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Test the consistency of the PMU's CPUID and its features
+ *
+ * Copyright (C) 2023, Tencent, Inc.
+ *
+ * Check that the VM's PMU behaviour is consistent with the
+ * VM CPUID definition.
+ */
+
+#define _GNU_SOURCE /* for program_invocation_short_name */
+#include <x86intrin.h>
+
+#include "pmu.h"
+
+/* Guest payload for any performance counter counting */
+#define NUM_BRANCHES 10
+
+static struct kvm_vm *pmu_vm_create_with_one_vcpu(struct kvm_vcpu **vcpu,
+ void *guest_code)
+{
+ struct kvm_vm *vm;
+
+ vm = vm_create_with_one_vcpu(vcpu, guest_code);
+ vm_init_descriptor_tables(vm);
+ vcpu_init_descriptor_tables(*vcpu);
+
+ return vm;
+}
+
+static uint64_t run_vcpu(struct kvm_vcpu *vcpu, uint64_t *ucall_arg)
+{
+ struct ucall uc;
+
+ vcpu_run(vcpu);
+ switch (get_ucall(vcpu, &uc)) {
+ case UCALL_SYNC:
+ *ucall_arg = uc.args[1];
+ break;
+ case UCALL_DONE:
+ break;
+ default:
+ TEST_FAIL("Unexpected ucall: %lu", uc.cmd);
+ }
+ return uc.cmd;
+}
+
+static void guest_measure_loop(uint64_t event_code)
+{
+ uint32_t nr_gp_counters = this_cpu_property(X86_PROPERTY_PMU_NR_GP_COUNTERS);
+ uint32_t pmu_version = this_cpu_property(X86_PROPERTY_PMU_VERSION);
+ uint32_t counter_msr;
+ unsigned int i;
+
+ if (rdmsr(MSR_IA32_PERF_CAPABILITIES) & PMU_CAP_FW_WRITES)
+ counter_msr = MSR_IA32_PMC0;
+ else
+ counter_msr = MSR_IA32_PERFCTR0;
+
+ for (i = 0; i < nr_gp_counters; i++) {
+ wrmsr(counter_msr + i, 0);
+ wrmsr(MSR_P6_EVNTSEL0 + i, ARCH_PERFMON_EVENTSEL_OS |
+ ARCH_PERFMON_EVENTSEL_ENABLE | event_code);
+
+ if (pmu_version > 1) {
+ wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, BIT_ULL(i));
+ __asm__ __volatile__("loop ." : "+c"((int){NUM_BRANCHES}));
+ wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0);
+ GUEST_SYNC(_rdpmc(i));
+ } else {
+ __asm__ __volatile__("loop ." : "+c"((int){NUM_BRANCHES}));
+ GUEST_SYNC(_rdpmc(i));
+ }
+ }
+
+ GUEST_DONE();
+}
+
+static void test_arch_events_cpuid(struct kvm_vcpu *vcpu,
+ uint8_t arch_events_bitmap_size,
+ uint8_t arch_events_unavailable_mask,
+ uint8_t idx)
+{
+ uint64_t counter_val = 0;
+ bool is_supported;
+
+ vcpu_set_cpuid_property(vcpu, X86_PROPERTY_PMU_EBX_BIT_VECTOR_LENGTH,
+ arch_events_bitmap_size);
+ vcpu_set_cpuid_property(vcpu, X86_PROPERTY_PMU_EVENTS_MASK,
+ arch_events_unavailable_mask);
+
+ is_supported = arch_event_is_supported(vcpu, idx);
+ vcpu_args_set(vcpu, 1, intel_arch_events[idx]);
+
+ while (run_vcpu(vcpu, &counter_val) != UCALL_DONE)
+ TEST_ASSERT_EQ(is_supported, !!counter_val);
+}
+
+static void intel_check_arch_event_is_unavl(uint8_t idx)
+{
+ uint8_t eax_evt_vec, ebx_unavl_mask, i, j;
+ struct kvm_vcpu *vcpu;
+ struct kvm_vm *vm;
+
+ /*
+ * A brute force iteration of all combinations of values is likely to
+ * exhaust the limit of the single-threaded thread fd nums, so it's
+ * tested here by iterating through all valid values on a single bit.
+ */
+ for (i = 0; i < ARRAY_SIZE(intel_arch_events); i++) {
+ eax_evt_vec = BIT_ULL(i);
+ for (j = 0; j < ARRAY_SIZE(intel_arch_events); j++) {
+ ebx_unavl_mask = BIT_ULL(j);
+ vm = pmu_vm_create_with_one_vcpu(&vcpu,
+ guest_measure_loop);
+ test_arch_events_cpuid(vcpu, eax_evt_vec,
+ ebx_unavl_mask, idx);
+
+ kvm_vm_free(vm);
+ }
+ }
+}
+
+static void intel_test_arch_events(void)
+{
+ uint8_t idx;
+
+ for (idx = 0; idx < ARRAY_SIZE(intel_arch_events); idx++) {
+ /*
+ * Given the stability of performance event recurrence,
+ * only these arch events are currently being tested:
+ *
+ * - Core cycle event (idx = 0)
+ * - Instruction retired event (idx = 1)
+ * - Reference cycles event (idx = 2)
+ * - Branch instruction retired event (idx = 5)
+ */
+ if (idx > INTEL_ARCH_INSTRUCTIONS_RETIRED &&
+ idx != INTEL_ARCH_BRANCHES_RETIRED)
+ continue;
+
+ intel_check_arch_event_is_unavl(idx);
+ }
+}
+
+int main(int argc, char *argv[])
+{
+ TEST_REQUIRE(get_kvm_param_bool("enable_pmu"));
+
+ TEST_REQUIRE(host_cpu_is_intel);
+ TEST_REQUIRE(kvm_cpu_has_p(X86_PROPERTY_PMU_VERSION));
+ TEST_REQUIRE(kvm_cpu_property(X86_PROPERTY_PMU_VERSION) > 0);
+ TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_PDCM));
+
+ intel_test_arch_events();
+
+ return 0;
+}
--
2.39.3
From: Jinrong Liang <[email protected]>
By defining the PMU performance events and masks relevant for x86 in
the new pmu.h header, it becomes easier to reference them, minimizing
potential errors in code that handles these values.
Suggested-by: Sean Christopherson <[email protected]>
Signed-off-by: Jinrong Liang <[email protected]>
---
.../selftests/kvm/include/x86_64/pmu.h | 124 ++++++++++++++++++
1 file changed, 124 insertions(+)
create mode 100644 tools/testing/selftests/kvm/include/x86_64/pmu.h
diff --git a/tools/testing/selftests/kvm/include/x86_64/pmu.h b/tools/testing/selftests/kvm/include/x86_64/pmu.h
new file mode 100644
index 000000000000..eb60b2065fac
--- /dev/null
+++ b/tools/testing/selftests/kvm/include/x86_64/pmu.h
@@ -0,0 +1,124 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * tools/testing/selftests/kvm/include/x86_64/pmu.h
+ *
+ * Copyright (C) 2023, Tencent, Inc.
+ */
+#ifndef SELFTEST_KVM_PMU_H
+#define SELFTEST_KVM_PMU_H
+
+#include "processor.h"
+
+#define GP_COUNTER_NR_OFS_BIT 8
+#define EVENT_LENGTH_OFS_BIT 24
+#define INTEL_PMC_IDX_FIXED 32
+
+#define AMD64_NR_COUNTERS 4
+#define AMD64_NR_COUNTERS_CORE 6
+
+#define PMU_CAP_FW_WRITES BIT_ULL(13)
+#define RDPMC_FIXED_BASE BIT_ULL(30)
+
+#define ARCH_PERFMON_EVENTSEL_EVENT GENMASK_ULL(7, 0)
+#define ARCH_PERFMON_EVENTSEL_UMASK GENMASK_ULL(15, 8)
+#define ARCH_PERFMON_EVENTSEL_USR BIT_ULL(16)
+#define ARCH_PERFMON_EVENTSEL_OS BIT_ULL(17)
+
+#define ARCH_PERFMON_EVENTSEL_EDGE BIT_ULL(18)
+#define ARCH_PERFMON_EVENTSEL_PIN_CONTROL BIT_ULL(19)
+#define ARCH_PERFMON_EVENTSEL_INT BIT_ULL(20)
+#define ARCH_PERFMON_EVENTSEL_ANY BIT_ULL(21)
+#define ARCH_PERFMON_EVENTSEL_ENABLE BIT_ULL(22)
+#define ARCH_PERFMON_EVENTSEL_INV BIT_ULL(23)
+#define ARCH_PERFMON_EVENTSEL_CMASK GENMASK_ULL(31, 24)
+
+#define PMU_VERSION_MASK GENMASK_ULL(7, 0)
+#define EVENT_LENGTH_MASK GENMASK_ULL(31, EVENT_LENGTH_OFS_BIT)
+#define GP_COUNTER_NR_MASK GENMASK_ULL(15, GP_COUNTER_NR_OFS_BIT)
+#define FIXED_COUNTER_NR_MASK GENMASK_ULL(4, 0)
+
+/* Definitions for Architectural Performance Events */
+#define ARCH_EVENT(select, umask) (((select) & 0xff) | ((umask) & 0xff) << 8)
+
+enum intel_pmu_architectural_events {
+ /*
+ * The order of the architectural events matters as support for each
+ * event is enumerated via CPUID using the index of the event.
+ */
+ INTEL_ARCH_CPU_CYCLES,
+ INTEL_ARCH_INSTRUCTIONS_RETIRED,
+ INTEL_ARCH_REFERENCE_CYCLES,
+ INTEL_ARCH_LLC_REFERENCES,
+ INTEL_ARCH_LLC_MISSES,
+ INTEL_ARCH_BRANCHES_RETIRED,
+ INTEL_ARCH_BRANCHES_MISPREDICTED,
+
+ NR_REAL_INTEL_ARCH_EVENTS,
+
+ /*
+ * Pseudo-architectural event used to implement IA32_FIXED_CTR2, a.k.a.
+ * TSC reference cycles. The architectural reference cycles event may
+ * or may not actually use the TSC as the reference, e.g. might use the
+ * core crystal clock or the bus clock (yeah, "architectural").
+ */
+ PSEUDO_ARCH_REFERENCE_CYCLES = NR_REAL_INTEL_ARCH_EVENTS,
+ NR_INTEL_ARCH_EVENTS,
+};
+
+static const uint64_t intel_arch_events[] = {
+ [INTEL_ARCH_CPU_CYCLES] = ARCH_EVENT(0x3c, 0x0),
+ [INTEL_ARCH_INSTRUCTIONS_RETIRED] = ARCH_EVENT(0xc0, 0x0),
+ [INTEL_ARCH_REFERENCE_CYCLES] = ARCH_EVENT(0x3c, 0x1),
+ [INTEL_ARCH_LLC_REFERENCES] = ARCH_EVENT(0x2e, 0x4f),
+ [INTEL_ARCH_LLC_MISSES] = ARCH_EVENT(0x2e, 0x41),
+ [INTEL_ARCH_BRANCHES_RETIRED] = ARCH_EVENT(0xc4, 0x0),
+ [INTEL_ARCH_BRANCHES_MISPREDICTED] = ARCH_EVENT(0xc5, 0x0),
+ [PSEUDO_ARCH_REFERENCE_CYCLES] = ARCH_EVENT(0xa4, 0x1),
+};
+
+/* mapping between fixed pmc index and intel_arch_events array */
+static const int fixed_pmc_events[] = {
+ [0] = INTEL_ARCH_INSTRUCTIONS_RETIRED,
+ [1] = INTEL_ARCH_CPU_CYCLES,
+ [2] = PSEUDO_ARCH_REFERENCE_CYCLES,
+};
+
+enum amd_pmu_k7_events {
+ AMD_ZEN_CORE_CYCLES,
+ AMD_ZEN_INSTRUCTIONS,
+ AMD_ZEN_BRANCHES,
+ AMD_ZEN_BRANCH_MISSES,
+};
+
+static const uint64_t amd_arch_events[] = {
+ [AMD_ZEN_CORE_CYCLES] = ARCH_EVENT(0x76, 0x00),
+ [AMD_ZEN_INSTRUCTIONS] = ARCH_EVENT(0xc0, 0x00),
+ [AMD_ZEN_BRANCHES] = ARCH_EVENT(0xc2, 0x00),
+ [AMD_ZEN_BRANCH_MISSES] = ARCH_EVENT(0xc3, 0x00),
+};
+
+static inline bool arch_event_is_supported(struct kvm_vcpu *vcpu,
+ uint8_t arch_event)
+{
+ struct kvm_cpuid_entry2 *entry;
+
+ entry = vcpu_get_cpuid_entry(vcpu, 0xa);
+
+ return !(entry->ebx & BIT_ULL(arch_event)) &&
+ (kvm_cpuid_property(vcpu->cpuid,
+ X86_PROPERTY_PMU_EBX_BIT_VECTOR_LENGTH) > arch_event);
+}
+
+static inline bool fixed_counter_is_supported(struct kvm_vcpu *vcpu,
+ uint8_t fixed_counter_idx)
+{
+ struct kvm_cpuid_entry2 *entry;
+
+ entry = vcpu_get_cpuid_entry(vcpu, 0xa);
+
+ return (entry->ecx & BIT_ULL(fixed_counter_idx) ||
+ (kvm_cpuid_property(vcpu->cpuid, X86_PROPERTY_PMU_NR_FIXED_COUNTERS) >
+ fixed_counter_idx));
+}
+
+#endif /* SELFTEST_KVM_PMU_H */
--
2.39.3