Hi,
This is a successor to a previous patch set from Jason Baron. Let's convert
kvm_pmu_ops to use static_call. Shows good performance gains for
a typical perf use case [2] in the guest (results in patch 3/3).
[1] https://lore.kernel.org/lkml/[email protected]/
[2] perf record -e branch-instructions -e branch-misses \
-e cache-misses -e cache-references -e cpu-cycles \
-e instructions ./workload
Thanks,
Like Xu (3):
KVM: x86: Copy kvm_pmu_ops by value to eliminate layer of indirection
KVM: x86: Introduce definitions to support static calls for
kvm_pmu_ops
KVM: x86: Use static calls to reduce kvm_pmu_ops overhead
arch/x86/include/asm/kvm-x86-pmu-ops.h | 32 ++++++++++++++++++
arch/x86/kvm/pmu.c | 46 +++++++++++++++-----------
arch/x86/kvm/pmu.h | 19 ++++++++++-
arch/x86/kvm/vmx/nested.c | 2 +-
arch/x86/kvm/x86.c | 5 +++
5 files changed, 83 insertions(+), 21 deletions(-)
create mode 100644 arch/x86/include/asm/kvm-x86-pmu-ops.h
--
2.33.0
Replace the kvm_pmu_ops pointer in common x86 with an instance of the
struct to save one pointer dereference when invoking functions. Copy the
struct by value to set the ops during kvm_init().
Using kvm_x86_ops.hardware_enable to track whether or not the
ops have been initialized, i.e. a vendor KVM module has been loaded.
Signed-off-by: Like Xu <[email protected]>
---
arch/x86/kvm/pmu.c | 41 +++++++++++++++++++++------------------
arch/x86/kvm/pmu.h | 4 +++-
arch/x86/kvm/vmx/nested.c | 2 +-
arch/x86/kvm/x86.c | 3 +++
4 files changed, 29 insertions(+), 21 deletions(-)
diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
index 0772bad9165c..0db1887137d9 100644
--- a/arch/x86/kvm/pmu.c
+++ b/arch/x86/kvm/pmu.c
@@ -47,6 +47,9 @@
* * AMD: [0 .. AMD64_NUM_COUNTERS-1] <=> gp counters
*/
+struct kvm_pmu_ops kvm_pmu_ops __read_mostly;
+EXPORT_SYMBOL_GPL(kvm_pmu_ops);
+
static void kvm_pmi_trigger_fn(struct irq_work *irq_work)
{
struct kvm_pmu *pmu = container_of(irq_work, struct kvm_pmu, irq_work);
@@ -214,7 +217,7 @@ void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel)
ARCH_PERFMON_EVENTSEL_CMASK |
HSW_IN_TX |
HSW_IN_TX_CHECKPOINTED))) {
- config = kvm_x86_ops.pmu_ops->find_arch_event(pmc_to_pmu(pmc),
+ config = kvm_pmu_ops.find_arch_event(pmc_to_pmu(pmc),
event_select,
unit_mask);
if (config != PERF_COUNT_HW_MAX)
@@ -268,7 +271,7 @@ void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 ctrl, int idx)
pmc->current_config = (u64)ctrl;
pmc_reprogram_counter(pmc, PERF_TYPE_HARDWARE,
- kvm_x86_ops.pmu_ops->find_fixed_event(idx),
+ kvm_pmu_ops.find_fixed_event(idx),
!(en_field & 0x2), /* exclude user */
!(en_field & 0x1), /* exclude kernel */
pmi, false, false);
@@ -277,7 +280,7 @@ EXPORT_SYMBOL_GPL(reprogram_fixed_counter);
void reprogram_counter(struct kvm_pmu *pmu, int pmc_idx)
{
- struct kvm_pmc *pmc = kvm_x86_ops.pmu_ops->pmc_idx_to_pmc(pmu, pmc_idx);
+ struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, pmc_idx);
if (!pmc)
return;
@@ -299,7 +302,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)
int bit;
for_each_set_bit(bit, pmu->reprogram_pmi, X86_PMC_IDX_MAX) {
- struct kvm_pmc *pmc = kvm_x86_ops.pmu_ops->pmc_idx_to_pmc(pmu, bit);
+ struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, bit);
if (unlikely(!pmc || !pmc->perf_event)) {
clear_bit(bit, pmu->reprogram_pmi);
@@ -321,7 +324,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)
/* check if idx is a valid index to access PMU */
int kvm_pmu_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx)
{
- return kvm_x86_ops.pmu_ops->is_valid_rdpmc_ecx(vcpu, idx);
+ return kvm_pmu_ops.is_valid_rdpmc_ecx(vcpu, idx);
}
bool is_vmware_backdoor_pmc(u32 pmc_idx)
@@ -371,7 +374,7 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
if (is_vmware_backdoor_pmc(idx))
return kvm_pmu_rdpmc_vmware(vcpu, idx, data);
- pmc = kvm_x86_ops.pmu_ops->rdpmc_ecx_to_pmc(vcpu, idx, &mask);
+ pmc = kvm_pmu_ops.rdpmc_ecx_to_pmc(vcpu, idx, &mask);
if (!pmc)
return 1;
@@ -387,22 +390,22 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu)
{
if (lapic_in_kernel(vcpu)) {
- if (kvm_x86_ops.pmu_ops->deliver_pmi)
- kvm_x86_ops.pmu_ops->deliver_pmi(vcpu);
+ if (kvm_pmu_ops.deliver_pmi)
+ kvm_pmu_ops.deliver_pmi(vcpu);
kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC);
}
}
bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr)
{
- return kvm_x86_ops.pmu_ops->msr_idx_to_pmc(vcpu, msr) ||
- kvm_x86_ops.pmu_ops->is_valid_msr(vcpu, msr);
+ return kvm_pmu_ops.msr_idx_to_pmc(vcpu, msr) ||
+ kvm_pmu_ops.is_valid_msr(vcpu, msr);
}
static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr)
{
struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
- struct kvm_pmc *pmc = kvm_x86_ops.pmu_ops->msr_idx_to_pmc(vcpu, msr);
+ struct kvm_pmc *pmc = kvm_pmu_ops.msr_idx_to_pmc(vcpu, msr);
if (pmc)
__set_bit(pmc->idx, pmu->pmc_in_use);
@@ -410,13 +413,13 @@ static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr)
int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
{
- return kvm_x86_ops.pmu_ops->get_msr(vcpu, msr_info);
+ return kvm_pmu_ops.get_msr(vcpu, msr_info);
}
int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
{
kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index);
- return kvm_x86_ops.pmu_ops->set_msr(vcpu, msr_info);
+ return kvm_pmu_ops.set_msr(vcpu, msr_info);
}
/* refresh PMU settings. This function generally is called when underlying
@@ -425,7 +428,7 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
*/
void kvm_pmu_refresh(struct kvm_vcpu *vcpu)
{
- kvm_x86_ops.pmu_ops->refresh(vcpu);
+ kvm_pmu_ops.refresh(vcpu);
}
void kvm_pmu_reset(struct kvm_vcpu *vcpu)
@@ -433,7 +436,7 @@ void kvm_pmu_reset(struct kvm_vcpu *vcpu)
struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
irq_work_sync(&pmu->irq_work);
- kvm_x86_ops.pmu_ops->reset(vcpu);
+ kvm_pmu_ops.reset(vcpu);
}
void kvm_pmu_init(struct kvm_vcpu *vcpu)
@@ -441,7 +444,7 @@ void kvm_pmu_init(struct kvm_vcpu *vcpu)
struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
memset(pmu, 0, sizeof(*pmu));
- kvm_x86_ops.pmu_ops->init(vcpu);
+ kvm_pmu_ops.init(vcpu);
init_irq_work(&pmu->irq_work, kvm_pmi_trigger_fn);
pmu->event_count = 0;
pmu->need_cleanup = false;
@@ -473,14 +476,14 @@ void kvm_pmu_cleanup(struct kvm_vcpu *vcpu)
pmu->pmc_in_use, X86_PMC_IDX_MAX);
for_each_set_bit(i, bitmask, X86_PMC_IDX_MAX) {
- pmc = kvm_x86_ops.pmu_ops->pmc_idx_to_pmc(pmu, i);
+ pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, i);
if (pmc && pmc->perf_event && !pmc_speculative_in_use(pmc))
pmc_stop_counter(pmc);
}
- if (kvm_x86_ops.pmu_ops->cleanup)
- kvm_x86_ops.pmu_ops->cleanup(vcpu);
+ if (kvm_pmu_ops.cleanup)
+ kvm_pmu_ops.cleanup(vcpu);
bitmap_zero(pmu->pmc_in_use, X86_PMC_IDX_MAX);
}
diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h
index 0e4f2b1fa9fb..b2fe135d395a 100644
--- a/arch/x86/kvm/pmu.h
+++ b/arch/x86/kvm/pmu.h
@@ -17,6 +17,8 @@
#define MAX_FIXED_COUNTERS 3
+extern struct kvm_pmu_ops kvm_pmu_ops;
+
struct kvm_event_hw_type_mapping {
u8 eventsel;
u8 unit_mask;
@@ -92,7 +94,7 @@ static inline bool pmc_is_fixed(struct kvm_pmc *pmc)
static inline bool pmc_is_enabled(struct kvm_pmc *pmc)
{
- return kvm_x86_ops.pmu_ops->pmc_is_enabled(pmc);
+ return kvm_pmu_ops.pmc_is_enabled(pmc);
}
static inline bool kvm_valid_perf_global_ctrl(struct kvm_pmu *pmu,
diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index b4ee5e9f9e20..1e793e44b5ff 100644
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -4796,7 +4796,7 @@ void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu)
return;
vmx = to_vmx(vcpu);
- if (kvm_x86_ops.pmu_ops->is_valid_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) {
+ if (kvm_pmu_ops.is_valid_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) {
vmx->nested.msrs.entry_ctls_high |=
VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
vmx->nested.msrs.exit_ctls_high |=
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index ac83d873d65b..72d286595012 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -11317,6 +11317,9 @@ int kvm_arch_hardware_setup(void *opaque)
memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));
kvm_ops_static_call_update();
+ if (kvm_x86_ops.hardware_enable)
+ memcpy(&kvm_pmu_ops, kvm_x86_ops.pmu_ops, sizeof(kvm_pmu_ops));
+
if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))
supported_xss = 0;
--
2.33.0
Use static calls to improve kvm_pmu_ops performance. Introduce the
definitions that will be used by a subsequent patch to actualize the
savings. Add a new kvm-x86-pmu-ops.h header that can be used for the
definition of static calls. This header is also intended to be
used to simplify the defition of amd_pmu_ops and intel_pmu_ops.
Like what we did for kvm_x86_ops, 'pmu_ops' can be covered by
static calls in a simlilar manner for insignificant but not
negligible performance impact, especially on older models.
Signed-off-by: Like Xu <[email protected]>
---
arch/x86/include/asm/kvm-x86-pmu-ops.h | 32 ++++++++++++++++++++++++++
arch/x86/kvm/pmu.c | 7 ++++++
arch/x86/kvm/pmu.h | 15 ++++++++++++
3 files changed, 54 insertions(+)
create mode 100644 arch/x86/include/asm/kvm-x86-pmu-ops.h
diff --git a/arch/x86/include/asm/kvm-x86-pmu-ops.h b/arch/x86/include/asm/kvm-x86-pmu-ops.h
new file mode 100644
index 000000000000..b7713b16d21d
--- /dev/null
+++ b/arch/x86/include/asm/kvm-x86-pmu-ops.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#if !defined(KVM_X86_PMU_OP) || !defined(KVM_X86_PMU_OP_NULL)
+BUILD_BUG_ON(1)
+#endif
+
+/*
+ * KVM_X86_PMU_OP() and KVM_X86_PMU_OP_NULL() are used to
+ * help generate "static_call()"s. They are also intended for use when defining
+ * the amd/intel KVM_X86_PMU_OPs. KVM_X86_PMU_OP() can be used
+ * for those functions that follow the [amd|intel]_func_name convention.
+ * KVM_X86_PMU_OP_NULL() can leave a NULL definition for the
+ * case where there is no definition or a function name that
+ * doesn't match the typical naming convention is supplied.
+ */
+KVM_X86_PMU_OP(find_arch_event);
+KVM_X86_PMU_OP(find_fixed_event);
+KVM_X86_PMU_OP(pmc_is_enabled);
+KVM_X86_PMU_OP(pmc_idx_to_pmc);
+KVM_X86_PMU_OP(rdpmc_ecx_to_pmc);
+KVM_X86_PMU_OP(msr_idx_to_pmc);
+KVM_X86_PMU_OP(is_valid_rdpmc_ecx);
+KVM_X86_PMU_OP(is_valid_msr);
+KVM_X86_PMU_OP(get_msr);
+KVM_X86_PMU_OP(set_msr);
+KVM_X86_PMU_OP(refresh);
+KVM_X86_PMU_OP(init);
+KVM_X86_PMU_OP(reset);
+KVM_X86_PMU_OP_NULL(deliver_pmi);
+KVM_X86_PMU_OP_NULL(cleanup);
+
+#undef KVM_X86_PMU_OP
+#undef KVM_X86_PMU_OP_NULL
diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
index 0db1887137d9..b6f08c719125 100644
--- a/arch/x86/kvm/pmu.c
+++ b/arch/x86/kvm/pmu.c
@@ -50,6 +50,13 @@
struct kvm_pmu_ops kvm_pmu_ops __read_mostly;
EXPORT_SYMBOL_GPL(kvm_pmu_ops);
+#define KVM_X86_PMU_OP(func) \
+ DEFINE_STATIC_CALL_NULL(kvm_x86_pmu_##func, \
+ *(((struct kvm_pmu_ops *)0)->func))
+#define KVM_X86_PMU_OP_NULL KVM_X86_PMU_OP
+#include <asm/kvm-x86-pmu-ops.h>
+EXPORT_STATIC_CALL_GPL(kvm_x86_pmu_is_valid_msr);
+
static void kvm_pmi_trigger_fn(struct irq_work *irq_work)
{
struct kvm_pmu *pmu = container_of(irq_work, struct kvm_pmu, irq_work);
diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h
index b2fe135d395a..e5550d4acf14 100644
--- a/arch/x86/kvm/pmu.h
+++ b/arch/x86/kvm/pmu.h
@@ -3,6 +3,8 @@
#define __KVM_X86_PMU_H
#include <linux/nospec.h>
+#include <linux/static_call_types.h>
+#include <linux/static_call.h>
#define vcpu_to_pmu(vcpu) (&(vcpu)->arch.pmu)
#define pmu_to_vcpu(pmu) (container_of((pmu), struct kvm_vcpu, arch.pmu))
@@ -45,6 +47,19 @@ struct kvm_pmu_ops {
void (*cleanup)(struct kvm_vcpu *vcpu);
};
+#define KVM_X86_PMU_OP(func) \
+ DECLARE_STATIC_CALL(kvm_x86_pmu_##func, *(((struct kvm_pmu_ops *)0)->func))
+#define KVM_X86_PMU_OP_NULL KVM_X86_PMU_OP
+#include <asm/kvm-x86-pmu-ops.h>
+
+static inline void kvm_pmu_ops_static_call_update(void)
+{
+#define KVM_X86_PMU_OP(func) \
+ static_call_update(kvm_x86_pmu_##func, kvm_pmu_ops.func)
+#define KVM_X86_PMU_OP_NULL KVM_X86_PMU_OP
+#include <asm/kvm-x86-pmu-ops.h>
+}
+
static inline u64 pmc_bitmask(struct kvm_pmc *pmc)
{
struct kvm_pmu *pmu = pmc_to_pmu(pmc);
--
2.33.0
Convert kvm_pmu_ops to use static calls.
Here are the worst sched_clock() nanosecond numbers for the kvm_pmu_ops
functions that is most often called (up to 7 digits of calls) when running
a single perf test case in a guest on an ICX 2.70GHz host (mitigations=on):
| legacy | static call
------------------------------------------------------------
.pmc_idx_to_pmc | 10946 | 10047 (8%)
.pmc_is_enabled | 11291 | 11175 (1%)
.msr_idx_to_pmc | 13526 | 12346 (8%)
.is_valid_msr | 10895 | 10484 (3%)
Signed-off-by: Like Xu <[email protected]>
---
arch/x86/kvm/pmu.c | 36 +++++++++++++++++-------------------
arch/x86/kvm/pmu.h | 2 +-
arch/x86/kvm/vmx/nested.c | 2 +-
arch/x86/kvm/x86.c | 4 +++-
4 files changed, 22 insertions(+), 22 deletions(-)
diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
index b6f08c719125..193f925e2064 100644
--- a/arch/x86/kvm/pmu.c
+++ b/arch/x86/kvm/pmu.c
@@ -224,7 +224,7 @@ void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel)
ARCH_PERFMON_EVENTSEL_CMASK |
HSW_IN_TX |
HSW_IN_TX_CHECKPOINTED))) {
- config = kvm_pmu_ops.find_arch_event(pmc_to_pmu(pmc),
+ config = static_call(kvm_x86_pmu_find_arch_event)(pmc_to_pmu(pmc),
event_select,
unit_mask);
if (config != PERF_COUNT_HW_MAX)
@@ -278,7 +278,7 @@ void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 ctrl, int idx)
pmc->current_config = (u64)ctrl;
pmc_reprogram_counter(pmc, PERF_TYPE_HARDWARE,
- kvm_pmu_ops.find_fixed_event(idx),
+ static_call(kvm_x86_pmu_find_fixed_event)(idx),
!(en_field & 0x2), /* exclude user */
!(en_field & 0x1), /* exclude kernel */
pmi, false, false);
@@ -287,7 +287,7 @@ EXPORT_SYMBOL_GPL(reprogram_fixed_counter);
void reprogram_counter(struct kvm_pmu *pmu, int pmc_idx)
{
- struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, pmc_idx);
+ struct kvm_pmc *pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, pmc_idx);
if (!pmc)
return;
@@ -309,7 +309,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)
int bit;
for_each_set_bit(bit, pmu->reprogram_pmi, X86_PMC_IDX_MAX) {
- struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, bit);
+ struct kvm_pmc *pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, bit);
if (unlikely(!pmc || !pmc->perf_event)) {
clear_bit(bit, pmu->reprogram_pmi);
@@ -331,7 +331,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)
/* check if idx is a valid index to access PMU */
int kvm_pmu_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx)
{
- return kvm_pmu_ops.is_valid_rdpmc_ecx(vcpu, idx);
+ return static_call(kvm_x86_pmu_is_valid_rdpmc_ecx)(vcpu, idx);
}
bool is_vmware_backdoor_pmc(u32 pmc_idx)
@@ -381,7 +381,7 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
if (is_vmware_backdoor_pmc(idx))
return kvm_pmu_rdpmc_vmware(vcpu, idx, data);
- pmc = kvm_pmu_ops.rdpmc_ecx_to_pmc(vcpu, idx, &mask);
+ pmc = static_call(kvm_x86_pmu_rdpmc_ecx_to_pmc)(vcpu, idx, &mask);
if (!pmc)
return 1;
@@ -397,22 +397,21 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu)
{
if (lapic_in_kernel(vcpu)) {
- if (kvm_pmu_ops.deliver_pmi)
- kvm_pmu_ops.deliver_pmi(vcpu);
+ static_call_cond(kvm_x86_pmu_deliver_pmi)(vcpu);
kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC);
}
}
bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr)
{
- return kvm_pmu_ops.msr_idx_to_pmc(vcpu, msr) ||
- kvm_pmu_ops.is_valid_msr(vcpu, msr);
+ return static_call(kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr) ||
+ static_call(kvm_x86_pmu_is_valid_msr)(vcpu, msr);
}
static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr)
{
struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
- struct kvm_pmc *pmc = kvm_pmu_ops.msr_idx_to_pmc(vcpu, msr);
+ struct kvm_pmc *pmc = static_call(kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr);
if (pmc)
__set_bit(pmc->idx, pmu->pmc_in_use);
@@ -420,13 +419,13 @@ static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr)
int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
{
- return kvm_pmu_ops.get_msr(vcpu, msr_info);
+ return static_call(kvm_x86_pmu_get_msr)(vcpu, msr_info);
}
int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
{
kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index);
- return kvm_pmu_ops.set_msr(vcpu, msr_info);
+ return static_call(kvm_x86_pmu_set_msr)(vcpu, msr_info);
}
/* refresh PMU settings. This function generally is called when underlying
@@ -435,7 +434,7 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
*/
void kvm_pmu_refresh(struct kvm_vcpu *vcpu)
{
- kvm_pmu_ops.refresh(vcpu);
+ static_call(kvm_x86_pmu_refresh)(vcpu);
}
void kvm_pmu_reset(struct kvm_vcpu *vcpu)
@@ -443,7 +442,7 @@ void kvm_pmu_reset(struct kvm_vcpu *vcpu)
struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
irq_work_sync(&pmu->irq_work);
- kvm_pmu_ops.reset(vcpu);
+ static_call(kvm_x86_pmu_reset)(vcpu);
}
void kvm_pmu_init(struct kvm_vcpu *vcpu)
@@ -451,7 +450,7 @@ void kvm_pmu_init(struct kvm_vcpu *vcpu)
struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
memset(pmu, 0, sizeof(*pmu));
- kvm_pmu_ops.init(vcpu);
+ static_call(kvm_x86_pmu_init)(vcpu);
init_irq_work(&pmu->irq_work, kvm_pmi_trigger_fn);
pmu->event_count = 0;
pmu->need_cleanup = false;
@@ -483,14 +482,13 @@ void kvm_pmu_cleanup(struct kvm_vcpu *vcpu)
pmu->pmc_in_use, X86_PMC_IDX_MAX);
for_each_set_bit(i, bitmask, X86_PMC_IDX_MAX) {
- pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, i);
+ pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, i);
if (pmc && pmc->perf_event && !pmc_speculative_in_use(pmc))
pmc_stop_counter(pmc);
}
- if (kvm_pmu_ops.cleanup)
- kvm_pmu_ops.cleanup(vcpu);
+ static_call_cond(kvm_x86_pmu_cleanup)(vcpu);
bitmap_zero(pmu->pmc_in_use, X86_PMC_IDX_MAX);
}
diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h
index e5550d4acf14..1818d1371ece 100644
--- a/arch/x86/kvm/pmu.h
+++ b/arch/x86/kvm/pmu.h
@@ -109,7 +109,7 @@ static inline bool pmc_is_fixed(struct kvm_pmc *pmc)
static inline bool pmc_is_enabled(struct kvm_pmc *pmc)
{
- return kvm_pmu_ops.pmc_is_enabled(pmc);
+ return static_call(kvm_x86_pmu_pmc_is_enabled)(pmc);
}
static inline bool kvm_valid_perf_global_ctrl(struct kvm_pmu *pmu,
diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index 1e793e44b5ff..a61661de1f39 100644
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -4796,7 +4796,7 @@ void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu)
return;
vmx = to_vmx(vcpu);
- if (kvm_pmu_ops.is_valid_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) {
+ if (static_call(kvm_x86_pmu_is_valid_msr)(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) {
vmx->nested.msrs.entry_ctls_high |=
VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
vmx->nested.msrs.exit_ctls_high |=
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 72d286595012..88a3ef809c98 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -11317,8 +11317,10 @@ int kvm_arch_hardware_setup(void *opaque)
memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));
kvm_ops_static_call_update();
- if (kvm_x86_ops.hardware_enable)
+ if (kvm_x86_ops.hardware_enable) {
memcpy(&kvm_pmu_ops, kvm_x86_ops.pmu_ops, sizeof(kvm_pmu_ops));
+ kvm_pmu_ops_static_call_update();
+ }
if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))
supported_xss = 0;
--
2.33.0
On Wed, Nov 03, 2021 at 03:03:10PM +0800, Like Xu wrote:
> Convert kvm_pmu_ops to use static calls.
>
> Here are the worst sched_clock() nanosecond numbers for the kvm_pmu_ops
> functions that is most often called (up to 7 digits of calls) when running
> a single perf test case in a guest on an ICX 2.70GHz host (mitigations=on):
>
> | legacy | static call
> ------------------------------------------------------------
> .pmc_idx_to_pmc | 10946 | 10047 (8%)
> .pmc_is_enabled | 11291 | 11175 (1%)
> .msr_idx_to_pmc | 13526 | 12346 (8%)
> .is_valid_msr | 10895 | 10484 (3%)
>
> Signed-off-by: Like Xu <[email protected]>
> ---
> arch/x86/kvm/pmu.c | 36 +++++++++++++++++-------------------
> arch/x86/kvm/pmu.h | 2 +-
> arch/x86/kvm/vmx/nested.c | 2 +-
> arch/x86/kvm/x86.c | 4 +++-
> 4 files changed, 22 insertions(+), 22 deletions(-)
>
> diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
> index b6f08c719125..193f925e2064 100644
> --- a/arch/x86/kvm/pmu.c
> +++ b/arch/x86/kvm/pmu.c
> @@ -224,7 +224,7 @@ void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel)
> ARCH_PERFMON_EVENTSEL_CMASK |
> HSW_IN_TX |
> HSW_IN_TX_CHECKPOINTED))) {
> - config = kvm_pmu_ops.find_arch_event(pmc_to_pmu(pmc),
> + config = static_call(kvm_x86_pmu_find_arch_event)(pmc_to_pmu(pmc),
Why you need change them into kvm_pmu_ops.XXX then convert
them into static call ? Move the instance definition of
kvm_pmu_ops from patch 1 into patch 3 and then drop patch 1,
will this work ?
> event_select,
> unit_mask);
> if (config != PERF_COUNT_HW_MAX)
> @@ -278,7 +278,7 @@ void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 ctrl, int idx)
>
> pmc->current_config = (u64)ctrl;
> pmc_reprogram_counter(pmc, PERF_TYPE_HARDWARE,
> - kvm_pmu_ops.find_fixed_event(idx),
> + static_call(kvm_x86_pmu_find_fixed_event)(idx),
> !(en_field & 0x2), /* exclude user */
> !(en_field & 0x1), /* exclude kernel */
> pmi, false, false);
> @@ -287,7 +287,7 @@ EXPORT_SYMBOL_GPL(reprogram_fixed_counter);
>
> void reprogram_counter(struct kvm_pmu *pmu, int pmc_idx)
> {
> - struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, pmc_idx);
> + struct kvm_pmc *pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, pmc_idx);
>
> if (!pmc)
> return;
> @@ -309,7 +309,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)
> int bit;
>
> for_each_set_bit(bit, pmu->reprogram_pmi, X86_PMC_IDX_MAX) {
> - struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, bit);
> + struct kvm_pmc *pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, bit);
>
> if (unlikely(!pmc || !pmc->perf_event)) {
> clear_bit(bit, pmu->reprogram_pmi);
> @@ -331,7 +331,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)
> /* check if idx is a valid index to access PMU */
> int kvm_pmu_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx)
> {
> - return kvm_pmu_ops.is_valid_rdpmc_ecx(vcpu, idx);
> + return static_call(kvm_x86_pmu_is_valid_rdpmc_ecx)(vcpu, idx);
> }
>
> bool is_vmware_backdoor_pmc(u32 pmc_idx)
> @@ -381,7 +381,7 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
> if (is_vmware_backdoor_pmc(idx))
> return kvm_pmu_rdpmc_vmware(vcpu, idx, data);
>
> - pmc = kvm_pmu_ops.rdpmc_ecx_to_pmc(vcpu, idx, &mask);
> + pmc = static_call(kvm_x86_pmu_rdpmc_ecx_to_pmc)(vcpu, idx, &mask);
> if (!pmc)
> return 1;
>
> @@ -397,22 +397,21 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
> void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu)
> {
> if (lapic_in_kernel(vcpu)) {
> - if (kvm_pmu_ops.deliver_pmi)
> - kvm_pmu_ops.deliver_pmi(vcpu);
> + static_call_cond(kvm_x86_pmu_deliver_pmi)(vcpu);
> kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC);
> }
> }
>
> bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr)
> {
> - return kvm_pmu_ops.msr_idx_to_pmc(vcpu, msr) ||
> - kvm_pmu_ops.is_valid_msr(vcpu, msr);
> + return static_call(kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr) ||
> + static_call(kvm_x86_pmu_is_valid_msr)(vcpu, msr);
> }
>
> static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr)
> {
> struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
> - struct kvm_pmc *pmc = kvm_pmu_ops.msr_idx_to_pmc(vcpu, msr);
> + struct kvm_pmc *pmc = static_call(kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr);
>
> if (pmc)
> __set_bit(pmc->idx, pmu->pmc_in_use);
> @@ -420,13 +419,13 @@ static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr)
>
> int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
> {
> - return kvm_pmu_ops.get_msr(vcpu, msr_info);
> + return static_call(kvm_x86_pmu_get_msr)(vcpu, msr_info);
> }
>
> int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
> {
> kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index);
> - return kvm_pmu_ops.set_msr(vcpu, msr_info);
> + return static_call(kvm_x86_pmu_set_msr)(vcpu, msr_info);
> }
>
> /* refresh PMU settings. This function generally is called when underlying
> @@ -435,7 +434,7 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
> */
> void kvm_pmu_refresh(struct kvm_vcpu *vcpu)
> {
> - kvm_pmu_ops.refresh(vcpu);
> + static_call(kvm_x86_pmu_refresh)(vcpu);
> }
>
> void kvm_pmu_reset(struct kvm_vcpu *vcpu)
> @@ -443,7 +442,7 @@ void kvm_pmu_reset(struct kvm_vcpu *vcpu)
> struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
>
> irq_work_sync(&pmu->irq_work);
> - kvm_pmu_ops.reset(vcpu);
> + static_call(kvm_x86_pmu_reset)(vcpu);
> }
>
> void kvm_pmu_init(struct kvm_vcpu *vcpu)
> @@ -451,7 +450,7 @@ void kvm_pmu_init(struct kvm_vcpu *vcpu)
> struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
>
> memset(pmu, 0, sizeof(*pmu));
> - kvm_pmu_ops.init(vcpu);
> + static_call(kvm_x86_pmu_init)(vcpu);
> init_irq_work(&pmu->irq_work, kvm_pmi_trigger_fn);
> pmu->event_count = 0;
> pmu->need_cleanup = false;
> @@ -483,14 +482,13 @@ void kvm_pmu_cleanup(struct kvm_vcpu *vcpu)
> pmu->pmc_in_use, X86_PMC_IDX_MAX);
>
> for_each_set_bit(i, bitmask, X86_PMC_IDX_MAX) {
> - pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, i);
> + pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, i);
>
> if (pmc && pmc->perf_event && !pmc_speculative_in_use(pmc))
> pmc_stop_counter(pmc);
> }
>
> - if (kvm_pmu_ops.cleanup)
> - kvm_pmu_ops.cleanup(vcpu);
> + static_call_cond(kvm_x86_pmu_cleanup)(vcpu);
>
> bitmap_zero(pmu->pmc_in_use, X86_PMC_IDX_MAX);
> }
> diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h
> index e5550d4acf14..1818d1371ece 100644
> --- a/arch/x86/kvm/pmu.h
> +++ b/arch/x86/kvm/pmu.h
> @@ -109,7 +109,7 @@ static inline bool pmc_is_fixed(struct kvm_pmc *pmc)
>
> static inline bool pmc_is_enabled(struct kvm_pmc *pmc)
> {
> - return kvm_pmu_ops.pmc_is_enabled(pmc);
> + return static_call(kvm_x86_pmu_pmc_is_enabled)(pmc);
> }
>
> static inline bool kvm_valid_perf_global_ctrl(struct kvm_pmu *pmu,
> diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
> index 1e793e44b5ff..a61661de1f39 100644
> --- a/arch/x86/kvm/vmx/nested.c
> +++ b/arch/x86/kvm/vmx/nested.c
> @@ -4796,7 +4796,7 @@ void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu)
> return;
>
> vmx = to_vmx(vcpu);
> - if (kvm_pmu_ops.is_valid_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) {
> + if (static_call(kvm_x86_pmu_is_valid_msr)(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) {
> vmx->nested.msrs.entry_ctls_high |=
> VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
> vmx->nested.msrs.exit_ctls_high |=
> diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
> index 72d286595012..88a3ef809c98 100644
> --- a/arch/x86/kvm/x86.c
> +++ b/arch/x86/kvm/x86.c
> @@ -11317,8 +11317,10 @@ int kvm_arch_hardware_setup(void *opaque)
> memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));
> kvm_ops_static_call_update();
>
> - if (kvm_x86_ops.hardware_enable)
> + if (kvm_x86_ops.hardware_enable) {
> memcpy(&kvm_pmu_ops, kvm_x86_ops.pmu_ops, sizeof(kvm_pmu_ops));
> + kvm_pmu_ops_static_call_update();
> + }
>
> if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))
> supported_xss = 0;
> --
> 2.33.0
>
On 3/11/2021 8:08 pm, Yao Yuan wrote:
> On Wed, Nov 03, 2021 at 03:03:10PM +0800, Like Xu wrote:
>> Convert kvm_pmu_ops to use static calls.
>>
>> Here are the worst sched_clock() nanosecond numbers for the kvm_pmu_ops
>> functions that is most often called (up to 7 digits of calls) when running
>> a single perf test case in a guest on an ICX 2.70GHz host (mitigations=on):
>>
>> | legacy | static call
>> ------------------------------------------------------------
>> .pmc_idx_to_pmc | 10946 | 10047 (8%)
>> .pmc_is_enabled | 11291 | 11175 (1%)
>> .msr_idx_to_pmc | 13526 | 12346 (8%)
>> .is_valid_msr | 10895 | 10484 (3%)
>>
>> Signed-off-by: Like Xu <[email protected]>
>> ---
>> arch/x86/kvm/pmu.c | 36 +++++++++++++++++-------------------
>> arch/x86/kvm/pmu.h | 2 +-
>> arch/x86/kvm/vmx/nested.c | 2 +-
>> arch/x86/kvm/x86.c | 4 +++-
>> 4 files changed, 22 insertions(+), 22 deletions(-)
>>
>> diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
>> index b6f08c719125..193f925e2064 100644
>> --- a/arch/x86/kvm/pmu.c
>> +++ b/arch/x86/kvm/pmu.c
>> @@ -224,7 +224,7 @@ void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel)
>> ARCH_PERFMON_EVENTSEL_CMASK |
>> HSW_IN_TX |
>> HSW_IN_TX_CHECKPOINTED))) {
>> - config = kvm_pmu_ops.find_arch_event(pmc_to_pmu(pmc),
>> + config = static_call(kvm_x86_pmu_find_arch_event)(pmc_to_pmu(pmc),
>
> Why you need change them into kvm_pmu_ops.XXX then convert
> them into static call ? Move the instance definition of
> kvm_pmu_ops from patch 1 into patch 3 and then drop patch 1,
> will this work ?
You may git squash all commits to get it *work*.
With reference to afaf0b2f9b801c6eb2278b52d49e6a7d7b659cf1[1],
doing one thing at a time will make things go smoother.
[1]
https://lore.kernel.org/lkml/[email protected]/
>
>> event_select,
>> unit_mask);
>> if (config != PERF_COUNT_HW_MAX)
>> @@ -278,7 +278,7 @@ void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 ctrl, int idx)
>>
>> pmc->current_config = (u64)ctrl;
>> pmc_reprogram_counter(pmc, PERF_TYPE_HARDWARE,
>> - kvm_pmu_ops.find_fixed_event(idx),
>> + static_call(kvm_x86_pmu_find_fixed_event)(idx),
>> !(en_field & 0x2), /* exclude user */
>> !(en_field & 0x1), /* exclude kernel */
>> pmi, false, false);
>> @@ -287,7 +287,7 @@ EXPORT_SYMBOL_GPL(reprogram_fixed_counter);
>>
>> void reprogram_counter(struct kvm_pmu *pmu, int pmc_idx)
>> {
>> - struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, pmc_idx);
>> + struct kvm_pmc *pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, pmc_idx);
>>
>> if (!pmc)
>> return;
>> @@ -309,7 +309,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)
>> int bit;
>>
>> for_each_set_bit(bit, pmu->reprogram_pmi, X86_PMC_IDX_MAX) {
>> - struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, bit);
>> + struct kvm_pmc *pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, bit);
>>
>> if (unlikely(!pmc || !pmc->perf_event)) {
>> clear_bit(bit, pmu->reprogram_pmi);
>> @@ -331,7 +331,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)
>> /* check if idx is a valid index to access PMU */
>> int kvm_pmu_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx)
>> {
>> - return kvm_pmu_ops.is_valid_rdpmc_ecx(vcpu, idx);
>> + return static_call(kvm_x86_pmu_is_valid_rdpmc_ecx)(vcpu, idx);
>> }
>>
>> bool is_vmware_backdoor_pmc(u32 pmc_idx)
>> @@ -381,7 +381,7 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
>> if (is_vmware_backdoor_pmc(idx))
>> return kvm_pmu_rdpmc_vmware(vcpu, idx, data);
>>
>> - pmc = kvm_pmu_ops.rdpmc_ecx_to_pmc(vcpu, idx, &mask);
>> + pmc = static_call(kvm_x86_pmu_rdpmc_ecx_to_pmc)(vcpu, idx, &mask);
>> if (!pmc)
>> return 1;
>>
>> @@ -397,22 +397,21 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
>> void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu)
>> {
>> if (lapic_in_kernel(vcpu)) {
>> - if (kvm_pmu_ops.deliver_pmi)
>> - kvm_pmu_ops.deliver_pmi(vcpu);
>> + static_call_cond(kvm_x86_pmu_deliver_pmi)(vcpu);
>> kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC);
>> }
>> }
>>
>> bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr)
>> {
>> - return kvm_pmu_ops.msr_idx_to_pmc(vcpu, msr) ||
>> - kvm_pmu_ops.is_valid_msr(vcpu, msr);
>> + return static_call(kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr) ||
>> + static_call(kvm_x86_pmu_is_valid_msr)(vcpu, msr);
>> }
>>
>> static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr)
>> {
>> struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
>> - struct kvm_pmc *pmc = kvm_pmu_ops.msr_idx_to_pmc(vcpu, msr);
>> + struct kvm_pmc *pmc = static_call(kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr);
>>
>> if (pmc)
>> __set_bit(pmc->idx, pmu->pmc_in_use);
>> @@ -420,13 +419,13 @@ static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr)
>>
>> int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
>> {
>> - return kvm_pmu_ops.get_msr(vcpu, msr_info);
>> + return static_call(kvm_x86_pmu_get_msr)(vcpu, msr_info);
>> }
>>
>> int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
>> {
>> kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index);
>> - return kvm_pmu_ops.set_msr(vcpu, msr_info);
>> + return static_call(kvm_x86_pmu_set_msr)(vcpu, msr_info);
>> }
>>
>> /* refresh PMU settings. This function generally is called when underlying
>> @@ -435,7 +434,7 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
>> */
>> void kvm_pmu_refresh(struct kvm_vcpu *vcpu)
>> {
>> - kvm_pmu_ops.refresh(vcpu);
>> + static_call(kvm_x86_pmu_refresh)(vcpu);
>> }
>>
>> void kvm_pmu_reset(struct kvm_vcpu *vcpu)
>> @@ -443,7 +442,7 @@ void kvm_pmu_reset(struct kvm_vcpu *vcpu)
>> struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
>>
>> irq_work_sync(&pmu->irq_work);
>> - kvm_pmu_ops.reset(vcpu);
>> + static_call(kvm_x86_pmu_reset)(vcpu);
>> }
>>
>> void kvm_pmu_init(struct kvm_vcpu *vcpu)
>> @@ -451,7 +450,7 @@ void kvm_pmu_init(struct kvm_vcpu *vcpu)
>> struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
>>
>> memset(pmu, 0, sizeof(*pmu));
>> - kvm_pmu_ops.init(vcpu);
>> + static_call(kvm_x86_pmu_init)(vcpu);
>> init_irq_work(&pmu->irq_work, kvm_pmi_trigger_fn);
>> pmu->event_count = 0;
>> pmu->need_cleanup = false;
>> @@ -483,14 +482,13 @@ void kvm_pmu_cleanup(struct kvm_vcpu *vcpu)
>> pmu->pmc_in_use, X86_PMC_IDX_MAX);
>>
>> for_each_set_bit(i, bitmask, X86_PMC_IDX_MAX) {
>> - pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, i);
>> + pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, i);
>>
>> if (pmc && pmc->perf_event && !pmc_speculative_in_use(pmc))
>> pmc_stop_counter(pmc);
>> }
>>
>> - if (kvm_pmu_ops.cleanup)
>> - kvm_pmu_ops.cleanup(vcpu);
>> + static_call_cond(kvm_x86_pmu_cleanup)(vcpu);
>>
>> bitmap_zero(pmu->pmc_in_use, X86_PMC_IDX_MAX);
>> }
>> diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h
>> index e5550d4acf14..1818d1371ece 100644
>> --- a/arch/x86/kvm/pmu.h
>> +++ b/arch/x86/kvm/pmu.h
>> @@ -109,7 +109,7 @@ static inline bool pmc_is_fixed(struct kvm_pmc *pmc)
>>
>> static inline bool pmc_is_enabled(struct kvm_pmc *pmc)
>> {
>> - return kvm_pmu_ops.pmc_is_enabled(pmc);
>> + return static_call(kvm_x86_pmu_pmc_is_enabled)(pmc);
>> }
>>
>> static inline bool kvm_valid_perf_global_ctrl(struct kvm_pmu *pmu,
>> diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
>> index 1e793e44b5ff..a61661de1f39 100644
>> --- a/arch/x86/kvm/vmx/nested.c
>> +++ b/arch/x86/kvm/vmx/nested.c
>> @@ -4796,7 +4796,7 @@ void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu)
>> return;
>>
>> vmx = to_vmx(vcpu);
>> - if (kvm_pmu_ops.is_valid_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) {
>> + if (static_call(kvm_x86_pmu_is_valid_msr)(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) {
>> vmx->nested.msrs.entry_ctls_high |=
>> VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
>> vmx->nested.msrs.exit_ctls_high |=
>> diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
>> index 72d286595012..88a3ef809c98 100644
>> --- a/arch/x86/kvm/x86.c
>> +++ b/arch/x86/kvm/x86.c
>> @@ -11317,8 +11317,10 @@ int kvm_arch_hardware_setup(void *opaque)
>> memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));
>> kvm_ops_static_call_update();
>>
>> - if (kvm_x86_ops.hardware_enable)
>> + if (kvm_x86_ops.hardware_enable) {
>> memcpy(&kvm_pmu_ops, kvm_x86_ops.pmu_ops, sizeof(kvm_pmu_ops));
>> + kvm_pmu_ops_static_call_update();
>> + }
>>
>> if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))
>> supported_xss = 0;
>> --
>> 2.33.0
>>
>
On Wed, Nov 03, 2021, Like Xu wrote:
> Replace the kvm_pmu_ops pointer in common x86 with an instance of the
> struct to save one pointer dereference when invoking functions. Copy the
> struct by value to set the ops during kvm_init().
>
> Using kvm_x86_ops.hardware_enable to track whether or not the
> ops have been initialized, i.e. a vendor KVM module has been loaded.
>
> Signed-off-by: Like Xu <[email protected]>
> ---
> arch/x86/kvm/pmu.c | 41 +++++++++++++++++++++------------------
> arch/x86/kvm/pmu.h | 4 +++-
> arch/x86/kvm/vmx/nested.c | 2 +-
> arch/x86/kvm/x86.c | 3 +++
> 4 files changed, 29 insertions(+), 21 deletions(-)
>
> diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
> index 0772bad9165c..0db1887137d9 100644
> --- a/arch/x86/kvm/pmu.c
> +++ b/arch/x86/kvm/pmu.c
> @@ -47,6 +47,9 @@
> * * AMD: [0 .. AMD64_NUM_COUNTERS-1] <=> gp counters
> */
>
> +struct kvm_pmu_ops kvm_pmu_ops __read_mostly;
> +EXPORT_SYMBOL_GPL(kvm_pmu_ops);
> +
...
> diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
> index b4ee5e9f9e20..1e793e44b5ff 100644
> --- a/arch/x86/kvm/vmx/nested.c
> +++ b/arch/x86/kvm/vmx/nested.c
> @@ -4796,7 +4796,7 @@ void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu)
> return;
>
> vmx = to_vmx(vcpu);
> - if (kvm_x86_ops.pmu_ops->is_valid_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) {
> + if (kvm_pmu_ops.is_valid_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) {
I would much prefer we export kvm_pmu_is_valid_msr() and go through that for nVMX
than export all of kvm_pmu_ops for this one case.
> vmx->nested.msrs.entry_ctls_high |=
> VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
> vmx->nested.msrs.exit_ctls_high |=
> diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
> index ac83d873d65b..72d286595012 100644
> --- a/arch/x86/kvm/x86.c
> +++ b/arch/x86/kvm/x86.c
> @@ -11317,6 +11317,9 @@ int kvm_arch_hardware_setup(void *opaque)
> memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));
> kvm_ops_static_call_update();
>
> + if (kvm_x86_ops.hardware_enable)
Huh? Did you intend this to be?
if (kvm_x86_ops.pmu_ops)
Either way, I don't see the point, VMX and SVM unconditionally provide the ops.
I would also say land this memcpy() above kvm_ops_static_call_update(), then the
enabling patch can do the static call updates in kvm_ops_static_call_update()
instead of adding another helper.
> + memcpy(&kvm_pmu_ops, kvm_x86_ops.pmu_ops, sizeof(kvm_pmu_ops));
As part of this change, the pmu_ops should be moved to kvm_x86_init_ops and tagged
as __initdata. That'll save those precious few bytes, and more importantly make
the original ops unreachable, i.e. make it harder to sneak in post-init modification
bugs.
> +
> if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))
> supported_xss = 0;
>
> --
> 2.33.0
>
On Fri, Nov 05, 2021, Sean Christopherson wrote:
> On Wed, Nov 03, 2021, Like Xu wrote:
> I would also say land this memcpy() above kvm_ops_static_call_update(), then the
> enabling patch can do the static call updates in kvm_ops_static_call_update()
> instead of adding another helper.
Ugh, kvm_ops_static_call_update() is defined in kvm_host.h. That's completely
unnecessary, it should have exactly one caller, kvm_arch_hardware_setup(). As a
prep match, move kvm_ops_static_call_update() to x86.c, then it can reference the
pmu ops.
On Wed, Nov 03, 2021, Like Xu wrote:
> diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
> index 0db1887137d9..b6f08c719125 100644
> --- a/arch/x86/kvm/pmu.c
> +++ b/arch/x86/kvm/pmu.c
> @@ -50,6 +50,13 @@
> struct kvm_pmu_ops kvm_pmu_ops __read_mostly;
> EXPORT_SYMBOL_GPL(kvm_pmu_ops);
>
> +#define KVM_X86_PMU_OP(func) \
> + DEFINE_STATIC_CALL_NULL(kvm_x86_pmu_##func, \
> + *(((struct kvm_pmu_ops *)0)->func))
> +#define KVM_X86_PMU_OP_NULL KVM_X86_PMU_OP
More of a question for the existing code, what's the point of KVM_X86_OP_NULL?
AFAICT, it always resolves to KVM_X86_OP. Unless there's some magic I'm missing,
I vote we remove KVM_X86_OP_NULL and then not introduce KVM_X86_PMU_OP_NULL.
And I'm pretty sure it's useless, e.g. get_cs_db_l_bits is defined with the NULL
variant, but it's never NULL and its calls aren't guarded with anything. And if
KVM_X86_OP_NULL is intended to aid in documenting behavior, it's doing a pretty
miserable job of that :-)
> +#include <asm/kvm-x86-pmu-ops.h>
> +EXPORT_STATIC_CALL_GPL(kvm_x86_pmu_is_valid_msr);
I'll double down on my nVMX suggestion so that this export can be avoided.
> static void kvm_pmi_trigger_fn(struct irq_work *irq_work)
> {
> struct kvm_pmu *pmu = container_of(irq_work, struct kvm_pmu, irq_work);
> diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h
> index b2fe135d395a..e5550d4acf14 100644
> --- a/arch/x86/kvm/pmu.h
> +++ b/arch/x86/kvm/pmu.h
> @@ -3,6 +3,8 @@
> #define __KVM_X86_PMU_H
>
> #include <linux/nospec.h>
> +#include <linux/static_call_types.h>
> +#include <linux/static_call.h>
>
> #define vcpu_to_pmu(vcpu) (&(vcpu)->arch.pmu)
> #define pmu_to_vcpu(pmu) (container_of((pmu), struct kvm_vcpu, arch.pmu))
> @@ -45,6 +47,19 @@ struct kvm_pmu_ops {
> void (*cleanup)(struct kvm_vcpu *vcpu);
> };
>
> +#define KVM_X86_PMU_OP(func) \
> + DECLARE_STATIC_CALL(kvm_x86_pmu_##func, *(((struct kvm_pmu_ops *)0)->func))
> +#define KVM_X86_PMU_OP_NULL KVM_X86_PMU_OP
> +#include <asm/kvm-x86-pmu-ops.h>
> +
> +static inline void kvm_pmu_ops_static_call_update(void)
> +{
> +#define KVM_X86_PMU_OP(func) \
> + static_call_update(kvm_x86_pmu_##func, kvm_pmu_ops.func)
> +#define KVM_X86_PMU_OP_NULL KVM_X86_PMU_OP
> +#include <asm/kvm-x86-pmu-ops.h>
> +}
As alluded to in patch 01, I'd prefer these go in kvm_ops_static_call_update()
to keep the static call magic somewhat contained.
> +
> static inline u64 pmc_bitmask(struct kvm_pmc *pmc)
> {
> struct kvm_pmu *pmu = pmc_to_pmu(pmc);
> --
> 2.33.0
>
On 5/11/2021 11:30 pm, Sean Christopherson wrote:
> On Wed, Nov 03, 2021, Like Xu wrote:
>> Replace the kvm_pmu_ops pointer in common x86 with an instance of the
>> struct to save one pointer dereference when invoking functions. Copy the
>> struct by value to set the ops during kvm_init().
>>
>> Using kvm_x86_ops.hardware_enable to track whether or not the
>> ops have been initialized, i.e. a vendor KVM module has been loaded.
>>
>> Signed-off-by: Like Xu <[email protected]>
>> ---
>> arch/x86/kvm/pmu.c | 41 +++++++++++++++++++++------------------
>> arch/x86/kvm/pmu.h | 4 +++-
>> arch/x86/kvm/vmx/nested.c | 2 +-
>> arch/x86/kvm/x86.c | 3 +++
>> 4 files changed, 29 insertions(+), 21 deletions(-)
>>
>> diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
>> index 0772bad9165c..0db1887137d9 100644
>> --- a/arch/x86/kvm/pmu.c
>> +++ b/arch/x86/kvm/pmu.c
>> @@ -47,6 +47,9 @@
>> * * AMD: [0 .. AMD64_NUM_COUNTERS-1] <=> gp counters
>> */
>>
>> +struct kvm_pmu_ops kvm_pmu_ops __read_mostly;
>> +EXPORT_SYMBOL_GPL(kvm_pmu_ops);
>> +
>
> ...
>
>> diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
>> index b4ee5e9f9e20..1e793e44b5ff 100644
>> --- a/arch/x86/kvm/vmx/nested.c
>> +++ b/arch/x86/kvm/vmx/nested.c
>> @@ -4796,7 +4796,7 @@ void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu)
>> return;
>>
>> vmx = to_vmx(vcpu);
>> - if (kvm_x86_ops.pmu_ops->is_valid_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) {
>> + if (kvm_pmu_ops.is_valid_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) {
>
> I would much prefer we export kvm_pmu_is_valid_msr() and go through that for nVMX
> than export all of kvm_pmu_ops for this one case.
Applied. Is it an abuse to export a function for only one case ?
>
>> vmx->nested.msrs.entry_ctls_high |=
>> VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
>> vmx->nested.msrs.exit_ctls_high |=
>> diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
>> index ac83d873d65b..72d286595012 100644
>> --- a/arch/x86/kvm/x86.c
>> +++ b/arch/x86/kvm/x86.c
>> @@ -11317,6 +11317,9 @@ int kvm_arch_hardware_setup(void *opaque)
>> memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));
>> kvm_ops_static_call_update();
>>
>> + if (kvm_x86_ops.hardware_enable)
>
> Huh? Did you intend this to be?
>
> if (kvm_x86_ops.pmu_ops)
>
> Either way, I don't see the point, VMX and SVM unconditionally provide the ops.
Let me drop it.
>
> I would also say land this memcpy() above kvm_ops_static_call_update(), then the
> enabling patch can do the static call updates in kvm_ops_static_call_update()
> instead of adding another helper.
Both applied.
>
>> + memcpy(&kvm_pmu_ops, kvm_x86_ops.pmu_ops, sizeof(kvm_pmu_ops));
>
> As part of this change, the pmu_ops should be moved to kvm_x86_init_ops and tagged
> as __initdata. That'll save those precious few bytes, and more importantly make
> the original ops unreachable, i.e. make it harder to sneak in post-init modification
> bugs.
Applied. Opportunistically,
-struct kvm_pmu_ops [amd|intel]_pmu_ops = {
+struct kvm_pmu_ops [amd|intel]_pmu_ops __initdata = {
>
>> +
>> if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))
>> supported_xss = 0;
>>
>> --
>> 2.33.0
>>
>
On 5/11/2021 11:36 pm, Sean Christopherson wrote:
> On Fri, Nov 05, 2021, Sean Christopherson wrote:
>> On Wed, Nov 03, 2021, Like Xu wrote:
>> I would also say land this memcpy() above kvm_ops_static_call_update(), then the
>> enabling patch can do the static call updates in kvm_ops_static_call_update()
>> instead of adding another helper.
>
> Ugh, kvm_ops_static_call_update() is defined in kvm_host.h. That's completely
> unnecessary, it should have exactly one caller, kvm_arch_hardware_setup(). As a
> prep match, move kvm_ops_static_call_update() to x86.c, then it can reference the
> pmu ops.
Quite good and thank you.
On 5/11/2021 11:48 pm, Sean Christopherson wrote:
> On Wed, Nov 03, 2021, Like Xu wrote:
>> diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
>> index 0db1887137d9..b6f08c719125 100644
>> --- a/arch/x86/kvm/pmu.c
>> +++ b/arch/x86/kvm/pmu.c
>> @@ -50,6 +50,13 @@
>> struct kvm_pmu_ops kvm_pmu_ops __read_mostly;
>> EXPORT_SYMBOL_GPL(kvm_pmu_ops);
>>
>> +#define KVM_X86_PMU_OP(func) \
>> + DEFINE_STATIC_CALL_NULL(kvm_x86_pmu_##func, \
>> + *(((struct kvm_pmu_ops *)0)->func))
>> +#define KVM_X86_PMU_OP_NULL KVM_X86_PMU_OP
>
> More of a question for the existing code, what's the point of KVM_X86_OP_NULL?
The comment says:
* KVM_X86_OP_NULL() can leave a NULL definition for the
* case where there is no definition or a function name that
* doesn't match the typical naming convention is supplied.
Does it help ?
> AFAICT, it always resolves to KVM_X86_OP. Unless there's some magic I'm missing,
> I vote we remove KVM_X86_OP_NULL and then not introduce KVM_X86_PMU_OP_NULL.
> And I'm pretty sure it's useless, e.g. get_cs_db_l_bits is defined with the NULL
This transitions will not be included in the next version. Open to you.
> variant, but it's never NULL and its calls aren't guarded with anything. And if
> KVM_X86_OP_NULL is intended to aid in documenting behavior, it's doing a pretty
> miserable job of that :-)
>
>> +#include <asm/kvm-x86-pmu-ops.h>
>> +EXPORT_STATIC_CALL_GPL(kvm_x86_pmu_is_valid_msr);
>
> I'll double down on my nVMX suggestion so that this export can be avoided.
Fine to me.
>
>> static void kvm_pmi_trigger_fn(struct irq_work *irq_work)
>> {
>> struct kvm_pmu *pmu = container_of(irq_work, struct kvm_pmu, irq_work);
>> diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h
>> index b2fe135d395a..e5550d4acf14 100644
>> --- a/arch/x86/kvm/pmu.h
>> +++ b/arch/x86/kvm/pmu.h
>> @@ -3,6 +3,8 @@
>> #define __KVM_X86_PMU_H
>>
>> #include <linux/nospec.h>
>> +#include <linux/static_call_types.h>
>> +#include <linux/static_call.h>
>>
>> #define vcpu_to_pmu(vcpu) (&(vcpu)->arch.pmu)
>> #define pmu_to_vcpu(pmu) (container_of((pmu), struct kvm_vcpu, arch.pmu))
>> @@ -45,6 +47,19 @@ struct kvm_pmu_ops {
>> void (*cleanup)(struct kvm_vcpu *vcpu);
>> };
>>
>> +#define KVM_X86_PMU_OP(func) \
>> + DECLARE_STATIC_CALL(kvm_x86_pmu_##func, *(((struct kvm_pmu_ops *)0)->func))
>> +#define KVM_X86_PMU_OP_NULL KVM_X86_PMU_OP
>> +#include <asm/kvm-x86-pmu-ops.h>
>> +
>> +static inline void kvm_pmu_ops_static_call_update(void)
>> +{
>> +#define KVM_X86_PMU_OP(func) \
>> + static_call_update(kvm_x86_pmu_##func, kvm_pmu_ops.func)
>> +#define KVM_X86_PMU_OP_NULL KVM_X86_PMU_OP
>> +#include <asm/kvm-x86-pmu-ops.h>
>> +}
>
> As alluded to in patch 01, I'd prefer these go in kvm_ops_static_call_update()
> to keep the static call magic somewhat contained.
Thank and applied.
>
>> +
>> static inline u64 pmc_bitmask(struct kvm_pmc *pmc)
>> {
>> struct kvm_pmu *pmu = pmc_to_pmu(pmc);
>> --
>> 2.33.0
>>
>
On Mon, Nov 08, 2021, Like Xu wrote:
> On 5/11/2021 11:48 pm, Sean Christopherson wrote:
> > On Wed, Nov 03, 2021, Like Xu wrote:
> > > diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
> > > index 0db1887137d9..b6f08c719125 100644
> > > --- a/arch/x86/kvm/pmu.c
> > > +++ b/arch/x86/kvm/pmu.c
> > > @@ -50,6 +50,13 @@
> > > struct kvm_pmu_ops kvm_pmu_ops __read_mostly;
> > > EXPORT_SYMBOL_GPL(kvm_pmu_ops);
> > > +#define KVM_X86_PMU_OP(func) \
> > > + DEFINE_STATIC_CALL_NULL(kvm_x86_pmu_##func, \
> > > + *(((struct kvm_pmu_ops *)0)->func))
> > > +#define KVM_X86_PMU_OP_NULL KVM_X86_PMU_OP
> >
> > More of a question for the existing code, what's the point of KVM_X86_OP_NULL?
>
> The comment says:
>
> * KVM_X86_OP_NULL() can leave a NULL definition for the
> * case where there is no definition or a function name that
> * doesn't match the typical naming convention is supplied.
>
> Does it help ?
No. I understand the original intent of KVM_X86_OP_NULL, but unless there's some
form of enforcement, it does more harm than good because it can very easily become
stale, e.g. see get_cs_db_l_bits(). I guess "what's the point of KVM_X86_OP_NULL?"
was somewhat of a rhetorical question.
> > AFAICT, it always resolves to KVM_X86_OP. Unless there's some magic I'm missing,
> > I vote we remove KVM_X86_OP_NULL and then not introduce KVM_X86_PMU_OP_NULL.
> > And I'm pretty sure it's useless, e.g. get_cs_db_l_bits is defined with the NULL