To aid in speculation control, the LFENCE instruction will be turned into
a serializing instruction. There is less performance impact using LFENCE
in this way compared to MFENCE.
With LFENCE now being a serializing instruction, it can be also used in
rdtsc_ordered() in preference to MFENCE_RDTSC. Since the kernel could
be running under a hypervisor that does not allow writing to that MSR,
it must be first verified that the write was successful before setting
the LFENCE_RDTSC feature.
The following patches are included in this series:
- Make LFENCE a serializing instruction on AMD
- Use LFENCE_RDTSC in preference to MFENCE_RDTSC on AMD
This patch series is based on tip:x86/pti.
---
Changes from v1:
- Add a check verifying the MSR was actually updated
- Remove the third patch that eliminates the MFENCE_RDTSC feature
(since the feature is still needed)
- Adding Dan Williams to the cc since this will impact nospec_barrier(),
which will require an alternative_2 to add an MFENCE instruction with
an MFENCE_RDTSC check
Tom Lendacky (2):
x86/cpu/AMD: Make LFENCE a serializing instruction
x86/cpu/AMD: Use LFENCE_RDTSC in preference to MFENCE_RDTSC
arch/x86/include/asm/msr-index.h | 3 +++
arch/x86/kernel/cpu/amd.c | 27 +++++++++++++++++++++++++--
2 files changed, 28 insertions(+), 2 deletions(-)
--
Tom Lendacky
To aid in speculation control, make LFENCE a serializing instruction
since it has less overhead than MFENCE. This is done by setting bit 1
of MSR 0xc0011029 (DE_CFG). Some families that support LFENCE do not
have this MSR. For these families, the LFENCE instruction is already
serializing.
Signed-off-by: Tom Lendacky <[email protected]>
---
arch/x86/include/asm/msr-index.h | 2 ++
arch/x86/kernel/cpu/amd.c | 10 ++++++++++
2 files changed, 12 insertions(+)
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index ab02261..1e7d710 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -352,6 +352,8 @@
#define FAM10H_MMIO_CONF_BASE_MASK 0xfffffffULL
#define FAM10H_MMIO_CONF_BASE_SHIFT 20
#define MSR_FAM10H_NODE_ID 0xc001100c
+#define MSR_F10H_DECFG 0xc0011029
+#define MSR_F10H_DECFG_LFENCE_SERIALIZE_BIT 1
/* K8 MSRs */
#define MSR_K8_TOP_MEM1 0xc001001a
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index bcb75dc..5b438d8 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -829,6 +829,16 @@ static void init_amd(struct cpuinfo_x86 *c)
set_cpu_cap(c, X86_FEATURE_K8);
if (cpu_has(c, X86_FEATURE_XMM2)) {
+ /*
+ * A serializing LFENCE has less overhead than MFENCE, so
+ * use it for execution serialization. On families which
+ * don't have that MSR, LFENCE is already serializing.
+ * msr_set_bit() uses the safe accessors, too, even if the MSR
+ * is not present.
+ */
+ msr_set_bit(MSR_F10H_DECFG,
+ MSR_F10H_DECFG_LFENCE_SERIALIZE_BIT);
+
/* MFENCE stops RDTSC speculation */
set_cpu_cap(c, X86_FEATURE_MFENCE_RDTSC);
}
With LFENCE now a serializing instruction, use LFENCE_RDTSC in preference
to MFENCE_RDTSC. However, since the kernel could be running under a
hypervisor that does not support writing that MSR, read the MSR back and
verify that the bit has been set successfully. If the MSR can be read
and the bit is set, then set the LFENCE_RDTSC feature, otherwise set the
MFENCE_RDTSC feature.
Signed-off-by: Tom Lendacky <[email protected]>
---
arch/x86/include/asm/msr-index.h | 1 +
arch/x86/kernel/cpu/amd.c | 17 +++++++++++++++--
2 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index 1e7d710..fa11fb1 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -354,6 +354,7 @@
#define MSR_FAM10H_NODE_ID 0xc001100c
#define MSR_F10H_DECFG 0xc0011029
#define MSR_F10H_DECFG_LFENCE_SERIALIZE_BIT 1
+#define MSR_F10H_DECFG_LFENCE_SERIALIZE BIT_ULL(MSR_F10H_DECFG_LFENCE_SERIALIZE_BIT)
/* K8 MSRs */
#define MSR_K8_TOP_MEM1 0xc001001a
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index 5b438d8..053f6c7 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -829,6 +829,9 @@ static void init_amd(struct cpuinfo_x86 *c)
set_cpu_cap(c, X86_FEATURE_K8);
if (cpu_has(c, X86_FEATURE_XMM2)) {
+ unsigned long long val;
+ int ret;
+
/*
* A serializing LFENCE has less overhead than MFENCE, so
* use it for execution serialization. On families which
@@ -839,8 +842,18 @@ static void init_amd(struct cpuinfo_x86 *c)
msr_set_bit(MSR_F10H_DECFG,
MSR_F10H_DECFG_LFENCE_SERIALIZE_BIT);
- /* MFENCE stops RDTSC speculation */
- set_cpu_cap(c, X86_FEATURE_MFENCE_RDTSC);
+ /*
+ * Verify that the MSR write was successful (could be running
+ * under a hypervisor) and only then assume that LFENCE is
+ * serializing.
+ */
+ ret = rdmsrl_safe(MSR_F10H_DECFG, &val);
+ if (!ret && (val & MSR_F10H_DECFG_LFENCE_SERIALIZE))
+ /* A serializing LFENCE stops RDTSC speculation */
+ set_cpu_cap(c, X86_FEATURE_LFENCE_RDTSC);
+ else
+ /* MFENCE stops RDTSC speculation */
+ set_cpu_cap(c, X86_FEATURE_MFENCE_RDTSC);
}
/*
On Mon, Jan 8, 2018 at 2:09 PM, Tom Lendacky <[email protected]> wrote:
> To aid in speculation control, the LFENCE instruction will be turned into
> a serializing instruction. There is less performance impact using LFENCE
> in this way compared to MFENCE.
>
> With LFENCE now being a serializing instruction, it can be also used in
> rdtsc_ordered() in preference to MFENCE_RDTSC. Since the kernel could
> be running under a hypervisor that does not allow writing to that MSR,
> it must be first verified that the write was successful before setting
> the LFENCE_RDTSC feature.
>
> The following patches are included in this series:
> - Make LFENCE a serializing instruction on AMD
> - Use LFENCE_RDTSC in preference to MFENCE_RDTSC on AMD
>
> This patch series is based on tip:x86/pti.
>
> ---
>
> Changes from v1:
> - Add a check verifying the MSR was actually updated
> - Remove the third patch that eliminates the MFENCE_RDTSC feature
> (since the feature is still needed)
> - Adding Dan Williams to the cc since this will impact nospec_barrier(),
> which will require an alternative_2 to add an MFENCE instruction with
> an MFENCE_RDTSC check
Thanks Tom, I'll include this in the next posting of the variant-1 patch series.
On Mon, Jan 08, 2018 at 04:09:12PM -0600, Tom Lendacky wrote:
> To aid in speculation control, the LFENCE instruction will be turned into
> a serializing instruction. There is less performance impact using LFENCE
> in this way compared to MFENCE.
>
> With LFENCE now being a serializing instruction, it can be also used in
> rdtsc_ordered() in preference to MFENCE_RDTSC. Since the kernel could
> be running under a hypervisor that does not allow writing to that MSR,
> it must be first verified that the write was successful before setting
> the LFENCE_RDTSC feature.
>
> The following patches are included in this series:
> - Make LFENCE a serializing instruction on AMD
> - Use LFENCE_RDTSC in preference to MFENCE_RDTSC on AMD
>
> This patch series is based on tip:x86/pti.
>
> ---
>
> Changes from v1:
> - Add a check verifying the MSR was actually updated
> - Remove the third patch that eliminates the MFENCE_RDTSC feature
> (since the feature is still needed)
> - Adding Dan Williams to the cc since this will impact nospec_barrier(),
> which will require an alternative_2 to add an MFENCE instruction with
> an MFENCE_RDTSC check
>
> Tom Lendacky (2):
> x86/cpu/AMD: Make LFENCE a serializing instruction
> x86/cpu/AMD: Use LFENCE_RDTSC in preference to MFENCE_RDTSC
Looks good to me.
Reviewed-by: Borislav Petkov <[email protected]>
--
Regards/Gruss,
Boris.
Good mailing practices for 400: avoid top-posting and trim the reply.
Commit-ID: e4d0e84e490790798691aaa0f2e598637f1867ec
Gitweb: https://git.kernel.org/tip/e4d0e84e490790798691aaa0f2e598637f1867ec
Author: Tom Lendacky <[email protected]>
AuthorDate: Mon, 8 Jan 2018 16:09:21 -0600
Committer: Thomas Gleixner <[email protected]>
CommitDate: Tue, 9 Jan 2018 01:43:10 +0100
x86/cpu/AMD: Make LFENCE a serializing instruction
To aid in speculation control, make LFENCE a serializing instruction
since it has less overhead than MFENCE. This is done by setting bit 1
of MSR 0xc0011029 (DE_CFG). Some families that support LFENCE do not
have this MSR. For these families, the LFENCE instruction is already
serializing.
Signed-off-by: Tom Lendacky <[email protected]>
Signed-off-by: Thomas Gleixner <[email protected]>
Reviewed-by: Reviewed-by: Borislav Petkov <[email protected]>
Cc: Peter Zijlstra <[email protected]>
Cc: Tim Chen <[email protected]>
Cc: Dave Hansen <[email protected]>
Cc: Borislav Petkov <[email protected]>
Cc: Dan Williams <[email protected]>
Cc: Linus Torvalds <[email protected]>
Cc: Greg Kroah-Hartman <[email protected]>
Cc: David Woodhouse <[email protected]>
Cc: Paul Turner <[email protected]>
Link: https://lkml.kernel.org/r/[email protected]
---
arch/x86/include/asm/msr-index.h | 2 ++
arch/x86/kernel/cpu/amd.c | 10 ++++++++++
2 files changed, 12 insertions(+)
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index ab02261..1e7d710 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -352,6 +352,8 @@
#define FAM10H_MMIO_CONF_BASE_MASK 0xfffffffULL
#define FAM10H_MMIO_CONF_BASE_SHIFT 20
#define MSR_FAM10H_NODE_ID 0xc001100c
+#define MSR_F10H_DECFG 0xc0011029
+#define MSR_F10H_DECFG_LFENCE_SERIALIZE_BIT 1
/* K8 MSRs */
#define MSR_K8_TOP_MEM1 0xc001001a
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index bcb75dc..5b438d8 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -829,6 +829,16 @@ static void init_amd(struct cpuinfo_x86 *c)
set_cpu_cap(c, X86_FEATURE_K8);
if (cpu_has(c, X86_FEATURE_XMM2)) {
+ /*
+ * A serializing LFENCE has less overhead than MFENCE, so
+ * use it for execution serialization. On families which
+ * don't have that MSR, LFENCE is already serializing.
+ * msr_set_bit() uses the safe accessors, too, even if the MSR
+ * is not present.
+ */
+ msr_set_bit(MSR_F10H_DECFG,
+ MSR_F10H_DECFG_LFENCE_SERIALIZE_BIT);
+
/* MFENCE stops RDTSC speculation */
set_cpu_cap(c, X86_FEATURE_MFENCE_RDTSC);
}
Commit-ID: 9c6a73c75864ad9fa49e5fa6513e4c4071c0e29f
Gitweb: https://git.kernel.org/tip/9c6a73c75864ad9fa49e5fa6513e4c4071c0e29f
Author: Tom Lendacky <[email protected]>
AuthorDate: Mon, 8 Jan 2018 16:09:32 -0600
Committer: Thomas Gleixner <[email protected]>
CommitDate: Tue, 9 Jan 2018 01:43:11 +0100
x86/cpu/AMD: Use LFENCE_RDTSC in preference to MFENCE_RDTSC
With LFENCE now a serializing instruction, use LFENCE_RDTSC in preference
to MFENCE_RDTSC. However, since the kernel could be running under a
hypervisor that does not support writing that MSR, read the MSR back and
verify that the bit has been set successfully. If the MSR can be read
and the bit is set, then set the LFENCE_RDTSC feature, otherwise set the
MFENCE_RDTSC feature.
Signed-off-by: Tom Lendacky <[email protected]>
Signed-off-by: Thomas Gleixner <[email protected]>
Reviewed-by: Reviewed-by: Borislav Petkov <[email protected]>
Cc: Peter Zijlstra <[email protected]>
Cc: Tim Chen <[email protected]>
Cc: Dave Hansen <[email protected]>
Cc: Borislav Petkov <[email protected]>
Cc: Dan Williams <[email protected]>
Cc: Linus Torvalds <[email protected]>
Cc: Greg Kroah-Hartman <[email protected]>
Cc: David Woodhouse <[email protected]>
Cc: Paul Turner <[email protected]>
Link: https://lkml.kernel.org/r/[email protected]
---
arch/x86/include/asm/msr-index.h | 1 +
arch/x86/kernel/cpu/amd.c | 18 ++++++++++++++++--
2 files changed, 17 insertions(+), 2 deletions(-)
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index 1e7d710..fa11fb1 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -354,6 +354,7 @@
#define MSR_FAM10H_NODE_ID 0xc001100c
#define MSR_F10H_DECFG 0xc0011029
#define MSR_F10H_DECFG_LFENCE_SERIALIZE_BIT 1
+#define MSR_F10H_DECFG_LFENCE_SERIALIZE BIT_ULL(MSR_F10H_DECFG_LFENCE_SERIALIZE_BIT)
/* K8 MSRs */
#define MSR_K8_TOP_MEM1 0xc001001a
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index 5b438d8..ea831c8 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -829,6 +829,9 @@ static void init_amd(struct cpuinfo_x86 *c)
set_cpu_cap(c, X86_FEATURE_K8);
if (cpu_has(c, X86_FEATURE_XMM2)) {
+ unsigned long long val;
+ int ret;
+
/*
* A serializing LFENCE has less overhead than MFENCE, so
* use it for execution serialization. On families which
@@ -839,8 +842,19 @@ static void init_amd(struct cpuinfo_x86 *c)
msr_set_bit(MSR_F10H_DECFG,
MSR_F10H_DECFG_LFENCE_SERIALIZE_BIT);
- /* MFENCE stops RDTSC speculation */
- set_cpu_cap(c, X86_FEATURE_MFENCE_RDTSC);
+ /*
+ * Verify that the MSR write was successful (could be running
+ * under a hypervisor) and only then assume that LFENCE is
+ * serializing.
+ */
+ ret = rdmsrl_safe(MSR_F10H_DECFG, &val);
+ if (!ret && (val & MSR_F10H_DECFG_LFENCE_SERIALIZE)) {
+ /* A serializing LFENCE stops RDTSC speculation */
+ set_cpu_cap(c, X86_FEATURE_LFENCE_RDTSC);
+ } else {
+ /* MFENCE stops RDTSC speculation */
+ set_cpu_cap(c, X86_FEATURE_MFENCE_RDTSC);
+ }
}
/*