These patches enable Intel Xeon Phi x200 feature to use MONITOR/MWAIT
instruction in ring 3 (userspace) Patches set MSR 0x140 for all logical CPUs.
Then expose it as CPU feature and introduces elf HWCAP capability for x86.
Reference:
https://software.intel.com/en-us/blogs/2016/10/06/intel-xeon-phi-product-family-x200-knl-user-mode-ring-3-monitor-and-mwait
Grzegorz Andrejczuk (4):
Add R3MWAIT register and bit to msr-info.h
Add enabling of the R3 MWAIT during boot for KNL
Add hwcap2 for x86
Add R3MWAIT to CPU features
arch/x86/include/asm/cpufeature.h | 6 ++++--
arch/x86/include/asm/cpufeatures.h | 6 +++++-
arch/x86/include/asm/disabled-features.h | 3 ++-
arch/x86/include/asm/elf.h | 7 +++++++
arch/x86/include/asm/msr-index.h | 5 +++++
arch/x86/include/asm/required-features.h | 3 ++-
arch/x86/include/uapi/asm/hwcap.h | 7 +++++++
arch/x86/kernel/cpu/common.c | 6 ++++++
arch/x86/kernel/cpu/intel.c | 27 +++++++++++++++++++++++++++
9 files changed, 65 insertions(+), 5 deletions(-)
create mode 100644 arch/x86/include/uapi/asm/hwcap.h
--
2.5.1
Add hwcap2 attribute for x86.
Reserve 1st bit of HWCAP2 for exposing Xeon Phi ring 3 monitor/mwait.
With this userspace apps can detect Ring 3 MONITOR/MWAIT instructions.
Change-Id: I37d0354d1e2b9594d7feebc2bacda30b68163efe
Signed-off-by: Grzegorz Andrejczuk <[email protected]>
---
arch/x86/include/asm/elf.h | 7 +++++++
arch/x86/include/uapi/asm/hwcap.h | 7 +++++++
arch/x86/kernel/cpu/common.c | 3 +++
3 files changed, 17 insertions(+)
create mode 100644 arch/x86/include/uapi/asm/hwcap.h
diff --git a/arch/x86/include/asm/elf.h b/arch/x86/include/asm/elf.h
index e7f155c..62d060a 100644
--- a/arch/x86/include/asm/elf.h
+++ b/arch/x86/include/asm/elf.h
@@ -258,6 +258,13 @@ extern int force_personality32;
#define ELF_HWCAP (boot_cpu_data.x86_capability[CPUID_1_EDX])
+extern unsigned int elf_hwcap2
+
+/* HWCAP2 supplies kernel enabled CPU feature, so that the application
+ can know that it can safely use them. The bits are defined in
+ uapi/asm/hwcap.h. */
+#define ELF_HWCAP2 elf_hwcap2
+
/* This yields a string that ld.so will use to load implementation
specific libraries for optimization. This is more specific in
intent than poking at uname or /proc/cpuinfo.
diff --git a/arch/x86/include/uapi/asm/hwcap.h b/arch/x86/include/uapi/asm/hwcap.h
new file mode 100644
index 0000000..d1f4f98
--- /dev/null
+++ b/arch/x86/include/uapi/asm/hwcap.h
@@ -0,0 +1,7 @@
+#ifndef _ASM_HWCAP_H
+#define _ASM_HWCAP_H 1
+
+/* Kernel enabled Ring 3 MWAIT for Xeon Phi*/
+#define HWCAP2_PHIR3MWAIT (1 << 0)
+/* upto bit 31 free */
+#endif
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index bcc9ccc..93ffaa5 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -35,6 +35,7 @@
#include <asm/desc.h>
#include <asm/fpu/internal.h>
#include <asm/mtrr.h>
+#include <asm/hwcap.h>
#include <linux/numa.h>
#include <asm/asm.h>
#include <asm/bugs.h>
@@ -51,6 +52,8 @@
#include "cpu.h"
+unsigned elf_hwcap2 __read_mostly;
+
/* all of these masks are initialized in setup_cpu_local_masks() */
cpumask_var_t cpu_initialized_mask;
cpumask_var_t cpu_callout_mask;
--
2.5.1
If processor is Intel Xeon Phi we enable user-level mwait feature.
Enabling this feature suppreses invalid-opcode error, when MONITOR/MWAIT
is called from ring 3.
Change-Id: I1c7defb99296b022790a068a6c725b3e860cd68c
Signed-off-by: Grzegorz Andrejczuk <[email protected]>
---
arch/x86/kernel/cpu/intel.c | 26 ++++++++++++++++++++++++++
1 file changed, 26 insertions(+)
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index fcd484d..7f0f01a 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -61,6 +61,14 @@ void check_mpx_erratum(struct cpuinfo_x86 *c)
}
}
+static int phir3mwait = 1;
+static int __init phir3mwait_disable(char *value)
+{
+ phir3mwait = 0;
+ return 1;
+}
+__setup("intel-phir3mwait=disable", phir3mwait_disable);
+
static void early_init_intel(struct cpuinfo_x86 *c)
{
u64 misc_enable;
@@ -211,6 +219,24 @@ static void early_init_intel(struct cpuinfo_x86 *c)
}
check_mpx_erratum(c);
+
+ /*
+ * Setting ring 3 MONITOR/MWAIT for all threads
+ * when CPU is Xeon Phi Family x200
+ * This can be disabled with phir3mwait=disable cmdline switch.
+ * We preserve the reserved values and set only 2nd bit.
+ * Ref:
+ * https://software.intel.com/en-us/blogs/2016/10/06/intel-xeon-phi-product-family-x200-knl-user-mode-ring-3-monitor-and-mwait
+ */
+ if (c->x86 == 6 &&
+ c->x86_model == INTEL_FAM6_XEON_PHI_KNL &&
+ phir3mwait) {
+ u64 prev;
+
+ rdmsrl(MSR_PHI_MISC_THD_FEATURE_ENABLE, prev);
+ wrmsrl(MSR_PHI_MISC_THD_FEATURE_ENABLE,
+ prev | MSR_PHI_MISC_THD_FEATURE_ENABLE_R3MWAIT);
+ }
}
#ifdef CONFIG_X86_32
--
2.5.1
Add cpu feature for ring 3 monitor/mwait.
Change-Id: Iba4d20639efd8d3637d37db9294cbc43a98f009a
Signed-off-by: Grzegorz Andrejczuk <[email protected]>
---
arch/x86/include/asm/cpufeature.h | 6 ++++--
arch/x86/include/asm/cpufeatures.h | 6 +++++-
arch/x86/include/asm/disabled-features.h | 3 ++-
arch/x86/include/asm/required-features.h | 3 ++-
arch/x86/kernel/cpu/common.c | 3 +++
arch/x86/kernel/cpu/intel.c | 1 +
6 files changed, 17 insertions(+), 5 deletions(-)
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index 1d2b69f..1baa1df 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -78,8 +78,9 @@ extern const char * const x86_bug_flags[NBUGINTS*32];
CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 15, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 16, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 17, feature_bit) || \
+ CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 18, feature_bit) || \
REQUIRED_MASK_CHECK || \
- BUILD_BUG_ON_ZERO(NCAPINTS != 18))
+ BUILD_BUG_ON_ZERO(NCAPINTS != 19))
#define DISABLED_MASK_BIT_SET(feature_bit) \
( CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 0, feature_bit) || \
@@ -100,8 +101,9 @@ extern const char * const x86_bug_flags[NBUGINTS*32];
CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 15, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 16, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 17, feature_bit) || \
+ CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 18, feature_bit) || \
DISABLED_MASK_CHECK || \
- BUILD_BUG_ON_ZERO(NCAPINTS != 18))
+ BUILD_BUG_ON_ZERO(NCAPINTS != 19))
#define cpu_has(c, bit) \
(__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 : \
diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index 92a8308..242cd16 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -12,7 +12,7 @@
/*
* Defines x86 CPU feature bits
*/
-#define NCAPINTS 18 /* N 32-bit words worth of info */
+#define NCAPINTS 19 /* N 32-bit words worth of info */
#define NBUGINTS 1 /* N 32-bit bug flags */
/*
@@ -286,6 +286,10 @@
#define X86_FEATURE_SUCCOR (17*32+1) /* Uncorrectable error containment and recovery */
#define X86_FEATURE_SMCA (17*32+3) /* Scalable MCA */
+
+/* non architectural Intel-defined CPU features not present in CPUID, word 18 */
+#define X86_FEATURE_PHIR3MWAIT (18*32+ 0)
+
/*
* BUG word(s)
*/
diff --git a/arch/x86/include/asm/disabled-features.h b/arch/x86/include/asm/disabled-features.h
index 85599ad..8b45e08 100644
--- a/arch/x86/include/asm/disabled-features.h
+++ b/arch/x86/include/asm/disabled-features.h
@@ -57,6 +57,7 @@
#define DISABLED_MASK15 0
#define DISABLED_MASK16 (DISABLE_PKU|DISABLE_OSPKE)
#define DISABLED_MASK17 0
-#define DISABLED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 18)
+#define DISABLED_MASK18 0
+#define DISABLED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 19)
#endif /* _ASM_X86_DISABLED_FEATURES_H */
diff --git a/arch/x86/include/asm/required-features.h b/arch/x86/include/asm/required-features.h
index fac9a5c..6847d85 100644
--- a/arch/x86/include/asm/required-features.h
+++ b/arch/x86/include/asm/required-features.h
@@ -100,6 +100,7 @@
#define REQUIRED_MASK15 0
#define REQUIRED_MASK16 0
#define REQUIRED_MASK17 0
-#define REQUIRED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 18)
+#define REQUIRED_MASK18 0
+#define REQUIRED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 19)
#endif /* _ASM_X86_REQUIRED_FEATURES_H */
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 93ffaa5..15fe27f 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1108,6 +1108,9 @@ static void identify_cpu(struct cpuinfo_x86 *c)
#endif
/* The boot/hotplug time assigment got cleared, restore it */
c->logical_proc_id = topology_phys_to_logical_pkg(c->phys_proc_id);
+
+ if (cpu_has(c, X86_FEATURE_PHIR3MWAIT))
+ elf_hwcap2 |= HWCAP2_PHIR3MWAIT;
}
/*
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index 7f0f01a..1f65815 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -236,6 +236,7 @@ static void early_init_intel(struct cpuinfo_x86 *c)
rdmsrl(MSR_PHI_MISC_THD_FEATURE_ENABLE, prev);
wrmsrl(MSR_PHI_MISC_THD_FEATURE_ENABLE,
prev | MSR_PHI_MISC_THD_FEATURE_ENABLE_R3MWAIT);
+ set_cpu_cap(c, X86_FEATURE_PHIR3MWAIT);
}
}
--
2.5.1
Intel Xeon Phi x200 (codenamed Knights Landing) has MSR
MISC_THD_FEATURE_ENABLE 0x140.
Setting its 2nd bit make MONITOR and MWAIT instructions do not cause
invalid-opcode exception.
This commit adds this register prefixed by PHI and bit to msr-info.h
Reference:
https://software.intel.com/en-us/blogs/2016/10/06/intel-xeon-phi-product-family-x200-knl-user-mode-ring-3-monitor-and-mwait
Change-Id: If3b14c78f4e66d734e5a00921023a8c7cafc0cf3
Signed-off-by: Grzegorz Andrejczuk <[email protected]>
---
arch/x86/include/asm/msr-index.h | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index 56f4c66..3eb1713 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -540,6 +540,11 @@
#define MSR_IA32_MISC_ENABLE_IP_PREF_DISABLE_BIT 39
#define MSR_IA32_MISC_ENABLE_IP_PREF_DISABLE (1ULL << MSR_IA32_MISC_ENABLE_IP_PREF_DISABLE_BIT)
+/* Intel Xeon Phi x200 ring 3 MONITOR/MWAIT */
+#define MSR_PHI_MISC_THD_FEATURE_ENABLE 0x00000140
+#define MSR_PHI_MISC_THD_FEATURE_ENABLE_R3MWAIT_BIT 1
+#define MSR_PHI_MISC_THD_FEATURE_ENABLE_R3MWAIT (1ULL << MSR_PHI_MISC_THD_FEATURE_ENABLE_R3MWAIT_BIT)
+
#define MSR_IA32_TSC_DEADLINE 0x000006E0
/* P4/Xeon+ specific */
--
2.5.1
Add hwcap2 attribute for x86.
Reserve 1st bit of HWCAP2 for exposing Xeon Phi ring 3 monitor/mwait.
With this userspace apps can detect Ring 3 MONITOR/MWAIT instructions.
Change-Id: I37d0354d1e2b9594d7feebc2bacda30b68163efe
Signed-off-by: Grzegorz Andrejczuk <[email protected]>
---
arch/x86/include/asm/elf.h | 7 +++++++
arch/x86/include/uapi/asm/hwcap.h | 7 +++++++
arch/x86/kernel/cpu/common.c | 3 +++
3 files changed, 17 insertions(+)
create mode 100644 arch/x86/include/uapi/asm/hwcap.h
diff --git a/arch/x86/include/asm/elf.h b/arch/x86/include/asm/elf.h
index e7f155c..a3f7856 100644
--- a/arch/x86/include/asm/elf.h
+++ b/arch/x86/include/asm/elf.h
@@ -258,6 +258,13 @@ extern int force_personality32;
#define ELF_HWCAP (boot_cpu_data.x86_capability[CPUID_1_EDX])
+extern unsigned int elf_hwcap2;
+
+/* HWCAP2 supplies kernel enabled CPU feature, so that the application
+ can know that it can safely use them. The bits are defined in
+ uapi/asm/hwcap.h. */
+#define ELF_HWCAP2 elf_hwcap2
+
/* This yields a string that ld.so will use to load implementation
specific libraries for optimization. This is more specific in
intent than poking at uname or /proc/cpuinfo.
diff --git a/arch/x86/include/uapi/asm/hwcap.h b/arch/x86/include/uapi/asm/hwcap.h
new file mode 100644
index 0000000..d1f4f98
--- /dev/null
+++ b/arch/x86/include/uapi/asm/hwcap.h
@@ -0,0 +1,7 @@
+#ifndef _ASM_HWCAP_H
+#define _ASM_HWCAP_H 1
+
+/* Kernel enabled Ring 3 MWAIT for Xeon Phi*/
+#define HWCAP2_PHIR3MWAIT (1 << 0)
+/* upto bit 31 free */
+#endif
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index bcc9ccc..93ffaa5 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -35,6 +35,7 @@
#include <asm/desc.h>
#include <asm/fpu/internal.h>
#include <asm/mtrr.h>
+#include <asm/hwcap.h>
#include <linux/numa.h>
#include <asm/asm.h>
#include <asm/bugs.h>
@@ -51,6 +52,8 @@
#include "cpu.h"
+unsigned elf_hwcap2 __read_mostly;
+
/* all of these masks are initialized in setup_cpu_local_masks() */
cpumask_var_t cpu_initialized_mask;
cpumask_var_t cpu_callout_mask;
--
2.5.1
On Wed, Oct 12, 2016 at 12:13:10PM +0200, Grzegorz Andrejczuk wrote:
> Add cpu feature for ring 3 monitor/mwait.
>
> Change-Id: Iba4d20639efd8d3637d37db9294cbc43a98f009a
Please no internal IDs in upstream submission.
> Signed-off-by: Grzegorz Andrejczuk <[email protected]>
...
> diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
> index 92a8308..242cd16 100644
> --- a/arch/x86/include/asm/cpufeatures.h
> +++ b/arch/x86/include/asm/cpufeatures.h
> @@ -12,7 +12,7 @@
> /*
> * Defines x86 CPU feature bits
> */
> -#define NCAPINTS 18 /* N 32-bit words worth of info */
> +#define NCAPINTS 19 /* N 32-bit words worth of info */
> #define NBUGINTS 1 /* N 32-bit bug flags */
>
> /*
> @@ -286,6 +286,10 @@
> #define X86_FEATURE_SUCCOR (17*32+1) /* Uncorrectable error containment and recovery */
> #define X86_FEATURE_SMCA (17*32+3) /* Scalable MCA */
>
> +
> +/* non architectural Intel-defined CPU features not present in CPUID, word 18 */
> +#define X86_FEATURE_PHIR3MWAIT (18*32+ 0)
Please use init_scattered_cpuid_features() for the whole
thing. There are some free bits in word 3 for example, see
arch/x86/include/asm/cpufeatures.h.
--
Regards/Gruss,
Boris.
SUSE Linux GmbH, GF: Felix Imendörffer, Jane Smithard, Graham Norton, HRB 21284 (AG Nürnberg)
--
On Wed, Oct 12, 2016 at 12:13:08PM +0200, Grzegorz Andrejczuk wrote:
> If processor is Intel Xeon Phi we enable user-level mwait feature.
> Enabling this feature suppreses invalid-opcode error, when MONITOR/MWAIT
> is called from ring 3.
>
> Change-Id: I1c7defb99296b022790a068a6c725b3e860cd68c
> Signed-off-by: Grzegorz Andrejczuk <[email protected]>
> ---
> arch/x86/kernel/cpu/intel.c | 26 ++++++++++++++++++++++++++
> 1 file changed, 26 insertions(+)
>
> diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
> index fcd484d..7f0f01a 100644
> --- a/arch/x86/kernel/cpu/intel.c
> +++ b/arch/x86/kernel/cpu/intel.c
> @@ -61,6 +61,14 @@ void check_mpx_erratum(struct cpuinfo_x86 *c)
> }
> }
>
> +static int phir3mwait = 1;
> +static int __init phir3mwait_disable(char *value)
> +{
> + phir3mwait = 0;
> + return 1;
> +}
> +__setup("intel-phir3mwait=disable", phir3mwait_disable);
That's a lot of typing on the cmdline. "r3mwait=disable" looks just as
fine to me, for example.
> static void early_init_intel(struct cpuinfo_x86 *c)
> {
> u64 misc_enable;
> @@ -211,6 +219,24 @@ static void early_init_intel(struct cpuinfo_x86 *c)
> }
>
> check_mpx_erratum(c);
> +
> + /*
> + * Setting ring 3 MONITOR/MWAIT for all threads
> + * when CPU is Xeon Phi Family x200
> + * This can be disabled with phir3mwait=disable cmdline switch.
> + * We preserve the reserved values and set only 2nd bit.
> + * Ref:
> + * https://software.intel.com/en-us/blogs/2016/10/06/intel-xeon-phi-product-family-x200-knl-user-mode-ring-3-monitor-and-mwait
> + */
> + if (c->x86 == 6 &&
> + c->x86_model == INTEL_FAM6_XEON_PHI_KNL &&
> + phir3mwait) {
> + u64 prev;
> +
> + rdmsrl(MSR_PHI_MISC_THD_FEATURE_ENABLE, prev);
> + wrmsrl(MSR_PHI_MISC_THD_FEATURE_ENABLE,
> + prev | MSR_PHI_MISC_THD_FEATURE_ENABLE_R3MWAIT);
Wanna test the MSR_PHI_MISC_THD_FEATURE_ENABLE_R3MWAIT bit before doing
the MSR write?
Btw, you might want to shorten those define names - they're huuge.
--
Regards/Gruss,
Boris.
SUSE Linux GmbH, GF: Felix Imendörffer, Jane Smithard, Graham Norton, HRB 21284 (AG Nürnberg)
--