Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1753868AbeAFCeT (ORCPT + 1 other); Fri, 5 Jan 2018 21:34:19 -0500 Received: from mga03.intel.com ([134.134.136.65]:10717 "EHLO mga03.intel.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1753694AbeAFCdG (ORCPT ); Fri, 5 Jan 2018 21:33:06 -0500 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.46,320,1511856000"; d="scan'208";a="190375383" From: Tim Chen To: Thomas Gleixner , Andy Lutomirski , Linus Torvalds , Greg KH Cc: Tim Chen , Dave Hansen , Andrea Arcangeli , Andi Kleen , Arjan Van De Ven , David Woodhouse , linux-kernel@vger.kernel.org Subject: [PATCH v2 4/8] x86/spec_ctrl: Add sysctl knobs to enable/disable SPEC_CTRL feature Date: Fri, 5 Jan 2018 18:12:19 -0800 Message-Id: X-Mailer: git-send-email 2.9.4 In-Reply-To: References: In-Reply-To: References: Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Return-Path: From: Tim Chen From: Andrea Arcangeli There are 2 ways to control IBRS 1. At boot time noibrs kernel boot parameter will disable IBRS usage Otherwise if the above parameters are not specified, the system will enable ibrs and ibpb usage if the cpu supports it. 2. At run time echo 0 > /sys/kernel/debug/x86/ibrs_enabled will turn off IBRS echo 1 > /sys/kernel/debug/x86/ibrs_enabled will turn on IBRS in kernel echo 2 > /sys/kernel/debug/x86/ibrs_enabled will turn on IBRS in both userspace and kernel The implementation was updated with input and suggestions from Andrea Arcangeli. Signed-off-by: Tim Chen --- arch/x86/entry/calling.h | 42 ++++++++-- arch/x86/include/asm/spec_ctrl.h | 15 ++++ arch/x86/kernel/cpu/Makefile | 1 + arch/x86/kernel/cpu/scattered.c | 2 + arch/x86/kernel/cpu/spec_ctrl.c | 160 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 214 insertions(+), 6 deletions(-) create mode 100644 arch/x86/include/asm/spec_ctrl.h create mode 100644 arch/x86/kernel/cpu/spec_ctrl.c diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h index 09c870d..6b65d47 100644 --- a/arch/x86/entry/calling.h +++ b/arch/x86/entry/calling.h @@ -373,35 +373,55 @@ For 32-bit we have the following conventions - kernel is built with .endm .macro ENABLE_IBRS - ALTERNATIVE "jmp .Lskip_\@", "", X86_FEATURE_SPEC_CTRL + testl $1, dynamic_ibrs + jz .Lskip_\@ + PUSH_MSR_REGS WRMSR_ASM $MSR_IA32_SPEC_CTRL, $SPEC_CTRL_FEATURE_ENABLE_IBRS POP_MSR_REGS + jmp .Ldone_\@ + .Lskip_\@: + lfence +.Ldone_\@: .endm .macro DISABLE_IBRS - ALTERNATIVE "jmp .Lskip_\@", "", X86_FEATURE_SPEC_CTRL + testl $1, dynamic_ibrs + jz .Lskip_\@ + PUSH_MSR_REGS WRMSR_ASM $MSR_IA32_SPEC_CTRL, $SPEC_CTRL_FEATURE_DISABLE_IBRS POP_MSR_REGS + .Lskip_\@: .endm .macro ENABLE_IBRS_CLOBBER - ALTERNATIVE "jmp .Lskip_\@", "", X86_FEATURE_SPEC_CTRL + testl $1, dynamic_ibrs + jz .Lskip_\@ + WRMSR_ASM $MSR_IA32_SPEC_CTRL, $SPEC_CTRL_FEATURE_ENABLE_IBRS + jmp .Ldone_\@ + .Lskip_\@: + lfence +.Ldone_\@: .endm .macro DISABLE_IBRS_CLOBBER - ALTERNATIVE "jmp .Lskip_\@", "", X86_FEATURE_SPEC_CTRL + testl $1, dynamic_ibrs + jz .Lskip_\@ + WRMSR_ASM $MSR_IA32_SPEC_CTRL, $SPEC_CTRL_FEATURE_DISABLE_IBRS + .Lskip_\@: .endm .macro ENABLE_IBRS_SAVE_AND_CLOBBER save_reg:req - ALTERNATIVE "jmp .Lskip_\@", "", X86_FEATURE_SPEC_CTRL + testl $1, dynamic_ibrs + jz .Lskip_\@ + movl $MSR_IA32_SPEC_CTRL, %ecx rdmsr movl %eax, \save_reg @@ -409,15 +429,25 @@ For 32-bit we have the following conventions - kernel is built with movl $0, %edx movl $SPEC_CTRL_FEATURE_ENABLE_IBRS, %eax wrmsr + jmp .Ldone_\@ + .Lskip_\@: + lfence +.Ldone_\@: .endm .macro RESTORE_IBRS_CLOBBER save_reg:req - ALTERNATIVE "jmp .Lskip_\@", "", X86_FEATURE_SPEC_CTRL + testl $1, dynamic_ibrs + jz .Lskip_\@ + /* Set IBRS to the value saved in the save_reg */ movl $MSR_IA32_SPEC_CTRL, %ecx movl $0, %edx movl \save_reg, %eax wrmsr + jmp .Ldone_\@ + .Lskip_\@: + lfence +.Ldone_\@: .endm diff --git a/arch/x86/include/asm/spec_ctrl.h b/arch/x86/include/asm/spec_ctrl.h new file mode 100644 index 0000000..4fda38b --- /dev/null +++ b/arch/x86/include/asm/spec_ctrl.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_X86_SPEC_CTRL_H +#define _ASM_X86_SPEC_CTRL_H + +#include +#include +#include + +void scan_spec_ctrl_feature(struct cpuinfo_x86 *c); +bool ibrs_inuse(void); + +extern unsigned int dynamic_ibrs; + +#endif /* _ASM_X86_SPEC_CTRL_H */ diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile index 570e8bb..3ffbd24 100644 --- a/arch/x86/kernel/cpu/Makefile +++ b/arch/x86/kernel/cpu/Makefile @@ -24,6 +24,7 @@ obj-y += match.o obj-y += bugs.o obj-y += aperfmperf.o obj-y += cpuid-deps.o +obj-y += spec_ctrl.o obj-$(CONFIG_PROC_FS) += proc.o obj-$(CONFIG_X86_FEATURE_NAMES) += capflags.o powerflags.o diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c index bc50c40..5756d14 100644 --- a/arch/x86/kernel/cpu/scattered.c +++ b/arch/x86/kernel/cpu/scattered.c @@ -8,6 +8,7 @@ #include #include +#include struct cpuid_bit { u16 feature; @@ -57,6 +58,7 @@ void init_scattered_cpuid_features(struct cpuinfo_x86 *c) if (regs[cb->reg] & (1 << cb->bit)) set_cpu_cap(c, cb->feature); } + scan_spec_ctrl_feature(c); } u32 get_scattered_cpuid_leaf(unsigned int level, unsigned int sub_leaf, diff --git a/arch/x86/kernel/cpu/spec_ctrl.c b/arch/x86/kernel/cpu/spec_ctrl.c new file mode 100644 index 0000000..1641bec --- /dev/null +++ b/arch/x86/kernel/cpu/spec_ctrl.c @@ -0,0 +1,160 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include +#include +#include + +#include +#include + +unsigned int dynamic_ibrs __read_mostly; +EXPORT_SYMBOL_GPL(dynamic_ibrs); + +enum { + IBRS_DISABLED, + /* in host kernel, disabled in guest and userland */ + IBRS_ENABLED, + /* in host kernel and host userland, disabled in guest */ + IBRS_ENABLED_USER, + IBRS_MAX = IBRS_ENABLED_USER, +}; +static unsigned int ibrs_enabled; +static bool ibrs_admin_disabled; + +/* mutex to serialize IBRS control changes */ +DEFINE_MUTEX(spec_ctrl_mutex); + +void scan_spec_ctrl_feature(struct cpuinfo_x86 *c) +{ + if ((!c->cpu_index) && (boot_cpu_has(X86_FEATURE_SPEC_CTRL))) { + if (!ibrs_admin_disabled) { + dynamic_ibrs = 1; + ibrs_enabled = IBRS_ENABLED; + } + } +} +EXPORT_SYMBOL_GPL(scan_spec_ctrl_feature); + +/* + * Used after boot phase to rescan spec_ctrl feature, + * serialize scan with spec_ctrl_mutex. + */ +void rescan_spec_ctrl_feature(struct cpuinfo_x86 *c) +{ + mutex_lock(&spec_ctrl_mutex); + if (boot_cpu_has(X86_FEATURE_SPEC_CTRL)) { + if (!ibrs_admin_disabled) { + dynamic_ibrs = 1; + ibrs_enabled = IBRS_ENABLED; + } + } + mutex_unlock(&spec_ctrl_mutex); +} +EXPORT_SYMBOL_GPL(rescan_spec_ctrl_feature); + +bool ibrs_inuse(void) +{ + return ibrs_enabled == IBRS_ENABLED; +} +EXPORT_SYMBOL_GPL(ibrs_inuse); + +static int __init noibrs(char *str) +{ + ibrs_admin_disabled = true; + ibrs_enabled = IBRS_DISABLED; + + return 0; +} +early_param("noibrs", noibrs); + +static ssize_t __enabled_read(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos, unsigned int *field) +{ + char buf[32]; + unsigned int len; + + len = sprintf(buf, "%d\n", READ_ONCE(*field)); + return simple_read_from_buffer(user_buf, count, ppos, buf, len); +} + +static ssize_t ibrs_enabled_read(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + return __enabled_read(file, user_buf, count, ppos, &ibrs_enabled); +} + +static void spec_ctrl_flush_all_cpus(u32 msr_nr, u64 val) +{ + int cpu; + + get_online_cpus(); + for_each_online_cpu(cpu) + wrmsrl_on_cpu(cpu, msr_nr, val); + put_online_cpus(); +} + +static ssize_t ibrs_enabled_write(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + char buf[32]; + ssize_t len; + unsigned int enable; + + len = min(count, sizeof(buf) - 1); + if (copy_from_user(buf, user_buf, len)) + return -EFAULT; + + buf[len] = '\0'; + if (kstrtouint(buf, 0, &enable)) + return -EINVAL; + + if (enable > IBRS_MAX) + return -EINVAL; + + if (!boot_cpu_has(X86_FEATURE_SPEC_CTRL)) { + ibrs_enabled = IBRS_DISABLED; + return -EINVAL; + } + + mutex_lock(&spec_ctrl_mutex); + + if (enable == IBRS_DISABLED) { + /* disable IBRS usage */ + ibrs_admin_disabled = true; + dynamic_ibrs = 0; + spec_ctrl_flush_all_cpus(MSR_IA32_SPEC_CTRL, + SPEC_CTRL_FEATURE_DISABLE_IBRS); + + } else if (enable == IBRS_ENABLED) { + /* enable IBRS usage in kernel */ + ibrs_admin_disabled = false; + dynamic_ibrs = 1; + + } else if (enable == IBRS_ENABLED_USER) { + /* enable IBRS all the time in both userspace and kernel */ + ibrs_admin_disabled = false; + dynamic_ibrs = 0; + spec_ctrl_flush_all_cpus(MSR_IA32_SPEC_CTRL, + SPEC_CTRL_FEATURE_ENABLE_IBRS); + } + + ibrs_enabled = enable; + + mutex_unlock(&spec_ctrl_mutex); + return count; +} + +static const struct file_operations fops_ibrs_enabled = { + .read = ibrs_enabled_read, + .write = ibrs_enabled_write, + .llseek = default_llseek, +}; + +static int __init debugfs_spec_ctrl(void) +{ + debugfs_create_file("ibrs_enabled", S_IRUSR | S_IWUSR, + arch_debugfs_dir, NULL, &fops_ibrs_enabled); + return 0; +} +late_initcall(debugfs_spec_ctrl); -- 2.9.4