Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S965259AbeALSqz (ORCPT + 1 other); Fri, 12 Jan 2018 13:46:55 -0500 Received: from mga05.intel.com ([192.55.52.43]:4981 "EHLO mga05.intel.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S965161AbeALSqC (ORCPT ); Fri, 12 Jan 2018 13:46:02 -0500 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.46,350,1511856000"; d="scan'208";a="19643789" From: Andi Kleen To: tglx@linutronix.de Cc: x86@kernel.org, dwmw@amazon.co.uk, linux-kernel@vger.kernel.org, pjt@google.com, torvalds@linux-foundation.org, gregkh@linux-foundation.org, peterz@infradead.org, luto@amacapital.net, thomas.lendacky@amd.com, arjan.van.de.ven@intel.com, Andi Kleen Subject: [PATCH 1/4] x86/retpoline: Add new mode RETPOLINE_UNDERFLOW Date: Fri, 12 Jan 2018 10:45:47 -0800 Message-Id: <20180112184550.6573-2-andi@firstfloor.org> X-Mailer: git-send-email 2.14.3 In-Reply-To: <20180112184550.6573-1-andi@firstfloor.org> References: <20180112184550.6573-1-andi@firstfloor.org> Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Return-Path: From: Andi Kleen On Skylake we want additional protections against spectre_v2 over the normal RETPOLINE against underflowing return buffers. On return buffer underflow the CPU could fall back to the poisoned indirect branch predictor. This is not needed on CPUs before Skylake. Add a new RETPOLINE_UNDERFLOW that enables the additional protections. This is RETPOLINE_GENERIC, but also adds additional mitigations for return buffer overflow/underflow. It enables a new FEATURE bit enabling the underflow protection mode. Automatically select this mode on Skylake Add a new fill_return_buffer() C function that depends on this mode. It will be used in follow-on patches. Signed-off-by: Andi Kleen --- arch/x86/include/asm/cpufeatures.h | 1 + arch/x86/include/asm/nospec-branch.h | 25 +++++++++++++++++++++++++ arch/x86/kernel/cpu/bugs.c | 30 ++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+) diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index f275447862f4..80610bcbce12 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -211,6 +211,7 @@ #define X86_FEATURE_AVX512_4FMAPS ( 7*32+17) /* AVX-512 Multiply Accumulation Single precision */ #define X86_FEATURE_MBA ( 7*32+18) /* Memory Bandwidth Allocation */ +#define X86_FEATURE_RETURN_UNDERFLOW ( 7*32+19) /* Avoid return stack underflows */ /* Virtualization flags: Linux defined, word 8 */ #define X86_FEATURE_TPR_SHADOW ( 8*32+ 0) /* Intel TPR Shadow */ diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index 402a11c803c3..780999a45b8c 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -186,6 +186,7 @@ enum spectre_v2_mitigation { SPECTRE_V2_RETPOLINE_MINIMAL, SPECTRE_V2_RETPOLINE_MINIMAL_AMD, SPECTRE_V2_RETPOLINE_GENERIC, + SPECTRE_V2_RETPOLINE_UNDERFLOW, SPECTRE_V2_RETPOLINE_AMD, SPECTRE_V2_IBRS, }; @@ -210,5 +211,29 @@ static inline void vmexit_fill_RSB(void) : "r" (loops) : "memory" ); #endif } + +/* + * Fill the return buffer to avoid return buffer overflow. + * + * This is different from the one above because it is controlled + * by a different feature bit. It should be used for any fixes + * for call chains deeper than 16, or the context switch. + */ +static inline void fill_return_buffer(void) +{ +#ifdef CONFIG_RETPOLINE + unsigned long loops = RSB_CLEAR_LOOPS / 2; + + asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE + ALTERNATIVE("jmp 910f", + __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)), + X86_FEATURE_RETURN_UNDERFLOW) + "910:" + : "=&r" (loops), ASM_CALL_CONSTRAINT + : "r" (loops) : "memory" ); +#endif +} + + #endif /* __ASSEMBLY__ */ #endif /* __NOSPEC_BRANCH_H__ */ diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index e4dc26185aa7..75addb8ef4a5 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -23,6 +23,7 @@ #include #include #include +#include static void __init spectre_v2_select_mitigation(void); @@ -77,6 +78,7 @@ enum spectre_v2_mitigation_cmd { SPECTRE_V2_CMD_FORCE, SPECTRE_V2_CMD_RETPOLINE, SPECTRE_V2_CMD_RETPOLINE_GENERIC, + SPECTRE_V2_CMD_RETPOLINE_UNDERFLOW, SPECTRE_V2_CMD_RETPOLINE_AMD, }; @@ -86,6 +88,7 @@ static const char *spectre_v2_strings[] = { [SPECTRE_V2_RETPOLINE_MINIMAL_AMD] = "Vulnerable: Minimal AMD ASM retpoline", [SPECTRE_V2_RETPOLINE_GENERIC] = "Mitigation: Full generic retpoline", [SPECTRE_V2_RETPOLINE_AMD] = "Mitigation: Full AMD retpoline", + [SPECTRE_V2_RETPOLINE_UNDERFLOW] = "Mitigation: Full retpoline with underflow protection", }; #undef pr_fmt @@ -117,6 +120,22 @@ static inline bool match_option(const char *arg, int arglen, const char *opt) return len == arglen && !strncmp(arg, opt, len); } +static bool cpu_needs_underflow_protection(void) +{ + if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL || + boot_cpu_data.x86 != 6) + return false; + switch (boot_cpu_data.x86_model) { + case INTEL_FAM6_SKYLAKE_MOBILE: + case INTEL_FAM6_SKYLAKE_DESKTOP: + case INTEL_FAM6_SKYLAKE_X: + case INTEL_FAM6_KABYLAKE_MOBILE: + case INTEL_FAM6_KABYLAKE_DESKTOP: + return true; + } + return false; +} + static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) { char arg[20]; @@ -143,6 +162,9 @@ static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) } else if (match_option(arg, ret, "retpoline,generic")) { spec2_print_if_insecure("generic retpoline selected on command line."); return SPECTRE_V2_CMD_RETPOLINE_GENERIC; + } else if (match_option(arg, ret, "retpoline,underflow")) { + spec2_print_if_insecure("generic retpoline with underflow protection selected on command line."); + return SPECTRE_V2_CMD_RETPOLINE_UNDERFLOW; } else if (match_option(arg, ret, "auto")) { return SPECTRE_V2_CMD_AUTO; } @@ -189,6 +211,9 @@ static void __init spectre_v2_select_mitigation(void) if (IS_ENABLED(CONFIG_RETPOLINE)) goto retpoline_auto; break; + case SPECTRE_V2_CMD_RETPOLINE_UNDERFLOW: + if (IS_ENABLED(CONFIG_RETPOLINE)) + goto retpoline_generic; } pr_err("kernel not compiled with retpoline; no mitigation available!"); return; @@ -209,6 +234,11 @@ static void __init spectre_v2_select_mitigation(void) mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_GENERIC : SPECTRE_V2_RETPOLINE_MINIMAL; setup_force_cpu_cap(X86_FEATURE_RETPOLINE); + if (mode == SPECTRE_V2_RETPOLINE_GENERIC && + cpu_needs_underflow_protection()) { + mode = SPECTRE_V2_RETPOLINE_UNDERFLOW; + setup_force_cpu_cap(X86_FEATURE_RETURN_UNDERFLOW); + } } spectre_v2_enabled = mode; -- 2.14.3