Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1751356AbeACXJr (ORCPT + 1 other); Wed, 3 Jan 2018 18:09:47 -0500 Received: from mga03.intel.com ([134.134.136.65]:11021 "EHLO mga03.intel.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1751014AbeACXJp (ORCPT ); Wed, 3 Jan 2018 18:09:45 -0500 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.45,504,1508828400"; d="scan'208";a="21190667" From: Andi Kleen To: tglx@linuxtronix.de Cc: torvalds@linux-foundation.org, gregkh@linux-foundation.org, dwmw@amazon.co.uk, tim.c.chen@linux.intel.com, linux-kernel@vger.kernel.org, dave.hansen@intel.com, Dave Hansen , Andi Kleen Subject: [PATCH 01/11] x86/retpoline: Define retpoline indirect thunk and macros Date: Wed, 3 Jan 2018 15:09:24 -0800 Message-Id: <20180103230934.15788-2-andi@firstfloor.org> X-Mailer: git-send-email 2.14.3 In-Reply-To: <20180103230934.15788-1-andi@firstfloor.org> References: <20180103230934.15788-1-andi@firstfloor.org> Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Return-Path: From: Dave Hansen From: David Woodhouse retpoline is a special sequence on Intel CPUs to stop speculation for indirect branches. Provide assembler infrastructure to use retpoline by the compiler and for assembler. We add the out of line trampoline used by the compiler, and NOSPEC_JUMP / NOSPEC_CALL macros for assembler [Originally from David and Tim, heavily hacked by AK] Signed-off-by: David Woodhouse Signed-off-by: Tim Chen Signed-off-by: Andi Kleen --- arch/x86/include/asm/jump-asm.h | 47 +++++++++++++++++++++++++++++++++++++++++ arch/x86/kernel/vmlinux.lds.S | 1 + arch/x86/lib/Makefile | 1 + arch/x86/lib/retpoline.S | 35 ++++++++++++++++++++++++++++++ 4 files changed, 84 insertions(+) create mode 100644 arch/x86/include/asm/jump-asm.h create mode 100644 arch/x86/lib/retpoline.S diff --git a/arch/x86/include/asm/jump-asm.h b/arch/x86/include/asm/jump-asm.h new file mode 100644 index 000000000000..953c391991b9 --- /dev/null +++ b/arch/x86/include/asm/jump-asm.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef JUMP_ASM_H +#define JUMP_ASM_H 1 + +#ifdef __ASSEMBLY__ + +/* + * Jump to an indirect pointer without speculation. + * + * The out of line __x86.indirect_thunk has special code sequences + * to stop speculation. + */ + +.macro NOSPEC_JMP target + push \target + jmp __x86.indirect_thunk +.endm + + +/* + * Call an indirect pointer without speculation. + */ + +.macro NOSPEC_CALL target + jmp 1221f +1222: + push \target + jmp __x86.indirect_thunk +1221: + call 1222b +.endm + +#else /* __ASSEMBLY__ */ + +#define NOSPEC_JUMP(t) \ + "push " t "; " \ + "jmp __x86.indirect_thunk; " + +#define NOSPEC_CALL(t) \ + " jmp 1221f; " \ + "1222: push " t ";" \ + " jmp __x86.indirect_thunk;" \ + "1221: call 1222b;" + +#endif /* !__ASSEMBLY */ + +#endif diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index 1e413a9326aa..2e64241a6664 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -103,6 +103,7 @@ SECTIONS /* bootstrapping code */ HEAD_TEXT . = ALIGN(8); + *(.text.__x86.indirect_thunk) TEXT_TEXT SCHED_TEXT CPUIDLE_TEXT diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile index 7b181b61170e..ec7a329b9b3c 100644 --- a/arch/x86/lib/Makefile +++ b/arch/x86/lib/Makefile @@ -26,6 +26,7 @@ lib-y += memcpy_$(BITS).o lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o insn-eval.o lib-$(CONFIG_RANDOMIZE_BASE) += kaslr.o +lib-y += retpoline.o obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S new file mode 100644 index 000000000000..cb40781adbfe --- /dev/null +++ b/arch/x86/lib/retpoline.S @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * Out of line jump trampoline for calls that disable speculation. + * + * This is a special sequence that prevents the CPU speculating + * for indirect calls. + * + * This can be called by gcc generated code, or with the asm macros + * in asm/jump-asm.h + */ + +#include +#include +#include + + .section .text.__x86.indirect_thunk,"ax" + +ENTRY(__x86.indirect_thunk) + CFI_STARTPROC + call retpoline_call_target +2: + lfence /* stop speculation */ + jmp 2b +retpoline_call_target: +#ifdef CONFIG_64BIT + lea 8(%rsp), %rsp +#else + lea 4(%esp), %esp +#endif + ret + CFI_ENDPROC +ENDPROC(__x86.indirect_thunk) + + EXPORT_SYMBOL(__x86.indirect_thunk) -- 2.14.3