From: Ard Biesheuvel <[email protected]>
The boot CPU runs some early startup C code using a 1:1 mapping of
memory, which deviates from the normal kernel virtual mapping that is
used for calculating statically initialized pointer variables.
This makes it necessary to strictly limit which C code will actually be
called from that early boot path. Implement this by moving the early
startup code into __pitext.
Signed-off-by: Ard Biesheuvel <[email protected]>
---
arch/x86/include/asm/init.h | 2 --
arch/x86/kernel/head64.c | 9 ++++----
arch/x86/kernel/head_64.S | 24 ++++++++++++--------
3 files changed, 20 insertions(+), 15 deletions(-)
diff --git a/arch/x86/include/asm/init.h b/arch/x86/include/asm/init.h
index cc9ccf61b6bd..5f1d3c421f68 100644
--- a/arch/x86/include/asm/init.h
+++ b/arch/x86/include/asm/init.h
@@ -2,8 +2,6 @@
#ifndef _ASM_X86_INIT_H
#define _ASM_X86_INIT_H
-#define __head __section(".head.text")
-
struct x86_mapping_info {
void *(*alloc_pgt_page)(void *); /* allocate buf for page table */
void *context; /* context for alloc_pgt_page */
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index 58c58c66dec9..0ecd36f5326a 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -70,7 +70,8 @@ static struct desc_struct startup_gdt[GDT_ENTRIES] __initconst = {
asm("movq $" __stringify(sym) ", %0":"=r"(__v)); \
__v; })
-static unsigned long __head sme_postprocess_startup(struct boot_params *bp, pmdval_t *pmd)
+static unsigned long __pitext sme_postprocess_startup(struct boot_params *bp,
+ pmdval_t *pmd)
{
unsigned long vaddr, vaddr_end;
int i;
@@ -113,7 +114,7 @@ static unsigned long __head sme_postprocess_startup(struct boot_params *bp, pmdv
return sme_get_me_mask();
}
-unsigned long __head __startup_64(struct boot_params *bp)
+unsigned long __pitext __startup_64(struct boot_params *bp)
{
unsigned long physaddr = (unsigned long)_text;
unsigned long load_delta, *p;
@@ -508,7 +509,7 @@ void __init __noreturn x86_64_start_reservations(char *real_mode_data)
*/
static gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_data;
-static void early_load_idt(void (*handler)(void))
+static void __pitext early_load_idt(void (*handler)(void))
{
gate_desc *idt = bringup_idt_table;
struct desc_ptr bringup_idt_descr;
@@ -539,7 +540,7 @@ void early_setup_idt(void)
/*
* Setup boot CPU state needed before kernel switches to virtual addresses.
*/
-void __head startup_64_setup_env(void)
+void __pitext startup_64_setup_env(void)
{
struct desc_ptr startup_gdt_descr;
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index b0508e84f756..e671caafd932 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -42,6 +42,15 @@ L3_START_KERNEL = pud_index(__START_KERNEL_map)
__HEAD
.code64
SYM_CODE_START_NOALIGN(startup_64)
+ UNWIND_HINT_END_OF_STACK
+ jmp primary_startup_64
+SYM_CODE_END(startup_64)
+
+ __PITEXT
+#include "verify_cpu.S"
+#include "sev_verify_cbit.S"
+
+SYM_CODE_START_LOCAL(primary_startup_64)
UNWIND_HINT_END_OF_STACK
/*
* At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0,
@@ -131,10 +140,12 @@ SYM_CODE_START_NOALIGN(startup_64)
movq %rax, %cr3
/* Branch to the common startup code at its kernel virtual address */
- movq $common_startup_64, %rax
ANNOTATE_RETPOLINE_SAFE
- jmp *%rax
-SYM_CODE_END(startup_64)
+ jmp *.Lcommon_startup_64(%rip)
+SYM_CODE_END(primary_startup_64)
+
+ __INITRODATA
+SYM_DATA_LOCAL(.Lcommon_startup_64, .quad common_startup_64)
.text
SYM_CODE_START(secondary_startup_64)
@@ -410,9 +421,6 @@ SYM_INNER_LABEL(common_startup_64, SYM_L_LOCAL)
int3
SYM_CODE_END(secondary_startup_64)
-#include "verify_cpu.S"
-#include "sev_verify_cbit.S"
-
#if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_AMD_MEM_ENCRYPT)
/*
* Entry point for soft restart of a CPU. Invoked from xxx_play_dead() for
@@ -539,10 +547,8 @@ SYM_CODE_END(early_idt_handler_common)
* paravirtualized INTERRUPT_RETURN and pv-ops don't work that early.
*
* XXX it does, fix this.
- *
- * This handler will end up in the .init.text section and not be
- * available to boot secondary CPUs.
*/
+ __PITEXT
SYM_CODE_START_NOALIGN(vc_no_ghcb)
UNWIND_HINT_IRET_REGS offset=8
ENDBR
--
2.43.0.429.g432eaa2c6b-goog