The trampoline_start64() vector is used when a secondary CPU starts in
64-bit mode. The current implementation directly enters compatibility
mode. It is necessary to disable paging and re-enable it in the correct
paging mode: either 4- or 5-level, depending on the configuration.
The X86S[1] ISA does not support compatibility mode in ring 0, and
paging cannot be disabled.
The trampoline_start64() function is reworked to only enter compatibility
mode if it is necessary to change the paging mode. If the CPU is already
in the desired paging mode, it will proceed in long mode.
This change will allow a secondary CPU to boot on an X86S machine as
long as the CPU is already in the correct paging mode.
In the future, there will be a mechanism to switch between paging modes
without disabling paging.
[1] https://www.intel.com/content/www/us/en/developer/articles/technical/envisioning-future-simplified-architecture.html
Signed-off-by: Kirill A. Shutemov <[email protected]>
Reviewed-by: Andi Kleen <[email protected]>
Reviewed-by: Kai Huang <[email protected]>
Cc: Sean Christopherson <[email protected]>
---
v3:
- tr_cr4 is 32-bit, use 32-bit XOR to access it (Sean).
- Use 32-bit TEST instead of AND to check if LA57 different between
CR4 and tr_cr4 (Sean).
v2:
- Fix build with GCC;
---
arch/x86/realmode/rm/trampoline_64.S | 31 +++++++++++++++++++++++++++-
1 file changed, 30 insertions(+), 1 deletion(-)
diff --git a/arch/x86/realmode/rm/trampoline_64.S b/arch/x86/realmode/rm/trampoline_64.S
index c9f76fae902e..608f108cba7d 100644
--- a/arch/x86/realmode/rm/trampoline_64.S
+++ b/arch/x86/realmode/rm/trampoline_64.S
@@ -37,13 +37,15 @@
.text
.code16
-.macro LOCK_AND_LOAD_REALMODE_ESP lock_pa=0
+.macro LOCK_AND_LOAD_REALMODE_ESP lock_pa=0 lock_rip=0
/*
* Make sure only one CPU fiddles with the realmode stack
*/
.Llock_rm\@:
.if \lock_pa
lock btsl $0, pa_tr_lock
+ .elseif \lock_rip
+ lock btsl $0, tr_lock(%rip)
.else
lock btsl $0, tr_lock
.endif
@@ -220,6 +222,33 @@ SYM_CODE_START(trampoline_start64)
lidt tr_idt(%rip)
lgdt tr_gdt64(%rip)
+ /* Check if paging mode has to be changed */
+ movq %cr4, %rax
+ xorl tr_cr4(%rip), %eax
+ testl $X86_CR4_LA57, %eax
+ jnz .L_switch_paging
+
+ /* Paging mode is correct proceed in 64-bit mode */
+
+ LOCK_AND_LOAD_REALMODE_ESP lock_rip=1
+
+ movw $__KERNEL_DS, %dx
+ movl %edx, %ss
+ addl $pa_real_mode_base, %esp
+ movl %edx, %ds
+ movl %edx, %es
+ movl %edx, %fs
+ movl %edx, %gs
+
+ movl $pa_trampoline_pgd, %eax
+ movq %rax, %cr3
+
+ jmpq *tr_start(%rip)
+.L_switch_paging:
+ /*
+ * To switch between 4- and 5-level paging modes, it is necessary
+ * to disable paging. This must be done in the compatibility mode.
+ */
ljmpl *tr_compat(%rip)
SYM_CODE_END(trampoline_start64)
--
2.43.0
> + /* Paging mode is correct proceed in 64-bit mode */
> +
> + LOCK_AND_LOAD_REALMODE_ESP lock_rip=1
> +
> + movw $__KERNEL_DS, %dx
> + movl %edx, %ss
> + addl $pa_real_mode_base, %esp
> + movl %edx, %ds
> + movl %edx, %es
> + movl %edx, %fs
> + movl %edx, %gs
> +
> + movl $pa_trampoline_pgd, %eax
> + movq %rax, %cr3
> +
> + jmpq *tr_start(%rip)
Still think we should add a far jump here so that we run on a defined
code segment. It probably doesn't matter since there are likely no
IRETs before reloading anyways, but it seems cleaner.
-Andi
On January 25, 2024 1:57:18 AM PST, Andi Kleen <[email protected]> wrote:
>> + /* Paging mode is correct proceed in 64-bit mode */
>> +
>> + LOCK_AND_LOAD_REALMODE_ESP lock_rip=1
>> +
>> + movw $__KERNEL_DS, %dx
>> + movl %edx, %ss
>> + addl $pa_real_mode_base, %esp
>> + movl %edx, %ds
>> + movl %edx, %es
>> + movl %edx, %fs
>> + movl %edx, %gs
>> +
>> + movl $pa_trampoline_pgd, %eax
>> + movq %rax, %cr3
>> +
>> + jmpq *tr_start(%rip)
>
>Still think we should add a far jump here so that we run on a defined
>code segment. It probably doesn't matter since there are likely no
>IRETs before reloading anyways, but it seems cleaner.
>
>-Andi
>
Agreed.
On Thu, Jan 25, 2024 at 01:57:18AM -0800, Andi Kleen wrote:
> > + /* Paging mode is correct proceed in 64-bit mode */
> > +
> > + LOCK_AND_LOAD_REALMODE_ESP lock_rip=1
> > +
> > + movw $__KERNEL_DS, %dx
> > + movl %edx, %ss
> > + addl $pa_real_mode_base, %esp
> > + movl %edx, %ds
> > + movl %edx, %es
> > + movl %edx, %fs
> > + movl %edx, %gs
> > +
> > + movl $pa_trampoline_pgd, %eax
> > + movq %rax, %cr3
> > +
> > + jmpq *tr_start(%rip)
>
> Still think we should add a far jump here so that we run on a defined
> code segment. It probably doesn't matter since there are likely no
> IRETs before reloading anyways, but it seems cleaner.
I think it is cleaner to switch to IRET here. Does this work for you?
diff --git a/arch/x86/realmode/rm/trampoline_64.S b/arch/x86/realmode/rm/trampoline_64.S
index 608f108cba7d..14d9c7daf90f 100644
--- a/arch/x86/realmode/rm/trampoline_64.S
+++ b/arch/x86/realmode/rm/trampoline_64.S
@@ -243,7 +243,9 @@ SYM_CODE_START(trampoline_start64)
movl $pa_trampoline_pgd, %eax
movq %rax, %cr3
- jmpq *tr_start(%rip)
+ pushq $__KERNEL_CS
+ pushq tr_start(%rip)
+ lretq
.L_switch_paging:
/*
* To switch between 4- and 5-level paging modes, it is necessary
--
Kiryl Shutsemau / Kirill A. Shutemov
> I think it is cleaner to switch to IRET here. Does this work for you?
>
> diff --git a/arch/x86/realmode/rm/trampoline_64.S b/arch/x86/realmode/rm/trampoline_64.S
> index 608f108cba7d..14d9c7daf90f 100644
> --- a/arch/x86/realmode/rm/trampoline_64.S
> +++ b/arch/x86/realmode/rm/trampoline_64.S
> @@ -243,7 +243,9 @@ SYM_CODE_START(trampoline_start64)
> movl $pa_trampoline_pgd, %eax
> movq %rax, %cr3
>
> - jmpq *tr_start(%rip)
> + pushq $__KERNEL_CS
> + pushq tr_start(%rip)
> + lretq
Looks good. Thanks.
-Andi