Now that the trampoline setup code and the actual invocation of it are
all done from the C routine, the trampoline cleanup can be merged into
it as well, instead of returning to asm just to call another C function.
Acked-by: Kirill A. Shutemov <[email protected]>
Signed-off-by: Ard Biesheuvel <[email protected]>
---
arch/x86/boot/compressed/head_64.S | 13 +++------
arch/x86/boot/compressed/pgtable_64.c | 28 ++++++++------------
2 files changed, 15 insertions(+), 26 deletions(-)
diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S
index 88b5e45b8ecb490b..eb33edf1e75d4b02 100644
--- a/arch/x86/boot/compressed/head_64.S
+++ b/arch/x86/boot/compressed/head_64.S
@@ -429,19 +429,14 @@ SYM_CODE_START(startup_64)
* set_paging_levels() updates the number of paging levels using a
* trampoline in 32-bit addressable memory if the current number does
* not match the desired number.
+ *
+ * RSI is the relocated address of the page table to use instead of
+ * page table in trampoline memory (if required).
*/
movq %r15, %rdi /* pass struct boot_params pointer */
+ leaq rva(top_pgtable)(%rbx), %rsi
call set_paging_levels
- /*
- * cleanup_trampoline() would restore trampoline memory.
- *
- * RDI is address of the page table to use instead of page table
- * in trampoline memory (if required).
- */
- leaq rva(top_pgtable)(%rbx), %rdi
- call cleanup_trampoline
-
/* Zero EFLAGS */
pushq $0
popfq
diff --git a/arch/x86/boot/compressed/pgtable_64.c b/arch/x86/boot/compressed/pgtable_64.c
index 4016444e6788304f..4f50af23a0854f18 100644
--- a/arch/x86/boot/compressed/pgtable_64.c
+++ b/arch/x86/boot/compressed/pgtable_64.c
@@ -101,9 +101,10 @@ static unsigned long find_trampoline_placement(void)
return bios_start - TRAMPOLINE_32BIT_SIZE;
}
-asmlinkage void set_paging_levels(void *rmode)
+asmlinkage void set_paging_levels(void *rmode, void *pgtable)
{
void (*toggle_la57)(void *trampoline, bool enable_5lvl);
+ void *trampoline_pgtable;
bool l5_required = false;
/* Initialize boot_params. Required for cmdline_find_option_bool(). */
@@ -133,7 +134,7 @@ asmlinkage void set_paging_levels(void *rmode)
* the desired one.
*/
if (l5_required == !!(native_read_cr4() & X86_CR4_LA57))
- return;
+ goto out;
trampoline_32bit = (unsigned long *)find_trampoline_placement();
@@ -163,6 +164,8 @@ asmlinkage void set_paging_levels(void *rmode)
* The new page table will be used by trampoline code for switching
* from 4- to 5-level paging or vice versa.
*/
+ trampoline_pgtable = trampoline_32bit +
+ TRAMPOLINE_32BIT_PGTABLE_OFFSET / sizeof(unsigned long);
if (l5_required) {
/*
@@ -182,31 +185,21 @@ asmlinkage void set_paging_levels(void *rmode)
* may be above 4G.
*/
src = *(unsigned long *)__native_read_cr3() & PAGE_MASK;
- memcpy(trampoline_32bit + TRAMPOLINE_32BIT_PGTABLE_OFFSET / sizeof(unsigned long),
- (void *)src, PAGE_SIZE);
+ memcpy(trampoline_pgtable, (void *)src, PAGE_SIZE);
}
toggle_la57(trampoline_32bit, l5_required);
-}
-
-void cleanup_trampoline(void *pgtable)
-{
- void *trampoline_pgtable;
-
- trampoline_pgtable = trampoline_32bit + TRAMPOLINE_32BIT_PGTABLE_OFFSET / sizeof(unsigned long);
/*
- * Move the top level page table out of trampoline memory,
- * if it's there.
+ * Move the top level page table out of trampoline memory.
*/
- if ((void *)__native_read_cr3() == trampoline_pgtable) {
- memcpy(pgtable, trampoline_pgtable, PAGE_SIZE);
- native_write_cr3((unsigned long)pgtable);
- }
+ memcpy(pgtable, trampoline_pgtable, PAGE_SIZE);
+ native_write_cr3((unsigned long)pgtable);
/* Restore trampoline memory */
memcpy(trampoline_32bit, trampoline_save, TRAMPOLINE_32BIT_SIZE);
+out:
/* Initialize variables for 5-level paging */
#ifdef CONFIG_X86_5LEVEL
if (__read_cr4() & X86_CR4_LA57) {
@@ -215,4 +208,5 @@ void cleanup_trampoline(void *pgtable)
ptrs_per_p4d = 512;
}
#endif
+ return;
}
--
2.39.2
On Tue, 1 Aug 2023 at 14:09, Borislav Petkov <[email protected]> wrote:
>
> On Fri, Jul 28, 2023 at 11:09:08AM +0200, Ard Biesheuvel wrote:
> > diff --git a/arch/x86/boot/compressed/pgtable_64.c b/arch/x86/boot/compressed/pgtable_64.c
> > index 4016444e6788304f..4f50af23a0854f18 100644
> > --- a/arch/x86/boot/compressed/pgtable_64.c
> > +++ b/arch/x86/boot/compressed/pgtable_64.c
> > @@ -101,9 +101,10 @@ static unsigned long find_trampoline_placement(void)
> > return bios_start - TRAMPOLINE_32BIT_SIZE;
> > }
> >
> > -asmlinkage void set_paging_levels(void *rmode)
> > +asmlinkage void set_paging_levels(void *rmode, void *pgtable)
>
> Please get rid of this silly rmode arg which gets passed in here as
> boot_params and then gets assigned to an extern pointer to boot_params.
> This is just silly. Just do what the other functions get from %r15 now
> - a struct boot_params *bp arg.
>
> But perhaps in a separate patch.
>
OK
> > {
> > void (*toggle_la57)(void *trampoline, bool enable_5lvl);
> > + void *trampoline_pgtable;
> > bool l5_required = false;
> >
> > /* Initialize boot_params. Required for cmdline_find_option_bool(). */
> > @@ -133,7 +134,7 @@ asmlinkage void set_paging_levels(void *rmode)
> > * the desired one.
> > */
> > if (l5_required == !!(native_read_cr4() & X86_CR4_LA57))
> > - return;
> > + goto out;
> >
> > trampoline_32bit = (unsigned long *)find_trampoline_placement();
> >
> > @@ -163,6 +164,8 @@ asmlinkage void set_paging_levels(void *rmode)
> > * The new page table will be used by trampoline code for switching
> > * from 4- to 5-level paging or vice versa.
> > */
> > + trampoline_pgtable = trampoline_32bit +
> > + TRAMPOLINE_32BIT_PGTABLE_OFFSET / sizeof(unsigned long);
> >
> > if (l5_required) {
> > /*
> > @@ -182,31 +185,21 @@ asmlinkage void set_paging_levels(void *rmode)
> > * may be above 4G.
> > */
> > src = *(unsigned long *)__native_read_cr3() & PAGE_MASK;
> > - memcpy(trampoline_32bit + TRAMPOLINE_32BIT_PGTABLE_OFFSET / sizeof(unsigned long),
> > - (void *)src, PAGE_SIZE);
> > + memcpy(trampoline_pgtable, (void *)src, PAGE_SIZE);
> > }
> >
> > toggle_la57(trampoline_32bit, l5_required);
> > -}
> > -
> > -void cleanup_trampoline(void *pgtable)
> > -{
> > - void *trampoline_pgtable;
> > -
> > - trampoline_pgtable = trampoline_32bit + TRAMPOLINE_32BIT_PGTABLE_OFFSET / sizeof(unsigned long);
> >
> > /*
> > - * Move the top level page table out of trampoline memory,
> > - * if it's there.
> > + * Move the top level page table out of trampoline memory.
> > */
> > - if ((void *)__native_read_cr3() == trampoline_pgtable) {
> > - memcpy(pgtable, trampoline_pgtable, PAGE_SIZE);
> > - native_write_cr3((unsigned long)pgtable);
> > - }
> > + memcpy(pgtable, trampoline_pgtable, PAGE_SIZE);
> > + native_write_cr3((unsigned long)pgtable);
> >
> > /* Restore trampoline memory */
> > memcpy(trampoline_32bit, trampoline_save, TRAMPOLINE_32BIT_SIZE);
> >
> > +out:
> > /* Initialize variables for 5-level paging */
> > #ifdef CONFIG_X86_5LEVEL
> > if (__read_cr4() & X86_CR4_LA57) {
> > @@ -215,4 +208,5 @@ void cleanup_trampoline(void *pgtable)
> > ptrs_per_p4d = 512;
> > }
> > #endif
> > + return;
>
> No need for that one. It'll return without it. :-)
>
Removing it breaks the build for !CONFIG_X86_5LEVEL
However, I can move these assignments into the conditional at the top
of the function (the one that sets l5_required) so we don't need the
label at all. Will fix for v8
On Fri, Jul 28, 2023 at 11:09:08AM +0200, Ard Biesheuvel wrote:
> diff --git a/arch/x86/boot/compressed/pgtable_64.c b/arch/x86/boot/compressed/pgtable_64.c
> index 4016444e6788304f..4f50af23a0854f18 100644
> --- a/arch/x86/boot/compressed/pgtable_64.c
> +++ b/arch/x86/boot/compressed/pgtable_64.c
> @@ -101,9 +101,10 @@ static unsigned long find_trampoline_placement(void)
> return bios_start - TRAMPOLINE_32BIT_SIZE;
> }
>
> -asmlinkage void set_paging_levels(void *rmode)
> +asmlinkage void set_paging_levels(void *rmode, void *pgtable)
Please get rid of this silly rmode arg which gets passed in here as
boot_params and then gets assigned to an extern pointer to boot_params.
This is just silly. Just do what the other functions get from %r15 now
- a struct boot_params *bp arg.
But perhaps in a separate patch.
> {
> void (*toggle_la57)(void *trampoline, bool enable_5lvl);
> + void *trampoline_pgtable;
> bool l5_required = false;
>
> /* Initialize boot_params. Required for cmdline_find_option_bool(). */
> @@ -133,7 +134,7 @@ asmlinkage void set_paging_levels(void *rmode)
> * the desired one.
> */
> if (l5_required == !!(native_read_cr4() & X86_CR4_LA57))
> - return;
> + goto out;
>
> trampoline_32bit = (unsigned long *)find_trampoline_placement();
>
> @@ -163,6 +164,8 @@ asmlinkage void set_paging_levels(void *rmode)
> * The new page table will be used by trampoline code for switching
> * from 4- to 5-level paging or vice versa.
> */
> + trampoline_pgtable = trampoline_32bit +
> + TRAMPOLINE_32BIT_PGTABLE_OFFSET / sizeof(unsigned long);
>
> if (l5_required) {
> /*
> @@ -182,31 +185,21 @@ asmlinkage void set_paging_levels(void *rmode)
> * may be above 4G.
> */
> src = *(unsigned long *)__native_read_cr3() & PAGE_MASK;
> - memcpy(trampoline_32bit + TRAMPOLINE_32BIT_PGTABLE_OFFSET / sizeof(unsigned long),
> - (void *)src, PAGE_SIZE);
> + memcpy(trampoline_pgtable, (void *)src, PAGE_SIZE);
> }
>
> toggle_la57(trampoline_32bit, l5_required);
> -}
> -
> -void cleanup_trampoline(void *pgtable)
> -{
> - void *trampoline_pgtable;
> -
> - trampoline_pgtable = trampoline_32bit + TRAMPOLINE_32BIT_PGTABLE_OFFSET / sizeof(unsigned long);
>
> /*
> - * Move the top level page table out of trampoline memory,
> - * if it's there.
> + * Move the top level page table out of trampoline memory.
> */
> - if ((void *)__native_read_cr3() == trampoline_pgtable) {
> - memcpy(pgtable, trampoline_pgtable, PAGE_SIZE);
> - native_write_cr3((unsigned long)pgtable);
> - }
> + memcpy(pgtable, trampoline_pgtable, PAGE_SIZE);
> + native_write_cr3((unsigned long)pgtable);
>
> /* Restore trampoline memory */
> memcpy(trampoline_32bit, trampoline_save, TRAMPOLINE_32BIT_SIZE);
>
> +out:
> /* Initialize variables for 5-level paging */
> #ifdef CONFIG_X86_5LEVEL
> if (__read_cr4() & X86_CR4_LA57) {
> @@ -215,4 +208,5 @@ void cleanup_trampoline(void *pgtable)
> ptrs_per_p4d = 512;
> }
> #endif
> + return;
No need for that one. It'll return without it. :-)
--
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquette
On Tue, 1 Aug 2023 at 14:41, Borislav Petkov <[email protected]> wrote:
>
> On Tue, Aug 01, 2023 at 02:11:21PM +0200, Ard Biesheuvel wrote:
> > Removing it breaks the build for !CONFIG_X86_5LEVEL
>
> How come?
>
Because a label cannot be at the end of a block. There needs to be at
least one statement.
> $ grep 5LEVEL .config
> # CONFIG_X86_5LEVEL is not set
>
> and it builds fine here.
>
I guess it depends on the compiler version. GCC 10 gives me this
<source>: In function 'foo':
<source>:4:5: error: label at end of compound statement
4 | bar:
| ^~~
but GCC 13 seems to be fine with it.
https://godbolt.org/z/KnWbY8vEY
On Tue, Aug 01, 2023 at 02:11:21PM +0200, Ard Biesheuvel wrote:
> Removing it breaks the build for !CONFIG_X86_5LEVEL
How come?
$ grep 5LEVEL .config
# CONFIG_X86_5LEVEL is not set
and it builds fine here.
--
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquette
On Tue, Aug 01, 2023 at 02:46:16PM +0200, Ard Biesheuvel wrote:
> but GCC 13 seems to be fine with it.
Ah, ok, yeah, it bombs here too with gcc-10.
So either change the assignments, please, or explain in a comment above
it why exactly is this needed.
Thx.
--
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquette