cpu_tlbstate is exported because various TLB related functions need access
to it, but cpu_tlbstate is sensitive information which should only be
accessed by well contained kernel functions and not be directly exposed to
modules.
In preparation of unexporting cpu_tlbstate move __get_current_cr3_fast()
into the x86 TLB management code.
No functional change.
Signed-off-by: Thomas Gleixner <[email protected]>
Reviewed-by: Alexandre Chartre <[email protected]>
Acked-by: Peter Zijlstra (Intel) <[email protected]>
---
arch/x86/include/asm/mmu_context.h | 19 +------------------
arch/x86/mm/tlb.c | 20 ++++++++++++++++++++
2 files changed, 21 insertions(+), 18 deletions(-)
--- a/arch/x86/include/asm/mmu_context.h
+++ b/arch/x86/include/asm/mmu_context.h
@@ -225,24 +225,7 @@ static inline bool arch_vma_access_permi
return __pkru_allows_pkey(vma_pkey(vma), write);
}
-/*
- * This can be used from process context to figure out what the value of
- * CR3 is without needing to do a (slow) __read_cr3().
- *
- * It's intended to be used for code like KVM that sneakily changes CR3
- * and needs to restore it. It needs to be used very carefully.
- */
-static inline unsigned long __get_current_cr3_fast(void)
-{
- unsigned long cr3 = build_cr3(this_cpu_read(cpu_tlbstate.loaded_mm)->pgd,
- this_cpu_read(cpu_tlbstate.loaded_mm_asid));
-
- /* For now, be very restrictive about when this can be called. */
- VM_WARN_ON(in_nmi() || preemptible());
-
- VM_BUG_ON(cr3 != __read_cr3());
- return cr3;
-}
+unsigned long __get_current_cr3_fast(void);
typedef struct {
struct mm_struct *mm;
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -844,6 +844,26 @@ void flush_tlb_kernel_range(unsigned lon
}
/*
+ * This can be used from process context to figure out what the value of
+ * CR3 is without needing to do a (slow) __read_cr3().
+ *
+ * It's intended to be used for code like KVM that sneakily changes CR3
+ * and needs to restore it. It needs to be used very carefully.
+ */
+unsigned long __get_current_cr3_fast(void)
+{
+ unsigned long cr3 = build_cr3(this_cpu_read(cpu_tlbstate.loaded_mm)->pgd,
+ this_cpu_read(cpu_tlbstate.loaded_mm_asid));
+
+ /* For now, be very restrictive about when this can be called. */
+ VM_WARN_ON(in_nmi() || preemptible());
+
+ VM_BUG_ON(cr3 != __read_cr3());
+ return cr3;
+}
+EXPORT_SYMBOL_GPL(__get_current_cr3_fast);
+
+/*
* arch_tlbbatch_flush() performs a full TLB flush regardless of the active mm.
* This means that the 'struct flush_tlb_info' that describes which mappings to
* flush is actually fixed. We therefore set a single fixed struct and use it in
On Tue, Apr 21, 2020 at 2:26 AM Thomas Gleixner <[email protected]> wrote:
>
> cpu_tlbstate is exported because various TLB related functions need access
> to it, but cpu_tlbstate is sensitive information which should only be
> accessed by well contained kernel functions and not be directly exposed to
> modules.
>
> In preparation of unexporting cpu_tlbstate move __get_current_cr3_fast()
> into the x86 TLB management code.
>
> No functional change.
>
> Signed-off-by: Thomas Gleixner <[email protected]>
> Reviewed-by: Alexandre Chartre <[email protected]>
> Acked-by: Peter Zijlstra (Intel) <[email protected]>
Acked-by: Andy Lutomirski <[email protected]>
> ---
> arch/x86/include/asm/mmu_context.h | 19 +------------------
> arch/x86/mm/tlb.c | 20 ++++++++++++++++++++
> 2 files changed, 21 insertions(+), 18 deletions(-)
>
> --- a/arch/x86/include/asm/mmu_context.h
> +++ b/arch/x86/include/asm/mmu_context.h
> @@ -225,24 +225,7 @@ static inline bool arch_vma_access_permi
> return __pkru_allows_pkey(vma_pkey(vma), write);
> }
>
> -/*
> - * This can be used from process context to figure out what the value of
> - * CR3 is without needing to do a (slow) __read_cr3().
> - *
> - * It's intended to be used for code like KVM that sneakily changes CR3
> - * and needs to restore it. It needs to be used very carefully.
> - */
> -static inline unsigned long __get_current_cr3_fast(void)
> -{
> - unsigned long cr3 = build_cr3(this_cpu_read(cpu_tlbstate.loaded_mm)->pgd,
> - this_cpu_read(cpu_tlbstate.loaded_mm_asid));
> -
> - /* For now, be very restrictive about when this can be called. */
> - VM_WARN_ON(in_nmi() || preemptible());
> -
> - VM_BUG_ON(cr3 != __read_cr3());
> - return cr3;
> -}
> +unsigned long __get_current_cr3_fast(void);
>
> typedef struct {
> struct mm_struct *mm;
> --- a/arch/x86/mm/tlb.c
> +++ b/arch/x86/mm/tlb.c
> @@ -844,6 +844,26 @@ void flush_tlb_kernel_range(unsigned lon
> }
>
> /*
> + * This can be used from process context to figure out what the value of
> + * CR3 is without needing to do a (slow) __read_cr3().
> + *
> + * It's intended to be used for code like KVM that sneakily changes CR3
> + * and needs to restore it. It needs to be used very carefully.
> + */
> +unsigned long __get_current_cr3_fast(void)
> +{
> + unsigned long cr3 = build_cr3(this_cpu_read(cpu_tlbstate.loaded_mm)->pgd,
> + this_cpu_read(cpu_tlbstate.loaded_mm_asid));
> +
> + /* For now, be very restrictive about when this can be called. */
> + VM_WARN_ON(in_nmi() || preemptible());
> +
> + VM_BUG_ON(cr3 != __read_cr3());
> + return cr3;
> +}
> +EXPORT_SYMBOL_GPL(__get_current_cr3_fast);
> +
> +/*
> * arch_tlbbatch_flush() performs a full TLB flush regardless of the active mm.
> * This means that the 'struct flush_tlb_info' that describes which mappings to
> * flush is actually fixed. We therefore set a single fixed struct and use it in
>
--
Andy Lutomirski
AMA Capital Management, LLC
The following commit has been merged into the x86/mm branch of tip:
Commit-ID: 8c5cc19e94703182647dfccc164e4437a04539c8
Gitweb: https://git.kernel.org/tip/8c5cc19e94703182647dfccc164e4437a04539c8
Author: Thomas Gleixner <[email protected]>
AuthorDate: Tue, 21 Apr 2020 11:20:28 +02:00
Committer: Borislav Petkov <[email protected]>
CommitterDate: Fri, 24 Apr 2020 15:55:50 +02:00
x86/tlb: Uninline __get_current_cr3_fast()
cpu_tlbstate is exported because various TLB-related functions need
access to it, but cpu_tlbstate is sensitive information which should
only be accessed by well-contained kernel functions and not be directly
exposed to modules.
In preparation for unexporting cpu_tlbstate move __get_current_cr3_fast()
into the x86 TLB management code.
No functional change.
Signed-off-by: Thomas Gleixner <[email protected]>
Signed-off-by: Borislav Petkov <[email protected]>
Reviewed-by: Alexandre Chartre <[email protected]>
Acked-by: Peter Zijlstra (Intel) <[email protected]>
Acked-by: Andy Lutomirski <[email protected]>
Link: https://lkml.kernel.org/r/[email protected]
---
arch/x86/include/asm/mmu_context.h | 19 +------------------
arch/x86/mm/tlb.c | 20 ++++++++++++++++++++
2 files changed, 21 insertions(+), 18 deletions(-)
diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h
index 4e55370..9608536 100644
--- a/arch/x86/include/asm/mmu_context.h
+++ b/arch/x86/include/asm/mmu_context.h
@@ -225,24 +225,7 @@ static inline bool arch_vma_access_permitted(struct vm_area_struct *vma,
return __pkru_allows_pkey(vma_pkey(vma), write);
}
-/*
- * This can be used from process context to figure out what the value of
- * CR3 is without needing to do a (slow) __read_cr3().
- *
- * It's intended to be used for code like KVM that sneakily changes CR3
- * and needs to restore it. It needs to be used very carefully.
- */
-static inline unsigned long __get_current_cr3_fast(void)
-{
- unsigned long cr3 = build_cr3(this_cpu_read(cpu_tlbstate.loaded_mm)->pgd,
- this_cpu_read(cpu_tlbstate.loaded_mm_asid));
-
- /* For now, be very restrictive about when this can be called. */
- VM_WARN_ON(in_nmi() || preemptible());
-
- VM_BUG_ON(cr3 != __read_cr3());
- return cr3;
-}
+unsigned long __get_current_cr3_fast(void);
typedef struct {
struct mm_struct *mm;
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 66f96f2..ea6f98a 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -844,6 +844,26 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end)
}
/*
+ * This can be used from process context to figure out what the value of
+ * CR3 is without needing to do a (slow) __read_cr3().
+ *
+ * It's intended to be used for code like KVM that sneakily changes CR3
+ * and needs to restore it. It needs to be used very carefully.
+ */
+unsigned long __get_current_cr3_fast(void)
+{
+ unsigned long cr3 = build_cr3(this_cpu_read(cpu_tlbstate.loaded_mm)->pgd,
+ this_cpu_read(cpu_tlbstate.loaded_mm_asid));
+
+ /* For now, be very restrictive about when this can be called. */
+ VM_WARN_ON(in_nmi() || preemptible());
+
+ VM_BUG_ON(cr3 != __read_cr3());
+ return cr3;
+}
+EXPORT_SYMBOL_GPL(__get_current_cr3_fast);
+
+/*
* arch_tlbbatch_flush() performs a full TLB flush regardless of the active mm.
* This means that the 'struct flush_tlb_info' that describes which mappings to
* flush is actually fixed. We therefore set a single fixed struct and use it in