Apologies if this is a duplicate mail, it will be the last one. Moto's SMTP
server sucks!!
Ensures that PXNTable can be set on all table descriptors allocated
through vmalloc. Normally, PXNTable is set only during initial memory
mapping and does not apply thereafter, making it possible for attackers
to target post-init allocated writable PTEs as a staging region for
injection of their code into the kernel. Presently it is not possible to
efficiently prevent these attacks as VMALLOC_END overlaps with _text,
e.g.:
VMALLOC_START ffff800080000000 VMALLOC_END fffffbfff0000000
_text ffffb6c0c1400000 _end ffffb6c0c3e40000
Setting VMALLOC_END to _text in init would resolve this issue with the
caveat of a sizeable reduction in the size of available vmalloc memory
due to requirements on aslr randomness. However, there are circumstances
where this trade-off is necessary: in particular, hypervisor-level
security monitors where 1) the microarchitecture contains race
conditions on PTE level updates or 2) a per-PTE update verifier comes at
a significant hit to performance.
Because the address of _text is aslr-sensitive and this patch associates
this value with VMALLOC_END, we remove the use of VMALLOC_END in a print
statement in mm/percpu.c. However, only the format string is updated in
crash_core.c, since we are dead at that point regardless. VMALLOC_END is
updated in kernel/setup.c to associate the feature closely with aslr and
region allocation code.
Signed-off-by: Maxwell Bland <[email protected]>
---
arch/arm64/Kconfig | 13 +++++++++++++
arch/arm64/include/asm/pgtable.h | 6 ++++++
arch/arm64/include/asm/vmalloc-pxn.h | 10 ++++++++++
arch/arm64/kernel/crash_core.c | 2 +-
arch/arm64/kernel/setup.c | 9 +++++++++
mm/percpu.c | 4 ++--
6 files changed, 41 insertions(+), 3 deletions(-)
create mode 100644 arch/arm64/include/asm/vmalloc-pxn.h
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index aa7c1d435139..5f1e75d70e14 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -2165,6 +2165,19 @@ config ARM64_DEBUG_PRIORITY_MASKING
If unsure, say N
endif # ARM64_PSEUDO_NMI
+config ARM64_VMALLOC_PXN
+ bool "Ensures table descriptors pointing to kernel data are PXNTable"
+ help
+ Reduces the range of the kernel data vmalloc region to remove any
+ overlap with kernel code, making it possible to enable the PXNTable
+ bit on table descriptors allocated after the kernel's initial memory
+ mapping.
+
+ This increases the performance of security monitors which protect
+ against malicious updates to page table entries.
+
+ If unsure, say N.
+
config RELOCATABLE
bool "Build a relocatable kernel image" if EXPERT
select ARCH_HAS_RELR
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 79ce70fbb751..49f64ea77c81 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -22,7 +22,9 @@
* and fixed mappings
*/
#define VMALLOC_START (MODULES_END)
+#ifndef CONFIG_ARM64_VMALLOC_PXN
#define VMALLOC_END (VMEMMAP_START - SZ_256M)
+#endif
#define vmemmap ((struct page *)VMEMMAP_START - (memstart_addr >> PAGE_SHIFT))
@@ -35,6 +37,10 @@
#include <linux/sched.h>
#include <linux/page_table_check.h>
+#ifdef CONFIG_ARM64_VMALLOC_PXN
+#include <asm/vmalloc-pxn.h>
+#endif
+
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
#define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
diff --git a/arch/arm64/include/asm/vmalloc-pxn.h
b/arch/arm64/include/asm/vmalloc-pxn.h
new file mode 100644
index 000000000000..c8c4f878eb62
--- /dev/null
+++ b/arch/arm64/include/asm/vmalloc-pxn.h
@@ -0,0 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_ARM64_VMALLOC_PXN_H
+#define _ASM_ARM64_VMALLOC_PXN_H
+
+#ifdef CONFIG_ARM64_VMALLOC_PXN
+extern u64 __vmalloc_end __ro_after_init;
+#define VMALLOC_END (__vmalloc_end)
+#endif /* CONFIG_ARM64_VMALLOC_PXN */
+
+#endif /* _ASM_ARM64_VMALLOC_PXN_H */
diff --git a/arch/arm64/kernel/crash_core.c b/arch/arm64/kernel/crash_core.c
index 66cde752cd74..39dccae11a40 100644
--- a/arch/arm64/kernel/crash_core.c
+++ b/arch/arm64/kernel/crash_core.c
@@ -24,7 +24,7 @@ void arch_crash_save_vmcoreinfo(void)
vmcoreinfo_append_str("NUMBER(MODULES_VADDR)=0x%lx\n", MODULES_VADDR);
vmcoreinfo_append_str("NUMBER(MODULES_END)=0x%lx\n", MODULES_END);
vmcoreinfo_append_str("NUMBER(VMALLOC_START)=0x%lx\n", VMALLOC_START);
- vmcoreinfo_append_str("NUMBER(VMALLOC_END)=0x%lx\n", VMALLOC_END);
+ vmcoreinfo_append_str("NUMBER(VMALLOC_END)=0x%llx\n", VMALLOC_END);
vmcoreinfo_append_str("NUMBER(VMEMMAP_START)=0x%lx\n", VMEMMAP_START);
vmcoreinfo_append_str("NUMBER(VMEMMAP_END)=0x%lx\n", VMEMMAP_END);
vmcoreinfo_append_str("NUMBER(kimage_voffset)=0x%llx\n",
diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index 42c690bb2d60..b7ccee672743 100644
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c
@@ -54,6 +54,11 @@
#include <asm/xen/hypervisor.h>
#include <asm/mmu_context.h>
+#ifdef CONFIG_ARM64_VMALLOC_PXN
+u64 __vmalloc_end __ro_after_init = VMEMMAP_START - SZ_256M;
+EXPORT_SYMBOL(__vmalloc_end);
+#endif /* CONFIG_ARM64_VMALLOC_PXN */
+
static int num_standard_resources;
static struct resource *standard_resources;
@@ -298,6 +303,10 @@ void __init __no_sanitize_address setup_arch(char
**cmdline_p)
kaslr_init();
+#ifdef CONFIG_ARM64_VMALLOC_PXN
+ __vmalloc_end = ALIGN_DOWN((u64) _text, PMD_SIZE);
+#endif
+
/*
* If know now we are going to need KPTI then use non-global
* mappings from the start, avoiding the cost of rewriting
diff --git a/mm/percpu.c b/mm/percpu.c
index 4e11fc1e6def..a902500ebfa0 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -3128,8 +3128,8 @@ int __init pcpu_embed_first_chunk(size_t
reserved_size, size_t dyn_size,
/* warn if maximum distance is further than 75% of vmalloc space */
if (max_distance > VMALLOC_TOTAL * 3 / 4) {
- pr_warn("max_distance=0x%lx too large for vmalloc space 0x%lx\n",
- max_distance, VMALLOC_TOTAL);
+ pr_warn("max_distance=0x%lx too large for vmalloc space\n",
+ max_distance);
#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
/* and fail if we have fallback */
rc = -EINVAL;
base-commit: 716f4aaa7b48a55c73d632d0657b35342b1fefd7
--
2.39.0
On Tue, Feb 13, 2024 at 10:05:45AM -0600, Maxwell Bland wrote:
> Apologies if this is a duplicate mail, it will be the last one. Moto's SMTP
> server sucks!!
This shouldn't be in the changelog, it needs to go below the --- line.
Also, your patch is corrupted and can not be applied :(
thanks,
greg k-h
> This shouldn't be in the changelog, it needs to go below the --- line.
Oh! Thanks!!!
> Also, your patch is corrupted and can not be applied :(
Shoot! Apologies, I just noticed the hard-wrap at 80. I am talking to Moto's IT right now, I will resend patch once I can fix the mail server config.
Maxwell
On Tue, Feb 13, 2024 at 10:05:45AM -0600, Maxwell Bland wrote:
> Apologies if this is a duplicate mail, it will be the last one. Moto's SMTP
> server sucks!!
>
> Ensures that PXNTable can be set on all table descriptors allocated
> through vmalloc. Normally, PXNTable is set only during initial memory
> mapping and does not apply thereafter, making it possible for attackers
> to target post-init allocated writable PTEs as a staging region for
> injection of their code into the kernel. Presently it is not possible to
> efficiently prevent these attacks as VMALLOC_END overlaps with _text,
> e.g.:
>
> VMALLOC_START ffff800080000000 VMALLOC_END fffffbfff0000000
> _text ffffb6c0c1400000 _end ffffb6c0c3e40000
>
> Setting VMALLOC_END to _text in init would resolve this issue with the
> caveat of a sizeable reduction in the size of available vmalloc memory
> due to requirements on aslr randomness. However, there are circumstances
> where this trade-off is necessary: in particular, hypervisor-level
> security monitors where 1) the microarchitecture contains race
> conditions on PTE level updates or 2) a per-PTE update verifier comes at
> a significant hit to performance.
Which "hypervisor-level security monitors" are you referring to? We don't
support any of those upstream AFAIK.
How much VA space are you potentially throwing away?
How does this work with other allocations of executable memory? e.g. modules,
BPF?
I'm not keen on this as-is.
Mark.
> Because the address of _text is aslr-sensitive and this patch associates
> this value with VMALLOC_END, we remove the use of VMALLOC_END in a print
> statement in mm/percpu.c. However, only the format string is updated in
> crash_core.c, since we are dead at that point regardless. VMALLOC_END is
> updated in kernel/setup.c to associate the feature closely with aslr and
> region allocation code.
>
> Signed-off-by: Maxwell Bland <[email protected]>
> ---
> arch/arm64/Kconfig | 13 +++++++++++++
> arch/arm64/include/asm/pgtable.h | 6 ++++++
> arch/arm64/include/asm/vmalloc-pxn.h | 10 ++++++++++
> arch/arm64/kernel/crash_core.c | 2 +-
> arch/arm64/kernel/setup.c | 9 +++++++++
> mm/percpu.c | 4 ++--
> 6 files changed, 41 insertions(+), 3 deletions(-)
> create mode 100644 arch/arm64/include/asm/vmalloc-pxn.h
>
> diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
> index aa7c1d435139..5f1e75d70e14 100644
> --- a/arch/arm64/Kconfig
> +++ b/arch/arm64/Kconfig
> @@ -2165,6 +2165,19 @@ config ARM64_DEBUG_PRIORITY_MASKING
> If unsure, say N
> endif # ARM64_PSEUDO_NMI
>
> +config ARM64_VMALLOC_PXN
> + bool "Ensures table descriptors pointing to kernel data are PXNTable"
> + help
> + Reduces the range of the kernel data vmalloc region to remove any
> + overlap with kernel code, making it possible to enable the PXNTable
> + bit on table descriptors allocated after the kernel's initial memory
> + mapping.
> +
> + This increases the performance of security monitors which protect
> + against malicious updates to page table entries.
> +
> + If unsure, say N.
> +
> config RELOCATABLE
> bool "Build a relocatable kernel image" if EXPERT
> select ARCH_HAS_RELR
> diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
> index 79ce70fbb751..49f64ea77c81 100644
> --- a/arch/arm64/include/asm/pgtable.h
> +++ b/arch/arm64/include/asm/pgtable.h
> @@ -22,7 +22,9 @@
> * and fixed mappings
> */
> #define VMALLOC_START (MODULES_END)
> +#ifndef CONFIG_ARM64_VMALLOC_PXN
> #define VMALLOC_END (VMEMMAP_START - SZ_256M)
> +#endif
>
> #define vmemmap ((struct page *)VMEMMAP_START - (memstart_addr >> PAGE_SHIFT))
>
> @@ -35,6 +37,10 @@
> #include <linux/sched.h>
> #include <linux/page_table_check.h>
>
> +#ifdef CONFIG_ARM64_VMALLOC_PXN
> +#include <asm/vmalloc-pxn.h>
> +#endif
> +
> #ifdef CONFIG_TRANSPARENT_HUGEPAGE
> #define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
>
> diff --git a/arch/arm64/include/asm/vmalloc-pxn.h
> b/arch/arm64/include/asm/vmalloc-pxn.h
> new file mode 100644
> index 000000000000..c8c4f878eb62
> --- /dev/null
> +++ b/arch/arm64/include/asm/vmalloc-pxn.h
> @@ -0,0 +1,10 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#ifndef _ASM_ARM64_VMALLOC_PXN_H
> +#define _ASM_ARM64_VMALLOC_PXN_H
> +
> +#ifdef CONFIG_ARM64_VMALLOC_PXN
> +extern u64 __vmalloc_end __ro_after_init;
> +#define VMALLOC_END (__vmalloc_end)
> +#endif /* CONFIG_ARM64_VMALLOC_PXN */
> +
> +#endif /* _ASM_ARM64_VMALLOC_PXN_H */
> diff --git a/arch/arm64/kernel/crash_core.c b/arch/arm64/kernel/crash_core.c
> index 66cde752cd74..39dccae11a40 100644
> --- a/arch/arm64/kernel/crash_core.c
> +++ b/arch/arm64/kernel/crash_core.c
> @@ -24,7 +24,7 @@ void arch_crash_save_vmcoreinfo(void)
> vmcoreinfo_append_str("NUMBER(MODULES_VADDR)=0x%lx\n", MODULES_VADDR);
> vmcoreinfo_append_str("NUMBER(MODULES_END)=0x%lx\n", MODULES_END);
> vmcoreinfo_append_str("NUMBER(VMALLOC_START)=0x%lx\n", VMALLOC_START);
> - vmcoreinfo_append_str("NUMBER(VMALLOC_END)=0x%lx\n", VMALLOC_END);
> + vmcoreinfo_append_str("NUMBER(VMALLOC_END)=0x%llx\n", VMALLOC_END);
> vmcoreinfo_append_str("NUMBER(VMEMMAP_START)=0x%lx\n", VMEMMAP_START);
> vmcoreinfo_append_str("NUMBER(VMEMMAP_END)=0x%lx\n", VMEMMAP_END);
> vmcoreinfo_append_str("NUMBER(kimage_voffset)=0x%llx\n",
> diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
> index 42c690bb2d60..b7ccee672743 100644
> --- a/arch/arm64/kernel/setup.c
> +++ b/arch/arm64/kernel/setup.c
> @@ -54,6 +54,11 @@
> #include <asm/xen/hypervisor.h>
> #include <asm/mmu_context.h>
>
> +#ifdef CONFIG_ARM64_VMALLOC_PXN
> +u64 __vmalloc_end __ro_after_init = VMEMMAP_START - SZ_256M;
> +EXPORT_SYMBOL(__vmalloc_end);
> +#endif /* CONFIG_ARM64_VMALLOC_PXN */
> +
> static int num_standard_resources;
> static struct resource *standard_resources;
>
> @@ -298,6 +303,10 @@ void __init __no_sanitize_address setup_arch(char
> **cmdline_p)
>
> kaslr_init();
>
> +#ifdef CONFIG_ARM64_VMALLOC_PXN
> + __vmalloc_end = ALIGN_DOWN((u64) _text, PMD_SIZE);
> +#endif
> +
> /*
> * If know now we are going to need KPTI then use non-global
> * mappings from the start, avoiding the cost of rewriting
> diff --git a/mm/percpu.c b/mm/percpu.c
> index 4e11fc1e6def..a902500ebfa0 100644
> --- a/mm/percpu.c
> +++ b/mm/percpu.c
> @@ -3128,8 +3128,8 @@ int __init pcpu_embed_first_chunk(size_t
> reserved_size, size_t dyn_size,
>
> /* warn if maximum distance is further than 75% of vmalloc space */
> if (max_distance > VMALLOC_TOTAL * 3 / 4) {
> - pr_warn("max_distance=0x%lx too large for vmalloc space 0x%lx\n",
> - max_distance, VMALLOC_TOTAL);
> + pr_warn("max_distance=0x%lx too large for vmalloc space\n",
> + max_distance);
> #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
> /* and fail if we have fallback */
> rc = -EINVAL;
>
> base-commit: 716f4aaa7b48a55c73d632d0657b35342b1fefd7
> --
> 2.39.0
On Tue, Feb 13, 2024 at 05:16:04PM +0000, Maxwell Bland wrote:
> > This shouldn't be in the changelog, it needs to go below the --- line.
>
> Oh! Thanks!!!
>
> > Also, your patch is corrupted and can not be applied :(
>
> Shoot! Apologies, I just noticed the hard-wrap at 80. I am talking to Moto's IT right now, I will resend patch once I can fix the mail server config.
tabs were also eaten :(
> From: Mark Rutland <[email protected]> On Tue, Feb 13, 2024 at 10:05:45AM
> -0600, Maxwell Bland wrote:
> > VMALLOC_START ffff800080000000 VMALLOC_END fffffbfff0000000 _text
> > ffffb6c0c1400000 _end ffffb6c0c3e40000
> >
> > Setting VMALLOC_END to _text in init would resolve this issue with the
> > caveat of a sizeable reduction in the size of available vmalloc memory due
> > to requirements on aslr randomness. However, there are circumstances where
> > this trade-off is necessary: in particular, hypervisor-level security
> > monitors where 1) the microarchitecture contains race conditions on PTE
> > level updates or 2) a per-PTE update verifier comes at a significant hit to
> > performance.
>
> Which "hypervisor-level security monitors" are you referring to?
Right now there are around 4 or 5 different attempts (from what I know: Moto,
Samsung, MediaTek, and Qualcomm) at making page tables immutable and reducing
the kernel threat surface to just dynamically allocated structs, e.g.
file_operations, in ARM, a revival of some of the ideas of:
https://wenboshen.org/publications/papers/tz-rkp-ccs14.pdf
Which are no longer possible to enforce for a number of reasons. As related to
this patch in particular: the performance hits involved in per-PTE update
verification are huge.
My goal is ultimately to prevent modern exploits like:
https://github.com/chompie1337/s8_2019_2215_poc
which modify dynamically allocated pointers, but trying to protect against these
exploits is disingenuous without first being able to enforce PXN on non-code
pages, i.e. there is a reason we do this in mm initialization, but we need to
enforce or support the enforcement of PXNTable dynamically too.
> We don't support any of those upstream AFAIK.
As is hopefully apparent from the above, though it will help downstream systems,
I do not see this patch as a support issue so much as a legitimate security
feature. There is the matter of deciding which subsystem should be responsible.
The generic vmalloc interface should provide a strong distinction between code
and data allocations, but enforcing this would become the responsibility of each
microarchitecture regardless.
>
> How much VA space are you potentially throwing away?
>
This is rough, I admit. )-: On the order of 70,000 GB, likely more in practice:
it restricts vmalloc to the region before _text. You may be thinking, "that is
ridiculous, c'mon Maxwell", and you would be right, but I was OK with this
trade-off for Moto systems, and was thinking the approach keeps the patch
changes small and simple.
I had a hard time thinking of a better way to do this while avoiding duplication
of vmalloc code into arm64 land. Potentially, though, it would be OK to add an
additional field to the generic vmalloc interface? I may need to reach out for
help here: maybe the solution to the issue will come more readily to those with
more experience.
> How does this work with other allocations of executable memory? e.g. modules,
> BPF?
It should work.
- arch/arm64/kernel/module.c uses __vmalloc_node_range with module_alloc_base
and module_alloc_end, bypassing the generic vmalloc_node region, and these
variables are decided based on a random offset between _text and _end.
- kernel/bpf/core.c uses bpf_jit_alloc_exec to create executable code regions,
which is a wrapper for module_alloc. In the interpreted BPF case, we do not
need to worry since the pages storing interpreted code are NX and can be
marked PXNTable regardless.
> I'm not keen on this as-is.
That's OK, so long as we agree enforcing PXNTable dynamically would be a good
thing. I look forward to your thoughts on the above, and I will go back and
iterate.
Working with IT to fix the email formatting now, so I will hopefully be able to
post a fetchable and runnable version of my initial patch shortly.
> -----Original Message-----
> From: Maxwell Bland On Tuesday, February 13, 2024 1:15 PM
> > From: Mark Rutland <[email protected]>
> > How does this work with other allocations of executable memory? e.g. modules,
> > BPF?
>
> It should work.
> - kernel/bpf/core.c uses bpf_jit_alloc_exec to create executable code regions,
> which is a wrapper for module_alloc. In the interpreted BPF case, we do not
> need to worry since the pages storing interpreted code are NX and can be
> marked PXNTable regardless.
Correction: I was wrong here. The _weak reference to bpf_jit_alloc_exec is overwritten in
arch/arm64/net/bpf_jit_comp.c to use a vmalloc call. This would need to be set back to
the generic BPF's use of "module_alloc". I will look into and correct this.
Ensures that PXNTable can be set on all table descriptors allocated
through vmalloc. Normally, PXNTable is set only during initial memory
mapping and does not apply thereafter, making it possible for attackers
to target post-init allocated writable PTEs as a staging region for
injection of their code into the kernel. Presently it is not possible to
efficiently prevent these attacks as VMALLOC_END overlaps with _text,
e.g.:
VMALLOC_START ffff800080000000 VMALLOC_END fffffbfff0000000
_text ffffb6c0c1400000 _end ffffb6c0c3e40000
Setting VMALLOC_END to _text in init would resolve this issue with the
caveat of a sizeable reduction in the size of available vmalloc memory
(~70,000 GB) due to requirements on aslr randomness. However, we need to
support the enforcement of PXNTable dynamically for our static
assignment of this flag during mm initialization to be effective.
Because the address of _text is aslr-sensitive and this patch associates
this value with VMALLOC_END, we remove the use of VMALLOC_END in a print
statement in mm/percpu.c. However, only the format string is updated in
crash_core.c, since we are dead at that point regardless. VMALLOC_END is
updated in kernel/setup.c to associate the feature closely with aslr and
region allocation code.
bpf_jit_comp.c must also be remediated to ensure that the module_alloc
rather than vmalloc interface is used, so that regions used for BPF
allocations are appropriately located into the _text region.
Signed-off-by: Maxwell Bland <[email protected]>
---
This is an attempt to get Moto's SMTP server to send the patch without ruining
the formatting. Based on Mark R.'s comments, though, it sounds like:
1) I need to figure out a way to reduce the reduction in virtual memory.
2) I need to actually enforce PXNTable dynamically, to make it clear this is a
real upstream issue.
3) I need some testing and quantification to make sure this does not ruin BPF
and module allocations.
https://lore.kernel.org/all/[email protected]/
Regardless, here's the original patch on the current Github linux main.
arch/arm64/Kconfig | 13 +++++++++++++
arch/arm64/include/asm/pgtable.h | 6 ++++++
arch/arm64/include/asm/vmalloc-pxn.h | 9 +++++++++
arch/arm64/kernel/crash_core.c | 2 +-
arch/arm64/kernel/setup.c | 9 +++++++++
arch/arm64/net/bpf_jit_comp.c | 5 +++--
mm/percpu.c | 4 ++--
7 files changed, 43 insertions(+), 5 deletions(-)
create mode 100644 arch/arm64/include/asm/vmalloc-pxn.h
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index aa7c1d435139..5f1e75d70e14 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -2165,6 +2165,19 @@ config ARM64_DEBUG_PRIORITY_MASKING
If unsure, say N
endif # ARM64_PSEUDO_NMI
+config ARM64_VMALLOC_PXN
+ bool "Ensures table descriptors pointing to kernel data are PXNTable"
+ help
+ Reduces the range of the kernel data vmalloc region to remove any
+ overlap with kernel code, making it possible to enable the PXNTable
+ bit on table descriptors allocated after the kernel's initial memory
+ mapping.
+
+ This increases the performance of security monitors which protect
+ against malicious updates to page table entries.
+
+ If unsure, say N.
+
config RELOCATABLE
bool "Build a relocatable kernel image" if EXPERT
select ARCH_HAS_RELR
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 79ce70fbb751..49f64ea77c81 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -22,7 +22,9 @@
* and fixed mappings
*/
#define VMALLOC_START (MODULES_END)
+#ifndef CONFIG_ARM64_VMALLOC_PXN
#define VMALLOC_END (VMEMMAP_START - SZ_256M)
+#endif
#define vmemmap ((struct page *)VMEMMAP_START - (memstart_addr >> PAGE_SHIFT))
@@ -35,6 +37,10 @@
#include <linux/sched.h>
#include <linux/page_table_check.h>
+#ifdef CONFIG_ARM64_VMALLOC_PXN
+#include <asm/vmalloc-pxn.h>
+#endif
+
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
#define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
diff --git a/arch/arm64/include/asm/vmalloc-pxn.h b/arch/arm64/include/asm/vmalloc-pxn.h
new file mode 100644
index 000000000000..d054427e2804
--- /dev/null
+++ b/arch/arm64/include/asm/vmalloc-pxn.h
@@ -0,0 +1,9 @@
+#ifndef _ASM_ARM64_VMALLOC_PXN_H
+#define _ASM_ARM64_VMALLOC_PXN_H
+
+#ifdef CONFIG_ARM64_VMALLOC_PXN
+extern u64 __vmalloc_end __ro_after_init;
+#define VMALLOC_END (__vmalloc_end)
+#endif /* CONFIG_ARM64_VMALLOC_PXN */
+
+#endif /* _ASM_ARM64_VMALLOC_PXN_H */
diff --git a/arch/arm64/kernel/crash_core.c b/arch/arm64/kernel/crash_core.c
index 66cde752cd74..39dccae11a40 100644
--- a/arch/arm64/kernel/crash_core.c
+++ b/arch/arm64/kernel/crash_core.c
@@ -24,7 +24,7 @@ void arch_crash_save_vmcoreinfo(void)
vmcoreinfo_append_str("NUMBER(MODULES_VADDR)=0x%lx\n", MODULES_VADDR);
vmcoreinfo_append_str("NUMBER(MODULES_END)=0x%lx\n", MODULES_END);
vmcoreinfo_append_str("NUMBER(VMALLOC_START)=0x%lx\n", VMALLOC_START);
- vmcoreinfo_append_str("NUMBER(VMALLOC_END)=0x%lx\n", VMALLOC_END);
+ vmcoreinfo_append_str("NUMBER(VMALLOC_END)=0x%llx\n", VMALLOC_END);
vmcoreinfo_append_str("NUMBER(VMEMMAP_START)=0x%lx\n", VMEMMAP_START);
vmcoreinfo_append_str("NUMBER(VMEMMAP_END)=0x%lx\n", VMEMMAP_END);
vmcoreinfo_append_str("NUMBER(kimage_voffset)=0x%llx\n",
diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index 42c690bb2d60..b7ccee672743 100644
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c
@@ -54,6 +54,11 @@
#include <asm/xen/hypervisor.h>
#include <asm/mmu_context.h>
+#ifdef CONFIG_ARM64_VMALLOC_PXN
+u64 __vmalloc_end __ro_after_init = VMEMMAP_START - SZ_256M;
+EXPORT_SYMBOL(__vmalloc_end);
+#endif /* CONFIG_ARM64_VMALLOC_PXN */
+
static int num_standard_resources;
static struct resource *standard_resources;
@@ -298,6 +303,10 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p)
kaslr_init();
+#ifdef CONFIG_ARM64_VMALLOC_PXN
+ __vmalloc_end = ALIGN_DOWN((u64) _text, PMD_SIZE);
+#endif
+
/*
* If know now we are going to need KPTI then use non-global
* mappings from the start, avoiding the cost of rewriting
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index 8955da5c47cf..1fe0d637792c 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -11,6 +11,7 @@
#include <linux/bpf.h>
#include <linux/filter.h>
#include <linux/memory.h>
+#include <linux/moduleloader.h>
#include <linux/printk.h>
#include <linux/slab.h>
@@ -1690,12 +1691,12 @@ u64 bpf_jit_alloc_exec_limit(void)
void *bpf_jit_alloc_exec(unsigned long size)
{
/* Memory is intended to be executable, reset the pointer tag. */
- return kasan_reset_tag(vmalloc(size));
+ return kasan_reset_tag(module_alloc(size));
}
void bpf_jit_free_exec(void *addr)
{
- return vfree(addr);
+ return module_memfree(addr);
}
/* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */
diff --git a/mm/percpu.c b/mm/percpu.c
index 4e11fc1e6def..a902500ebfa0 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -3128,8 +3128,8 @@ int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size,
/* warn if maximum distance is further than 75% of vmalloc space */
if (max_distance > VMALLOC_TOTAL * 3 / 4) {
- pr_warn("max_distance=0x%lx too large for vmalloc space 0x%lx\n",
- max_distance, VMALLOC_TOTAL);
+ pr_warn("max_distance=0x%lx too large for vmalloc space\n",
+ max_distance);
#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
/* and fail if we have fallback */
rc = -EINVAL;
--
2.39.2