If FEAT_MTE2 is disabled via the arm64.nomte command line argument on a
CPU that claims to support FEAT_MTE2, the kernel will use Tagged Normal
in the MAIR. If we interpret arm64.nomte to mean that the CPU does not
in fact implement FEAT_MTE2, setting the system register like this may
lead to UNSPECIFIED behavior. Fix it by arranging for MAIR to be set
in the C function cpu_enable_mte which is called based on the sanitized
version of the system register.
There is no need for the rest of the MTE-related system register
initialization to happen from assembly, with the exception of TCR_EL1,
which must be set to include at least TBI1 because the secondary CPUs
access KASan-allocated data structures early. Therefore, make the TCR_EL1
initialization unconditional and move the rest of the initialization to
cpu_enable_mte so that we no longer have a dependency on the unsanitized
ID register value.
Signed-off-by: Peter Collingbourne <[email protected]>
Suggested-by: Catalin Marinas <[email protected]>
Link: https://linux-review.googlesource.com/id/I2c7df6bd4ea2dfc59376a8b9b5d3562b015c7198
---
arch/arm64/kernel/cpufeature.c | 39 +++++++++++++++++++++++++++++++
arch/arm64/mm/proc.S | 42 ++--------------------------------
2 files changed, 41 insertions(+), 40 deletions(-)
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 907401e4fffb..3554ff869f4b 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -2030,8 +2030,47 @@ static void bti_enable(const struct arm64_cpu_capabilities *__unused)
#ifdef CONFIG_ARM64_MTE
static void cpu_enable_mte(struct arm64_cpu_capabilities const *cap)
{
+ u64 rgsr;
+
sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_ATA | SCTLR_EL1_ATA0);
+
+ /*
+ * CnP must be enabled only after the MAIR_EL1 register has been set
+ * up. Inconsistent MAIR_EL1 between CPUs sharing the same TLB may
+ * lead to the wrong memory type being used for a brief window during
+ * CPU power-up.
+ *
+ * CnP is not a boot feature so MTE gets enabled before CnP, but let's
+ * make sure that is the case.
+ */
+ BUG_ON(read_sysreg(ttbr0_el1) & TTBR_CNP_BIT);
+ BUG_ON(read_sysreg(ttbr1_el1) & TTBR_CNP_BIT);
+
+ /* Normal Tagged memory type at the corresponding MAIR index */
+ sysreg_clear_set(
+ mair_el1, MAIR_ATTRIDX(MAIR_ATTR_MASK, MT_NORMAL_TAGGED),
+ MAIR_ATTRIDX(MAIR_ATTR_NORMAL_TAGGED, MT_NORMAL_TAGGED));
+
+ write_sysreg_s(KERNEL_GCR_EL1, SYS_GCR_EL1);
+
+ /*
+ * If GCR_EL1.RRND=1 is implemented the same way as RRND=0, then
+ * RGSR_EL1.SEED must be non-zero for IRG to produce
+ * pseudorandom numbers. As RGSR_EL1 is UNKNOWN out of reset, we
+ * must initialize it.
+ */
+ rgsr = (read_sysreg(CNTVCT_EL0) << SYS_RGSR_EL1_SEED_SHIFT) &
+ SYS_RGSR_EL1_SEED_MASK;
+ if (rgsr == 0)
+ rgsr = 1 << SYS_RGSR_EL1_SEED_SHIFT;
+ write_sysreg_s(rgsr, SYS_RGSR_EL1);
+
+ /* clear any pending tag check faults in TFSR*_EL1 */
+ write_sysreg_s(0, SYS_TFSR_EL1);
+ write_sysreg_s(0, SYS_TFSRE0_EL1);
+
isb();
+ local_flush_tlb_all();
/*
* Clear the tags in the zero page. This needs to be done via the
diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index 7837a69524c5..6f01f5a54cc2 100644
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -58,7 +58,7 @@
/*
* Default MAIR_EL1. MT_NORMAL_TAGGED is initially mapped as Normal memory and
- * changed during __cpu_setup to Normal Tagged if the system supports MTE.
+ * changed during cpu_enable_mte to Normal Tagged if the system supports MTE.
*/
#define MAIR_EL1_SET \
(MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRnE, MT_DEVICE_nGnRnE) | \
@@ -426,46 +426,8 @@ SYM_FUNC_START(__cpu_setup)
mov_q mair, MAIR_EL1_SET
mov_q tcr, TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \
TCR_TG_FLAGS | TCR_KASLR_FLAGS | TCR_ASID16 | \
- TCR_TBI0 | TCR_A1 | TCR_KASAN_SW_FLAGS
+ TCR_TBI0 | TCR_A1 | TCR_KASAN_SW_FLAGS | TCR_MTE_FLAGS
-#ifdef CONFIG_ARM64_MTE
- /*
- * Update MAIR_EL1, GCR_EL1 and TFSR*_EL1 if MTE is supported
- * (ID_AA64PFR1_EL1[11:8] > 1).
- */
- mrs x10, ID_AA64PFR1_EL1
- ubfx x10, x10, #ID_AA64PFR1_MTE_SHIFT, #4
- cmp x10, #ID_AA64PFR1_MTE
- b.lt 1f
-
- /* Normal Tagged memory type at the corresponding MAIR index */
- mov x10, #MAIR_ATTR_NORMAL_TAGGED
- bfi mair, x10, #(8 * MT_NORMAL_TAGGED), #8
-
- mov x10, #KERNEL_GCR_EL1
- msr_s SYS_GCR_EL1, x10
-
- /*
- * If GCR_EL1.RRND=1 is implemented the same way as RRND=0, then
- * RGSR_EL1.SEED must be non-zero for IRG to produce
- * pseudorandom numbers. As RGSR_EL1 is UNKNOWN out of reset, we
- * must initialize it.
- */
- mrs x10, CNTVCT_EL0
- ands x10, x10, #SYS_RGSR_EL1_SEED_MASK
- csinc x10, x10, xzr, ne
- lsl x10, x10, #SYS_RGSR_EL1_SEED_SHIFT
- msr_s SYS_RGSR_EL1, x10
-
- /* clear any pending tag check faults in TFSR*_EL1 */
- msr_s SYS_TFSR_EL1, xzr
- msr_s SYS_TFSRE0_EL1, xzr
-
- /* set the TCR_EL1 bits */
- mov_q x10, TCR_MTE_FLAGS
- orr tcr, tcr, x10
-1:
-#endif
tcr_clear_errata_bits tcr, x9, x5
#ifdef CONFIG_ARM64_VA_BITS_52
--
2.37.1.595.g718a3a8f04-goog
On Thu, Aug 18, 2022 at 06:35:26PM -0700, Peter Collingbourne wrote:
> If FEAT_MTE2 is disabled via the arm64.nomte command line argument on a
> CPU that claims to support FEAT_MTE2, the kernel will use Tagged Normal
> in the MAIR. If we interpret arm64.nomte to mean that the CPU does not
> in fact implement FEAT_MTE2, setting the system register like this may
> lead to UNSPECIFIED behavior.
I'm not convinced by this wording. There is no UNDEFINED behaviour since
proc.S checks the raw ID regs. Just passing arm64.nomte currently still
allows fully defined behaviour (well, unless you try to map tag storage
into the linear map but changing MAIR doesn't solve that anyway).
[...]
> diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
> index 907401e4fffb..3554ff869f4b 100644
> --- a/arch/arm64/kernel/cpufeature.c
> +++ b/arch/arm64/kernel/cpufeature.c
> @@ -2030,8 +2030,47 @@ static void bti_enable(const struct arm64_cpu_capabilities *__unused)
> #ifdef CONFIG_ARM64_MTE
> static void cpu_enable_mte(struct arm64_cpu_capabilities const *cap)
> {
> + u64 rgsr;
> +
> sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_ATA | SCTLR_EL1_ATA0);
> +
> + /*
> + * CnP must be enabled only after the MAIR_EL1 register has been set
> + * up. Inconsistent MAIR_EL1 between CPUs sharing the same TLB may
> + * lead to the wrong memory type being used for a brief window during
> + * CPU power-up.
> + *
> + * CnP is not a boot feature so MTE gets enabled before CnP, but let's
> + * make sure that is the case.
> + */
> + BUG_ON(read_sysreg(ttbr0_el1) & TTBR_CNP_BIT);
> + BUG_ON(read_sysreg(ttbr1_el1) & TTBR_CNP_BIT);
Ah, good point. SCOPE_BOOT_CPU features are initialised before the
others even for late secondary CPUs, so that should work without having
to reorder the features table.
> +
> + /* Normal Tagged memory type at the corresponding MAIR index */
> + sysreg_clear_set(
> + mair_el1, MAIR_ATTRIDX(MAIR_ATTR_MASK, MT_NORMAL_TAGGED),
> + MAIR_ATTRIDX(MAIR_ATTR_NORMAL_TAGGED, MT_NORMAL_TAGGED));
Nitpick: keep 'mair_el1' on the same line with sysreg_clear_set, I think
it looks slightly better if MAIR_ATTRIDX are both aligned.
[...]
> - /* set the TCR_EL1 bits */
> - mov_q x10, TCR_MTE_FLAGS
> - orr tcr, tcr, x10
I'd keep the TCR setting under #ifdef MTE or rather the TCR_MTE_FLAGS
and make them 0 if !MTE. It gives us a chance to still test a kernel
configuration where TBI1 == 0.
BTW, we end up setting the TCMA1 bit even when MTE is not supported. It
shouldn't be a problem usually with RES0 bits which we know what they
do.
--
Catalin