The Armv8.8 extension adds new instructions to perform memcpy(), memset() and
memmove() operations in hardware (FEAT_MOPS). The aim is to avoid having many
different performance-optimal memcpy implementations in software (tailored to
CPU model and copy size) and the overhead of selecting between them. The new
instructions are intended to be at least as fast as any alternative instruction
sequence.
This series adds support for using the new instructions in userspace. The most
noteworthy part is some exception handling for when there are different memcpy
implementations in the same system. In addition, the first patches in the
series set up basic handling for the new HCRX_EL2 register (added in Armv8.7),
which is needed to disable MOPS in KVM guests for now.
Support for using memcpy instructions in KVM guests will follow in a future
series, as well as kselftests and more debug support.
Kristina Martsenko (10):
KVM: arm64: initialize HCRX_EL2
arm64: cpufeature: detect FEAT_HCX
KVM: arm64: switch HCRX_EL2 between host and guest
arm64: mops: document boot requirements for MOPS
arm64: mops: don't disable host MOPS instructions from EL2
KVM: arm64: hide MOPS from guests
arm64: mops: handle MOPS exceptions
arm64: mops: handle single stepping after MOPS exception
arm64: mops: detect and enable FEAT_MOPS
arm64: mops: allow disabling MOPS from the kernel command line
.../admin-guide/kernel-parameters.txt | 3 ++
Documentation/arm64/booting.rst | 8 +++
Documentation/arm64/cpu-feature-registers.rst | 2 +
Documentation/arm64/elf_hwcaps.rst | 3 ++
arch/arm64/include/asm/el2_setup.h | 8 ---
arch/arm64/include/asm/esr.h | 11 +++-
arch/arm64/include/asm/exception.h | 1 +
arch/arm64/include/asm/hwcap.h | 1 +
arch/arm64/include/asm/kvm_arm.h | 4 ++
arch/arm64/include/uapi/asm/hwcap.h | 1 +
arch/arm64/kernel/cpufeature.c | 31 +++++++++++
arch/arm64/kernel/cpuinfo.c | 1 +
arch/arm64/kernel/entry-common.c | 11 ++++
arch/arm64/kernel/head.S | 7 +++
arch/arm64/kernel/idreg-override.c | 2 +
arch/arm64/kernel/traps.c | 51 +++++++++++++++++++
arch/arm64/kvm/hyp/include/hyp/switch.h | 6 +++
arch/arm64/kvm/hyp/nvhe/hyp-init.S | 6 +++
arch/arm64/kvm/sys_regs.c | 1 +
arch/arm64/tools/cpucaps | 2 +
20 files changed, 151 insertions(+), 9 deletions(-)
--
2.25.1
ARMv8.7/9.2 adds a new hypervisor configuration register HCRX_EL2.
Initialize the register to a safe value (all fields 0), to be robust
against firmware that has not initialized it. This is also needed to
ensure that the register is reinitialized after kexec.
In addition, move SMPME setup over to the new flags, as it would
otherwise get overridden. It is safe to set the bit even if SME is not
(uniformly) supported, as it will write to a RES0 bit (having no
effect), and SME will be disabled by the cpufeature framework.
(Similar to how e.g. the API bit is handled in HCR_HOST_NVHE_FLAGS.)
Signed-off-by: Kristina Martsenko <[email protected]>
---
arch/arm64/include/asm/el2_setup.h | 8 --------
arch/arm64/include/asm/kvm_arm.h | 3 +++
arch/arm64/kernel/head.S | 7 +++++++
arch/arm64/kvm/hyp/nvhe/hyp-init.S | 6 ++++++
4 files changed, 16 insertions(+), 8 deletions(-)
diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h
index e62785923ff6..699154229b15 100644
--- a/arch/arm64/include/asm/el2_setup.h
+++ b/arch/arm64/include/asm/el2_setup.h
@@ -284,14 +284,6 @@
cbz x1, .Lskip_sme_\@
msr_s SYS_SMPRIMAP_EL2, xzr // Make all priorities equal
-
- mrs x1, id_aa64mmfr1_el1 // HCRX_EL2 present?
- ubfx x1, x1, #ID_AA64MMFR1_EL1_HCX_SHIFT, #4
- cbz x1, .Lskip_sme_\@
-
- mrs_s x1, SYS_HCRX_EL2
- orr x1, x1, #HCRX_EL2_SMPME_MASK // Enable priority mapping
- msr_s SYS_HCRX_EL2, x1
.Lskip_sme_\@:
.endm
diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h
index 5a4b2342d571..caa31f4ab1cd 100644
--- a/arch/arm64/include/asm/kvm_arm.h
+++ b/arch/arm64/include/asm/kvm_arm.h
@@ -9,6 +9,7 @@
#include <asm/esr.h>
#include <asm/memory.h>
+#include <asm/sysreg.h>
#include <asm/types.h>
/* Hyp Configuration Register (HCR) bits */
@@ -92,6 +93,8 @@
#define HCR_HOST_NVHE_PROTECTED_FLAGS (HCR_HOST_NVHE_FLAGS | HCR_TSC)
#define HCR_HOST_VHE_FLAGS (HCR_RW | HCR_TGE | HCR_E2H)
+#define HCRX_HOST_FLAGS (HCRX_EL2_SMPME)
+
/* TCR_EL2 Registers bits */
#define TCR_EL2_RES1 ((1U << 31) | (1 << 23))
#define TCR_EL2_TBI (1 << 20)
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 212d93aca5e6..e06b34322339 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -572,6 +572,13 @@ SYM_INNER_LABEL(init_el2, SYM_L_LOCAL)
msr hcr_el2, x0
isb
+ mrs x0, ID_AA64MMFR1_EL1
+ ubfx x0, x0, #ID_AA64MMFR1_EL1_HCX_SHIFT, #4
+ cbz x0, 3f
+ mov_q x1, HCRX_HOST_FLAGS
+ msr_s SYS_HCRX_EL2, x1
+ isb
+3:
init_el2_state
/* Hypervisor stub */
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
index a6d67c2bb5ae..01f854697c70 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
@@ -95,6 +95,12 @@ SYM_CODE_START_LOCAL(___kvm_hyp_init)
ldr x1, [x0, #NVHE_INIT_HCR_EL2]
msr hcr_el2, x1
+ mrs x1, ID_AA64MMFR1_EL1
+ ubfx x1, x1, #ID_AA64MMFR1_EL1_HCX_SHIFT, #4
+ cbz x1, 1f
+ mov_q x2, HCRX_HOST_FLAGS
+ msr_s SYS_HCRX_EL2, x2
+1:
ldr x1, [x0, #NVHE_INIT_VTTBR]
msr vttbr_el2, x1
--
2.25.1
Detect if the system has the new HCRX_EL2 register added in ARMv8.7/9.2,
so that subsequent patches can check for its presence.
Signed-off-by: Kristina Martsenko <[email protected]>
---
arch/arm64/kernel/cpufeature.c | 12 ++++++++++++
arch/arm64/tools/cpucaps | 1 +
2 files changed, 13 insertions(+)
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 45a42cf2191c..1b0a71541381 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -357,6 +357,7 @@ static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = {
static const struct arm64_ftr_bits ftr_id_aa64mmfr1[] = {
ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_TIDCP1_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_AFP_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_HCX_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_ETS_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_TWED_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_XNX_SHIFT, 4, 0),
@@ -2290,6 +2291,17 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
.matches = is_kvm_protected_mode,
},
+ {
+ .desc = "HCRX_EL2 register",
+ .capability = ARM64_HAS_HCX,
+ .type = ARM64_CPUCAP_SYSTEM_FEATURE,
+ .sys_reg = SYS_ID_AA64MMFR1_EL1,
+ .sign = FTR_UNSIGNED,
+ .field_pos = ID_AA64MMFR1_EL1_HCX_SHIFT,
+ .field_width = 4,
+ .min_field_value = ID_AA64MMFR1_EL1_HCX_IMP,
+ .matches = has_cpuid_feature,
+ },
#endif
{
.desc = "Kernel page table isolation (KPTI)",
diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps
index 10dcfa13390a..bf32a1122c04 100644
--- a/arch/arm64/tools/cpucaps
+++ b/arch/arm64/tools/cpucaps
@@ -31,6 +31,7 @@ HAS_GENERIC_AUTH_IMP_DEF
HAS_GIC_CPUIF_SYSREGS
HAS_GIC_PRIO_MASKING
HAS_GIC_PRIO_RELAXED_SYNC
+HAS_HCX
HAS_LDAPR
HAS_LSE_ATOMICS
HAS_NO_FPSIMD
--
2.25.1
Switch the HCRX_EL2 register between host and guest configurations, in
order to enable different features in the host and guest.
Note that the guest flags are only set if all CPUs have HCRX_EL2.
Asymmetric systems where only some CPUs have HCRX_EL2 are not supported
and will result in guests running with the host flags set (and a "SANITY
CHECK" warning printed for the host).
After this change, SMPME is no longer set for guests, which should have
no effect as SME is currently disabled for guests.
Signed-off-by: Kristina Martsenko <[email protected]>
---
I wasn't sure what to do about asymmetric systems. It seems a bit
fragile, maybe someone has a better idea?
arch/arm64/include/asm/kvm_arm.h | 1 +
arch/arm64/kvm/hyp/include/hyp/switch.h | 6 ++++++
2 files changed, 7 insertions(+)
diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h
index caa31f4ab1cd..cd8dd307aaba 100644
--- a/arch/arm64/include/asm/kvm_arm.h
+++ b/arch/arm64/include/asm/kvm_arm.h
@@ -93,6 +93,7 @@
#define HCR_HOST_NVHE_PROTECTED_FLAGS (HCR_HOST_NVHE_FLAGS | HCR_TSC)
#define HCR_HOST_VHE_FLAGS (HCR_RW | HCR_TGE | HCR_E2H)
+#define HCRX_GUEST_FLAGS 0
#define HCRX_HOST_FLAGS (HCRX_EL2_SMPME)
/* TCR_EL2 Registers bits */
diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h
index 07d37ff88a3f..a1bf2d879db5 100644
--- a/arch/arm64/kvm/hyp/include/hyp/switch.h
+++ b/arch/arm64/kvm/hyp/include/hyp/switch.h
@@ -129,6 +129,9 @@ static inline void ___activate_traps(struct kvm_vcpu *vcpu)
if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_VSE))
write_sysreg_s(vcpu->arch.vsesr_el2, SYS_VSESR_EL2);
+
+ if (cpus_have_final_cap(ARM64_HAS_HCX))
+ write_sysreg_s(HCRX_GUEST_FLAGS, SYS_HCRX_EL2);
}
static inline void ___deactivate_traps(struct kvm_vcpu *vcpu)
@@ -143,6 +146,9 @@ static inline void ___deactivate_traps(struct kvm_vcpu *vcpu)
vcpu->arch.hcr_el2 &= ~HCR_VSE;
vcpu->arch.hcr_el2 |= read_sysreg(hcr_el2) & HCR_VSE;
}
+
+ if (cpus_have_final_cap(ARM64_HAS_HCX))
+ write_sysreg_s(HCRX_HOST_FLAGS, SYS_HCRX_EL2);
}
static inline bool __populate_fault_info(struct kvm_vcpu *vcpu)
--
2.25.1
FEAT_MOPS introduces new instructions, we require that these
instructions not execute as UNDEFINED when we identify that the feature
is supported. Additionally, the instructions may generate a Memory Copy
or Memory Set exception, we require that the exception be taken to EL1
in case EL2 is not able to correctly handle it.
Signed-off-by: Kristina Martsenko <[email protected]>
---
Documentation/arm64/booting.rst | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/Documentation/arm64/booting.rst b/Documentation/arm64/booting.rst
index ffeccdd6bdac..f3913ee94c42 100644
--- a/Documentation/arm64/booting.rst
+++ b/Documentation/arm64/booting.rst
@@ -379,6 +379,14 @@ Before jumping into the kernel, the following conditions must be met:
- SMCR_EL2.EZT0 (bit 30) must be initialised to 0b1.
+ For CPUs with Memory Copy and Memory Set instructions (FEAT_MOPS):
+
+ - If the kernel is entered at EL1 and EL2 is present:
+
+ - HCRX_EL2.MSCEn (bit 11) must be initialised to 0b1.
+
+ - HCRX_EL2.MCE2 (bit 10) must be initialised to 0b0.
+
The requirements described above for CPU mode, caches, MMUs, architected
timers, coherency and system registers apply to all CPUs. All CPUs must
enter the kernel in the same exception level. Where the values documented
--
2.25.1
To allow nVHE host EL0 and EL1 to use FEAT_MOPS instructions, configure
EL2 to not cause these instructions to be treated as UNDEFINED. A VHE
host is unaffected by this control.
Signed-off-by: Kristina Martsenko <[email protected]>
---
arch/arm64/include/asm/kvm_arm.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h
index cd8dd307aaba..f37d37957322 100644
--- a/arch/arm64/include/asm/kvm_arm.h
+++ b/arch/arm64/include/asm/kvm_arm.h
@@ -94,7 +94,7 @@
#define HCR_HOST_VHE_FLAGS (HCR_RW | HCR_TGE | HCR_E2H)
#define HCRX_GUEST_FLAGS 0
-#define HCRX_HOST_FLAGS (HCRX_EL2_SMPME)
+#define HCRX_HOST_FLAGS (HCRX_EL2_SMPME | HCRX_EL2_MSCEn)
/* TCR_EL2 Registers bits */
#define TCR_EL2_RES1 ((1U << 31) | (1 << 23))
--
2.25.1
As FEAT_MOPS is not supported in guests yet, hide it from the ID
registers for guests.
The MOPS instructions are UNDEFINED in guests as HCRX_EL2.MSCEn is not
set in HCRX_GUEST_FLAGS, and will take an exception to EL1 if executed.
Signed-off-by: Kristina Martsenko <[email protected]>
---
arch/arm64/kvm/sys_regs.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 0bc7df55916e..3456205c6b92 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -1213,6 +1213,7 @@ static u64 read_id_reg(const struct kvm_vcpu *vcpu, struct sys_reg_desc const *r
ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_GPA3));
if (!cpus_have_final_cap(ARM64_HAS_WFXT))
val &= ~ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_WFxT);
+ val &= ~ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_MOPS);
break;
case SYS_ID_AA64DFR0_EL1:
/* Limit debug to ARMv8.0 */
--
2.25.1
The memory copy/set instructions added as part of FEAT_MOPS can take an
exception part-way through their execution and resume execution
afterwards. If however the task is re-scheduled and execution resumes on
a different CPU, then the CPU may take a new type of exception to
indicate this. In this case the OS has to reset the registers and restart
execution from the prologue instruction. The algorithm for doing this is
provided as part of the Arm ARM.
Add an exception handler for the new exception and wire it up for
userspace tasks.
Signed-off-by: Kristina Martsenko <[email protected]>
---
arch/arm64/include/asm/esr.h | 11 +++++++-
arch/arm64/include/asm/exception.h | 1 +
arch/arm64/kernel/entry-common.c | 11 ++++++++
arch/arm64/kernel/traps.c | 45 ++++++++++++++++++++++++++++++
4 files changed, 67 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h
index c9f15b9e3c71..96caaaee97a3 100644
--- a/arch/arm64/include/asm/esr.h
+++ b/arch/arm64/include/asm/esr.h
@@ -47,7 +47,7 @@
#define ESR_ELx_EC_DABT_LOW (0x24)
#define ESR_ELx_EC_DABT_CUR (0x25)
#define ESR_ELx_EC_SP_ALIGN (0x26)
-/* Unallocated EC: 0x27 */
+#define ESR_ELx_EC_MOPS (0x27)
#define ESR_ELx_EC_FP_EXC32 (0x28)
/* Unallocated EC: 0x29 - 0x2B */
#define ESR_ELx_EC_FP_EXC64 (0x2C)
@@ -352,6 +352,15 @@
#define ESR_ELx_SME_ISS_ZA_DISABLED 3
#define ESR_ELx_SME_ISS_ZT_DISABLED 4
+/* ISS field definitions for MOPS exceptions */
+#define ESR_ELx_MOPS_ISS_MEM_INST (UL(1) << 24)
+#define ESR_ELx_MOPS_ISS_FROM_EPILOGUE (UL(1) << 18)
+#define ESR_ELx_MOPS_ISS_WRONG_OPTION (UL(1) << 17)
+#define ESR_ELx_MOPS_ISS_OPTION_A (UL(1) << 16)
+#define ESR_ELx_MOPS_ISS_DESTREG(esr) (((esr) & (UL(0x1f) << 10)) >> 10)
+#define ESR_ELx_MOPS_ISS_SRCREG(esr) (((esr) & (UL(0x1f) << 5)) >> 5)
+#define ESR_ELx_MOPS_ISS_SIZEREG(esr) (((esr) & (UL(0x1f) << 0)) >> 0)
+
#ifndef __ASSEMBLY__
#include <asm/types.h>
diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h
index 92963f98afec..5a6dc3643e9b 100644
--- a/arch/arm64/include/asm/exception.h
+++ b/arch/arm64/include/asm/exception.h
@@ -77,6 +77,7 @@ void do_el0_svc(struct pt_regs *regs);
void do_el0_svc_compat(struct pt_regs *regs);
void do_el0_fpac(struct pt_regs *regs, unsigned long esr);
void do_el1_fpac(struct pt_regs *regs, unsigned long esr);
+void do_el0_mops(struct pt_regs *regs, unsigned long esr);
void do_serror(struct pt_regs *regs, unsigned long esr);
void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags);
diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
index cce1167199e3..2ef3ab5d7555 100644
--- a/arch/arm64/kernel/entry-common.c
+++ b/arch/arm64/kernel/entry-common.c
@@ -611,6 +611,14 @@ static void noinstr el0_bti(struct pt_regs *regs)
exit_to_user_mode(regs);
}
+static void noinstr el0_mops(struct pt_regs *regs, unsigned long esr)
+{
+ enter_from_user_mode(regs);
+ local_daif_restore(DAIF_PROCCTX);
+ do_el0_mops(regs, esr);
+ exit_to_user_mode(regs);
+}
+
static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr)
{
enter_from_user_mode(regs);
@@ -688,6 +696,9 @@ asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)
case ESR_ELx_EC_BTI:
el0_bti(regs);
break;
+ case ESR_ELx_EC_MOPS:
+ el0_mops(regs, esr);
+ break;
case ESR_ELx_EC_BREAKPT_LOW:
case ESR_ELx_EC_SOFTSTP_LOW:
case ESR_ELx_EC_WATCHPT_LOW:
diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
index 0ccc063daccb..689188712909 100644
--- a/arch/arm64/kernel/traps.c
+++ b/arch/arm64/kernel/traps.c
@@ -507,6 +507,50 @@ void do_el1_fpac(struct pt_regs *regs, unsigned long esr)
die("Oops - FPAC", regs, esr);
}
+void do_el0_mops(struct pt_regs *regs, unsigned long esr)
+{
+ bool wrong_option = esr & ESR_ELx_MOPS_ISS_WRONG_OPTION;
+ bool option_a = esr & ESR_ELx_MOPS_ISS_OPTION_A;
+ int dstreg = ESR_ELx_MOPS_ISS_DESTREG(esr);
+ int srcreg = ESR_ELx_MOPS_ISS_SRCREG(esr);
+ int sizereg = ESR_ELx_MOPS_ISS_SIZEREG(esr);
+ unsigned long dst, src, size;
+
+ dst = pt_regs_read_reg(regs, dstreg);
+ src = pt_regs_read_reg(regs, srcreg);
+ size = pt_regs_read_reg(regs, sizereg);
+
+ /*
+ * Put the registers back in the original format suitable for a
+ * prologue instruction, using the generic return routine from the
+ * Arm ARM (DDI 0487I.a) rules CNTMJ and MWFQH.
+ */
+ if (esr & ESR_ELx_MOPS_ISS_MEM_INST) {
+ if ((!option_a && wrong_option) || (option_a && !wrong_option)) {
+ pt_regs_write_reg(regs, dstreg, dst + size);
+ pt_regs_write_reg(regs, sizereg, -size);
+ }
+ } else {
+ if ((option_a && wrong_option) || (!option_a && !wrong_option)) {
+ if (regs->pstate & PSR_N_BIT) {
+ pt_regs_write_reg(regs, dstreg, dst - size);
+ pt_regs_write_reg(regs, srcreg, src - size);
+ }
+ } else {
+ if (size & BIT(63)) {
+ pt_regs_write_reg(regs, dstreg, dst + size);
+ pt_regs_write_reg(regs, srcreg, src + size);
+ pt_regs_write_reg(regs, sizereg, -size);
+ }
+ }
+ }
+
+ if (esr & ESR_ELx_MOPS_ISS_FROM_EPILOGUE)
+ regs->pc -= 8;
+ else
+ regs->pc -= 4;
+}
+
#define __user_cache_maint(insn, address, res) \
if (address >= TASK_SIZE_MAX) { \
res = -EFAULT; \
@@ -817,6 +861,7 @@ static const char *esr_class_str[] = {
[ESR_ELx_EC_DABT_LOW] = "DABT (lower EL)",
[ESR_ELx_EC_DABT_CUR] = "DABT (current EL)",
[ESR_ELx_EC_SP_ALIGN] = "SP Alignment",
+ [ESR_ELx_EC_MOPS] = "MOPS",
[ESR_ELx_EC_FP_EXC32] = "FP (AArch32)",
[ESR_ELx_EC_FP_EXC64] = "FP (AArch64)",
[ESR_ELx_EC_SERROR] = "SError",
--
2.25.1
When a MOPS main or epilogue instruction is being executed, the task may
get scheduled on a different CPU and restart execution from the prologue
instruction. If the main or epilogue instruction is being single stepped
then it makes sense to finish the step and take the step exception
before starting to execute the next (prologue) instruction. So
fast-forward the single step state machine when taking a MOPS exception.
This means that if a main or epilogue instruction is single stepped with
ptrace, the debugger will sometimes observe the PC moving back to the
prologue instruction. (As already mentioned, this should be rare as it
only happens when the task is scheduled to another CPU during the step.)
This also ensures that perf breakpoints count prologue instructions
consistently (i.e. every time they are executed), rather than skipping
them when there also happens to be a breakpoint on a main or epilogue
instruction.
Signed-off-by: Kristina Martsenko <[email protected]>
---
arch/arm64/kernel/traps.c | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
index 689188712909..3dfc901a430b 100644
--- a/arch/arm64/kernel/traps.c
+++ b/arch/arm64/kernel/traps.c
@@ -549,6 +549,12 @@ void do_el0_mops(struct pt_regs *regs, unsigned long esr)
regs->pc -= 8;
else
regs->pc -= 4;
+
+ /*
+ * If single stepping then finish the step before executing the
+ * prologue instruction.
+ */
+ user_fastforward_single_step(current);
}
#define __user_cache_maint(insn, address, res) \
--
2.25.1
The Arm v8.8/9.3 FEAT_MOPS feature provides new instructions that
perform a memory copy or set. Wire up the cpufeature code to detect the
presence of FEAT_MOPS and enable it.
Signed-off-by: Kristina Martsenko <[email protected]>
---
Documentation/arm64/cpu-feature-registers.rst | 2 ++
Documentation/arm64/elf_hwcaps.rst | 3 +++
arch/arm64/include/asm/hwcap.h | 1 +
arch/arm64/include/uapi/asm/hwcap.h | 1 +
arch/arm64/kernel/cpufeature.c | 19 +++++++++++++++++++
arch/arm64/kernel/cpuinfo.c | 1 +
arch/arm64/tools/cpucaps | 1 +
7 files changed, 28 insertions(+)
diff --git a/Documentation/arm64/cpu-feature-registers.rst b/Documentation/arm64/cpu-feature-registers.rst
index c7adc7897df6..4e4625f2455f 100644
--- a/Documentation/arm64/cpu-feature-registers.rst
+++ b/Documentation/arm64/cpu-feature-registers.rst
@@ -288,6 +288,8 @@ infrastructure:
+------------------------------+---------+---------+
| Name | bits | visible |
+------------------------------+---------+---------+
+ | MOPS | [19-16] | y |
+ +------------------------------+---------+---------+
| RPRES | [7-4] | y |
+------------------------------+---------+---------+
| WFXT | [3-0] | y |
diff --git a/Documentation/arm64/elf_hwcaps.rst b/Documentation/arm64/elf_hwcaps.rst
index 83e57e4d38e2..8f847d0dcf57 100644
--- a/Documentation/arm64/elf_hwcaps.rst
+++ b/Documentation/arm64/elf_hwcaps.rst
@@ -302,6 +302,9 @@ HWCAP2_SMEB16B16
HWCAP2_SMEF16F16
Functionality implied by ID_AA64SMFR0_EL1.F16F16 == 0b1
+HWCAP2_MOPS
+ Functionality implied by ID_AA64ISAR2_EL1.MOPS == 0b0001.
+
4. Unused AT_HWCAP bits
-----------------------
diff --git a/arch/arm64/include/asm/hwcap.h b/arch/arm64/include/asm/hwcap.h
index 5d45f19fda7f..692b1ec663b2 100644
--- a/arch/arm64/include/asm/hwcap.h
+++ b/arch/arm64/include/asm/hwcap.h
@@ -137,6 +137,7 @@
#define KERNEL_HWCAP_SME_BI32I32 __khwcap2_feature(SME_BI32I32)
#define KERNEL_HWCAP_SME_B16B16 __khwcap2_feature(SME_B16B16)
#define KERNEL_HWCAP_SME_F16F16 __khwcap2_feature(SME_F16F16)
+#define KERNEL_HWCAP_MOPS __khwcap2_feature(MOPS)
/*
* This yields a mask that user programs can use to figure out what
diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h
index 69a4fb749c65..a2cac4305b1e 100644
--- a/arch/arm64/include/uapi/asm/hwcap.h
+++ b/arch/arm64/include/uapi/asm/hwcap.h
@@ -102,5 +102,6 @@
#define HWCAP2_SME_BI32I32 (1UL << 40)
#define HWCAP2_SME_B16B16 (1UL << 41)
#define HWCAP2_SME_F16F16 (1UL << 42)
+#define HWCAP2_MOPS (1UL << 43)
#endif /* _UAPI__ASM_HWCAP_H */
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 1b0a71541381..70a08b2ddaa6 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -216,6 +216,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar2[] = {
ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_CSSC_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_RPRFM_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_AA64ISAR2_EL1_BC_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_MOPS_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
FTR_STRICT, FTR_EXACT, ID_AA64ISAR2_EL1_APA3_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
@@ -2158,6 +2159,11 @@ static void cpu_enable_dit(const struct arm64_cpu_capabilities *__unused)
set_pstate_dit(1);
}
+static void cpu_enable_mops(const struct arm64_cpu_capabilities *__unused)
+{
+ sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_MSCEn);
+}
+
/* Internal helper functions to match cpu capability type */
static bool
cpucap_late_cpu_optional(const struct arm64_cpu_capabilities *cap)
@@ -2759,6 +2765,18 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
.matches = has_cpuid_feature,
.cpu_enable = cpu_enable_dit,
},
+ {
+ .desc = "Memory Copy and Memory Set instructions",
+ .capability = ARM64_HAS_MOPS,
+ .type = ARM64_CPUCAP_SYSTEM_FEATURE,
+ .sys_reg = SYS_ID_AA64ISAR2_EL1,
+ .sign = FTR_UNSIGNED,
+ .field_pos = ID_AA64ISAR2_EL1_MOPS_SHIFT,
+ .field_width = 4,
+ .min_field_value = ID_AA64ISAR2_EL1_MOPS_IMP,
+ .matches = has_cpuid_feature,
+ .cpu_enable = cpu_enable_mops,
+ },
{},
};
@@ -2891,6 +2909,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
HWCAP_CAP(ID_AA64ISAR2_EL1, RPRFM, IMP, CAP_HWCAP, KERNEL_HWCAP_RPRFM),
HWCAP_CAP(ID_AA64ISAR2_EL1, RPRES, IMP, CAP_HWCAP, KERNEL_HWCAP_RPRES),
HWCAP_CAP(ID_AA64ISAR2_EL1, WFxT, IMP, CAP_HWCAP, KERNEL_HWCAP_WFXT),
+ HWCAP_CAP(ID_AA64ISAR2_EL1, MOPS, IMP, CAP_HWCAP, KERNEL_HWCAP_MOPS),
#ifdef CONFIG_ARM64_SME
HWCAP_CAP(ID_AA64PFR1_EL1, SME, IMP, CAP_HWCAP, KERNEL_HWCAP_SME),
HWCAP_CAP(ID_AA64SMFR0_EL1, FA64, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_FA64),
diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c
index eb4378c23b3c..076a124255d0 100644
--- a/arch/arm64/kernel/cpuinfo.c
+++ b/arch/arm64/kernel/cpuinfo.c
@@ -125,6 +125,7 @@ static const char *const hwcap_str[] = {
[KERNEL_HWCAP_SME_BI32I32] = "smebi32i32",
[KERNEL_HWCAP_SME_B16B16] = "smeb16b16",
[KERNEL_HWCAP_SME_F16F16] = "smef16f16",
+ [KERNEL_HWCAP_MOPS] = "mops",
};
#ifdef CONFIG_COMPAT
diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps
index bf32a1122c04..5018da44def2 100644
--- a/arch/arm64/tools/cpucaps
+++ b/arch/arm64/tools/cpucaps
@@ -34,6 +34,7 @@ HAS_GIC_PRIO_RELAXED_SYNC
HAS_HCX
HAS_LDAPR
HAS_LSE_ATOMICS
+HAS_MOPS
HAS_NO_FPSIMD
HAS_NO_HW_PREFETCH
HAS_PAN
--
2.25.1
Make it possible to disable the MOPS extension at runtime using the
kernel command line. This can be useful for testing or working around
hardware issues.
Signed-off-by: Kristina Martsenko <[email protected]>
---
Documentation/admin-guide/kernel-parameters.txt | 3 +++
arch/arm64/kernel/idreg-override.c | 2 ++
2 files changed, 5 insertions(+)
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 6cfa6e3996cf..ee86fe17352d 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -408,6 +408,9 @@
arm64.nosme [ARM64] Unconditionally disable Scalable Matrix
Extension support
+ arm64.nomops [ARM64] Unconditionally disable Memory Copy and Memory
+ Set instructions support
+
ataflop= [HW,M68k]
atarimouse= [HW,MOUSE] Atari Mouse
diff --git a/arch/arm64/kernel/idreg-override.c b/arch/arm64/kernel/idreg-override.c
index d833d78a7f31..d13ae90042cb 100644
--- a/arch/arm64/kernel/idreg-override.c
+++ b/arch/arm64/kernel/idreg-override.c
@@ -123,6 +123,7 @@ static const struct ftr_set_desc isar2 __initconst = {
.fields = {
FIELD("gpa3", ID_AA64ISAR2_EL1_GPA3_SHIFT, NULL),
FIELD("apa3", ID_AA64ISAR2_EL1_APA3_SHIFT, NULL),
+ FIELD("mops", ID_AA64ISAR2_EL1_MOPS_SHIFT, NULL),
{}
},
};
@@ -174,6 +175,7 @@ static const struct {
"id_aa64isar1.gpi=0 id_aa64isar1.gpa=0 "
"id_aa64isar1.api=0 id_aa64isar1.apa=0 "
"id_aa64isar2.gpa3=0 id_aa64isar2.apa3=0" },
+ { "arm64.nomops", "id_aa64isar2.mops=0" },
{ "arm64.nomte", "id_aa64pfr1.mte=0" },
{ "nokaslr", "kaslr.disabled=1" },
};
--
2.25.1
On Thu, Feb 16, 2023 at 04:00:05PM +0000, Kristina Martsenko wrote:
> After this change, SMPME is no longer set for guests, which should have
> no effect as SME is currently disabled for guests.
SMPME is mainly set for the benefit of any future guests, it is
used to turn off SME priorities for guests. While it's true that
we don't have any guests support yet it's there as safety to
ensure we don't forget to enable it later on or something
otherwise goes wrong. We don't need priority mapping on the host
since we don't have any priority mapping support at all, and it's
difficult to see a reason why we would want to remap our own
priorities in the host.
On Thu, Feb 16, 2023 at 04:00:11PM +0000, Kristina Martsenko wrote:
> The Arm v8.8/9.3 FEAT_MOPS feature provides new instructions that
> perform a memory copy or set. Wire up the cpufeature code to detect the
> presence of FEAT_MOPS and enable it.
>
> Signed-off-by: Kristina Martsenko <[email protected]>
> ---
> Documentation/arm64/cpu-feature-registers.rst | 2 ++
> Documentation/arm64/elf_hwcaps.rst | 3 +++
> arch/arm64/include/asm/hwcap.h | 1 +
> arch/arm64/include/uapi/asm/hwcap.h | 1 +
> arch/arm64/kernel/cpufeature.c | 19 +++++++++++++++++++
> arch/arm64/kernel/cpuinfo.c | 1 +
> arch/arm64/tools/cpucaps | 1 +
> 7 files changed, 28 insertions(+)
Please also add this to the hwcaps selftest.
On Thu, 16 Feb 2023 16:00:05 +0000,
Kristina Martsenko <[email protected]> wrote:
>
> Switch the HCRX_EL2 register between host and guest configurations, in
> order to enable different features in the host and guest.
>
> Note that the guest flags are only set if all CPUs have HCRX_EL2.
> Asymmetric systems where only some CPUs have HCRX_EL2 are not supported
> and will result in guests running with the host flags set (and a "SANITY
> CHECK" warning printed for the host).
>
> After this change, SMPME is no longer set for guests, which should have
> no effect as SME is currently disabled for guests.
Why not preserve the behaviour by propagating the flag into the guest
setup?
>
> Signed-off-by: Kristina Martsenko <[email protected]>
> ---
>
> I wasn't sure what to do about asymmetric systems. It seems a bit
> fragile, maybe someone has a better idea?
I would simply prevent these CPUs from booting if they come after a
primary CPU that has the feature. These hypothetical asymmetric setups
put a huge complexity on the kernel, and I'm worried that we're just
giving implementers too much freedom.
If someone comes up with that sort of stuff, they can write the
patches themselves... Or do you know of any braindead setup involving
an asymmetric FEAT_HCX implementation?
Thanks,
M.
--
Without deviation from the norm, progress is not possible.
On 16/02/2023 16:20, Mark Brown wrote:
> On Thu, Feb 16, 2023 at 04:00:05PM +0000, Kristina Martsenko wrote:
>
>> After this change, SMPME is no longer set for guests, which should have
>> no effect as SME is currently disabled for guests.
>
> SMPME is mainly set for the benefit of any future guests, it is
> used to turn off SME priorities for guests. While it's true that
> we don't have any guests support yet it's there as safety to
> ensure we don't forget to enable it later on or something
> otherwise goes wrong. We don't need priority mapping on the host
> since we don't have any priority mapping support at all, and it's
> difficult to see a reason why we would want to remap our own
> priorities in the host.
That makes sense. I'll fix this in v2 so that priority mapping is enabled for
guests and disabled for the host.
Thanks,
Kristina
On 16/02/2023 16:35, Marc Zyngier wrote:
> On Thu, 16 Feb 2023 16:00:05 +0000,
> Kristina Martsenko <[email protected]> wrote:
>>
>> Switch the HCRX_EL2 register between host and guest configurations, in
>> order to enable different features in the host and guest.
>>
>> Note that the guest flags are only set if all CPUs have HCRX_EL2.
>> Asymmetric systems where only some CPUs have HCRX_EL2 are not supported
>> and will result in guests running with the host flags set (and a "SANITY
>> CHECK" warning printed for the host).
>>
>> After this change, SMPME is no longer set for guests, which should have
>> no effect as SME is currently disabled for guests.
>
> Why not preserve the behaviour by propagating the flag into the guest
> setup?
I thought it made more sense to disable SMPME given that SME is not supported
in guests yet (and that the existing behavior was just a side effect of not
having support for switching HCRX), but I'd misunderstood what SMPME is for,
and following Mark's explanation I'll actually preserve the behavior for
guests, but now disable SMPME for the host instead (as SME priority mapping has
no benefit in the host and is not intended to be used there).
>
>>
>> Signed-off-by: Kristina Martsenko <[email protected]>
>> ---
>>
>> I wasn't sure what to do about asymmetric systems. It seems a bit
>> fragile, maybe someone has a better idea?
>
> I would simply prevent these CPUs from booting if they come after a
> primary CPU that has the feature.
I considered that but the concern I heard was that since virtualization is an
optional feature then people may still want to use the system without it.
> These hypothetical asymmetric setups
> put a huge complexity on the kernel, and I'm worried that we're just
> giving implementers too much freedom.
>
> If someone comes up with that sort of stuff, they can write the
> patches themselves...
I'll make it panic on a mismatch for now and it can be revisited in the future
if such a system actually appears (which does seem very unlikely).
> Or do you know of any braindead setup involving
> an asymmetric FEAT_HCX implementation?
Nope don't know of one, it was just hypothetical.
Thanks,
Kristina
On Thu, Feb 16, 2023 at 04:00:03PM +0000, Kristina Martsenko wrote:
> ARMv8.7/9.2 adds a new hypervisor configuration register HCRX_EL2.
> Initialize the register to a safe value (all fields 0), to be robust
> against firmware that has not initialized it.
I think the risk of firmware not initialising this register is small
given that EL3 needs to set SCR_EL3.HXEn to allow EL2 access. But it
doesn't hurt to re-initialise it in the hypervisor.
> diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
> index 212d93aca5e6..e06b34322339 100644
> --- a/arch/arm64/kernel/head.S
> +++ b/arch/arm64/kernel/head.S
> @@ -572,6 +572,13 @@ SYM_INNER_LABEL(init_el2, SYM_L_LOCAL)
> msr hcr_el2, x0
> isb
>
> + mrs x0, ID_AA64MMFR1_EL1
> + ubfx x0, x0, #ID_AA64MMFR1_EL1_HCX_SHIFT, #4
> + cbz x0, 3f
> + mov_q x1, HCRX_HOST_FLAGS
> + msr_s SYS_HCRX_EL2, x1
> + isb
> +3:
> init_el2_state
Nitpick: we can probably leave a single ISB after both HCR_EL2 and
HCRX_EL2 are initialised. Well, we could probably drop all of them
altogether, there's at least one down this path.
>
> /* Hypervisor stub */
> diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
> index a6d67c2bb5ae..01f854697c70 100644
> --- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S
> +++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
> @@ -95,6 +95,12 @@ SYM_CODE_START_LOCAL(___kvm_hyp_init)
> ldr x1, [x0, #NVHE_INIT_HCR_EL2]
> msr hcr_el2, x1
>
> + mrs x1, ID_AA64MMFR1_EL1
> + ubfx x1, x1, #ID_AA64MMFR1_EL1_HCX_SHIFT, #4
> + cbz x1, 1f
> + mov_q x2, HCRX_HOST_FLAGS
> + msr_s SYS_HCRX_EL2, x2
> +1:
Maybe you could use a macro to avoid writing this sequence twice. I lost
track of the KVM initialisation refactoring since pKVM, it looks like
the other register values are loaded from a structure here. I guess a
value of 0 doesn't make sense to store (unless at a later point it
becomes non-zero).
--
Catalin
On Thu, Feb 16, 2023 at 04:00:04PM +0000, Kristina Martsenko wrote:
> Detect if the system has the new HCRX_EL2 register added in ARMv8.7/9.2,
> so that subsequent patches can check for its presence.
>
> Signed-off-by: Kristina Martsenko <[email protected]>
Reviewed-by: Catalin Marinas <[email protected]>
On Thu, Feb 16, 2023 at 04:00:07PM +0000, Kristina Martsenko wrote:
> To allow nVHE host EL0 and EL1 to use FEAT_MOPS instructions, configure
> EL2 to not cause these instructions to be treated as UNDEFINED. A VHE
> host is unaffected by this control.
>
> Signed-off-by: Kristina Martsenko <[email protected]>
Reviewed-by: Catalin Marinas <[email protected]>
On Thu, Feb 16, 2023 at 04:00:06PM +0000, Kristina Martsenko wrote:
> + For CPUs with Memory Copy and Memory Set instructions (FEAT_MOPS):
> +
> + - If the kernel is entered at EL1 and EL2 is present:
> +
> + - HCRX_EL2.MSCEn (bit 11) must be initialised to 0b1.
> +
> + - HCRX_EL2.MCE2 (bit 10) must be initialised to 0b0.
Regarding MCE2, does EL1 actually care if EL2 wants to handle all the
memcpy/memset exceptions? There may even be a valid case to do this at
EL2 if you run a guest that uses these instructions but has no clue on
how to deal with the specific exception like WrongOption.
--
Catalin
On Thu, Feb 16, 2023 at 04:00:08PM +0000, Kristina Martsenko wrote:
> As FEAT_MOPS is not supported in guests yet, hide it from the ID
> registers for guests.
>
> The MOPS instructions are UNDEFINED in guests as HCRX_EL2.MSCEn is not
> set in HCRX_GUEST_FLAGS, and will take an exception to EL1 if executed.
>
> Signed-off-by: Kristina Martsenko <[email protected]>
Acked-by: Catalin Marinas <[email protected]>
On Thu, Feb 16, 2023 at 04:00:09PM +0000, Kristina Martsenko wrote:
> The memory copy/set instructions added as part of FEAT_MOPS can take an
> exception part-way through their execution and resume execution
> afterwards. If however the task is re-scheduled and execution resumes on
> a different CPU, then the CPU may take a new type of exception to
> indicate this. In this case the OS has to reset the registers and restart
> execution from the prologue instruction. The algorithm for doing this is
> provided as part of the Arm ARM.
>
> Add an exception handler for the new exception and wire it up for
> userspace tasks.
>
> Signed-off-by: Kristina Martsenko <[email protected]>
It may be worth adding a short note in the cover that the architecture
allows two options to implement the memory instructions and on a
heterogeneous system we can have different implementations between CPUs.
That's the reason for an exception after migration (it's not obvious
from your text above).
> diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h
> index c9f15b9e3c71..96caaaee97a3 100644
> --- a/arch/arm64/include/asm/esr.h
> +++ b/arch/arm64/include/asm/esr.h
> @@ -47,7 +47,7 @@
> #define ESR_ELx_EC_DABT_LOW (0x24)
> #define ESR_ELx_EC_DABT_CUR (0x25)
> #define ESR_ELx_EC_SP_ALIGN (0x26)
> -/* Unallocated EC: 0x27 */
> +#define ESR_ELx_EC_MOPS (0x27)
> #define ESR_ELx_EC_FP_EXC32 (0x28)
> /* Unallocated EC: 0x29 - 0x2B */
> #define ESR_ELx_EC_FP_EXC64 (0x2C)
> @@ -352,6 +352,15 @@
> #define ESR_ELx_SME_ISS_ZA_DISABLED 3
> #define ESR_ELx_SME_ISS_ZT_DISABLED 4
>
> +/* ISS field definitions for MOPS exceptions */
> +#define ESR_ELx_MOPS_ISS_MEM_INST (UL(1) << 24)
> +#define ESR_ELx_MOPS_ISS_FROM_EPILOGUE (UL(1) << 18)
> +#define ESR_ELx_MOPS_ISS_WRONG_OPTION (UL(1) << 17)
> +#define ESR_ELx_MOPS_ISS_OPTION_A (UL(1) << 16)
> +#define ESR_ELx_MOPS_ISS_DESTREG(esr) (((esr) & (UL(0x1f) << 10)) >> 10)
> +#define ESR_ELx_MOPS_ISS_SRCREG(esr) (((esr) & (UL(0x1f) << 5)) >> 5)
> +#define ESR_ELx_MOPS_ISS_SIZEREG(esr) (((esr) & (UL(0x1f) << 0)) >> 0)
> +
> #ifndef __ASSEMBLY__
> #include <asm/types.h>
>
> diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h
> index 92963f98afec..5a6dc3643e9b 100644
> --- a/arch/arm64/include/asm/exception.h
> +++ b/arch/arm64/include/asm/exception.h
> @@ -77,6 +77,7 @@ void do_el0_svc(struct pt_regs *regs);
> void do_el0_svc_compat(struct pt_regs *regs);
> void do_el0_fpac(struct pt_regs *regs, unsigned long esr);
> void do_el1_fpac(struct pt_regs *regs, unsigned long esr);
> +void do_el0_mops(struct pt_regs *regs, unsigned long esr);
> void do_serror(struct pt_regs *regs, unsigned long esr);
> void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags);
>
> diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
> index cce1167199e3..2ef3ab5d7555 100644
> --- a/arch/arm64/kernel/entry-common.c
> +++ b/arch/arm64/kernel/entry-common.c
> @@ -611,6 +611,14 @@ static void noinstr el0_bti(struct pt_regs *regs)
> exit_to_user_mode(regs);
> }
>
> +static void noinstr el0_mops(struct pt_regs *regs, unsigned long esr)
> +{
> + enter_from_user_mode(regs);
> + local_daif_restore(DAIF_PROCCTX);
> + do_el0_mops(regs, esr);
> + exit_to_user_mode(regs);
> +}
> +
> static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr)
> {
> enter_from_user_mode(regs);
> @@ -688,6 +696,9 @@ asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)
> case ESR_ELx_EC_BTI:
> el0_bti(regs);
> break;
> + case ESR_ELx_EC_MOPS:
> + el0_mops(regs, esr);
> + break;
> case ESR_ELx_EC_BREAKPT_LOW:
> case ESR_ELx_EC_SOFTSTP_LOW:
> case ESR_ELx_EC_WATCHPT_LOW:
> diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
> index 0ccc063daccb..689188712909 100644
> --- a/arch/arm64/kernel/traps.c
> +++ b/arch/arm64/kernel/traps.c
> @@ -507,6 +507,50 @@ void do_el1_fpac(struct pt_regs *regs, unsigned long esr)
> die("Oops - FPAC", regs, esr);
> }
>
> +void do_el0_mops(struct pt_regs *regs, unsigned long esr)
> +{
> + bool wrong_option = esr & ESR_ELx_MOPS_ISS_WRONG_OPTION;
> + bool option_a = esr & ESR_ELx_MOPS_ISS_OPTION_A;
> + int dstreg = ESR_ELx_MOPS_ISS_DESTREG(esr);
> + int srcreg = ESR_ELx_MOPS_ISS_SRCREG(esr);
> + int sizereg = ESR_ELx_MOPS_ISS_SIZEREG(esr);
> + unsigned long dst, src, size;
> +
> + dst = pt_regs_read_reg(regs, dstreg);
> + src = pt_regs_read_reg(regs, srcreg);
> + size = pt_regs_read_reg(regs, sizereg);
> +
> + /*
> + * Put the registers back in the original format suitable for a
> + * prologue instruction, using the generic return routine from the
> + * Arm ARM (DDI 0487I.a) rules CNTMJ and MWFQH.
> + */
> + if (esr & ESR_ELx_MOPS_ISS_MEM_INST) {
> + if ((!option_a && wrong_option) || (option_a && !wrong_option)) {
> + pt_regs_write_reg(regs, dstreg, dst + size);
> + pt_regs_write_reg(regs, sizereg, -size);
> + }
Please add a comment here that this is for the SET* instructions (rule
MWFQH). It confused me a bit when trying to review against the Arm ARM.
I'd also add the comments from the spec like "format is Option A" and
"forward copy".
> + } else {
> + if ((option_a && wrong_option) || (!option_a && !wrong_option)) {
> + if (regs->pstate & PSR_N_BIT) {
> + pt_regs_write_reg(regs, dstreg, dst - size);
> + pt_regs_write_reg(regs, srcreg, src - size);
> + }
> + } else {
> + if (size & BIT(63)) {
> + pt_regs_write_reg(regs, dstreg, dst + size);
> + pt_regs_write_reg(regs, srcreg, src + size);
> + pt_regs_write_reg(regs, sizereg, -size);
> + }
> + }
> + }
> +
> + if (esr & ESR_ELx_MOPS_ISS_FROM_EPILOGUE)
> + regs->pc -= 8;
> + else
> + regs->pc -= 4;
> +}
Same here about the comments in the Arm ARM, copy them over here as
well.
I think rule CNTMJ has a typo with the indentation as the return
address seems to only be updated on the second 'else' block above.
Otherwise the code looks fine.
Reviewed-by: Catalin Marinas <[email protected]>
On Thu, Feb 16, 2023 at 04:00:10PM +0000, Kristina Martsenko wrote:
> When a MOPS main or epilogue instruction is being executed, the task may
> get scheduled on a different CPU and restart execution from the prologue
> instruction. If the main or epilogue instruction is being single stepped
> then it makes sense to finish the step and take the step exception
> before starting to execute the next (prologue) instruction. So
> fast-forward the single step state machine when taking a MOPS exception.
>
> This means that if a main or epilogue instruction is single stepped with
> ptrace, the debugger will sometimes observe the PC moving back to the
> prologue instruction. (As already mentioned, this should be rare as it
> only happens when the task is scheduled to another CPU during the step.)
>
> This also ensures that perf breakpoints count prologue instructions
> consistently (i.e. every time they are executed), rather than skipping
> them when there also happens to be a breakpoint on a main or epilogue
> instruction.
>
> Signed-off-by: Kristina Martsenko <[email protected]>
Acked-by: Catalin Marinas <[email protected]>
On Thu, Feb 16, 2023 at 04:00:11PM +0000, Kristina Martsenko wrote:
> The Arm v8.8/9.3 FEAT_MOPS feature provides new instructions that
> perform a memory copy or set. Wire up the cpufeature code to detect the
> presence of FEAT_MOPS and enable it.
>
> Signed-off-by: Kristina Martsenko <[email protected]>
Reviewed-by: Catalin Marinas <[email protected]>
On Thu, Feb 16, 2023 at 04:00:12PM +0000, Kristina Martsenko wrote:
> Make it possible to disable the MOPS extension at runtime using the
> kernel command line. This can be useful for testing or working around
> hardware issues.
>
> Signed-off-by: Kristina Martsenko <[email protected]>
Yeah, I think that's useful if we want to test new cortext strings
routines for example.
Reviewed-by: Catalin Marinas <[email protected]>
On 17/03/2023 15:07, Catalin Marinas wrote:
> On Thu, Feb 16, 2023 at 04:00:06PM +0000, Kristina Martsenko wrote:
>> + For CPUs with Memory Copy and Memory Set instructions (FEAT_MOPS):
>> +
>> + - If the kernel is entered at EL1 and EL2 is present:
>> +
>> + - HCRX_EL2.MSCEn (bit 11) must be initialised to 0b1.
>> +
>> + - HCRX_EL2.MCE2 (bit 10) must be initialised to 0b0.
>
> Regarding MCE2, does EL1 actually care if EL2 wants to handle all the
> memcpy/memset exceptions?
No, EL1 does not need to handle the exceptions itself, but I don't see any
current use case for allowing EL2 to handle it.
If it was allowed, I think booting.txt would need to specify exactly what Linux
expects EL2 to do if MCE2 is set (eg, that EL2 handles the exception by
reformatting registers, modifying single step state, etc).
> There may even be a valid case to do this at
> EL2 if you run a guest that uses these instructions but has no clue on
> how to deal with the specific exception like WrongOption.
Not sure I follow - this series adds the exception handling, so how can a Linux
guest not know how to handle the exception?
Or do you mean that there may be times when EL1 can't take the exception but
EL2 may move it to another CPU, and so EL2 would need to handle the exception?
I'm not sure if Linux ever uses mops instructions at times like that. Note that
this series does not add support for mops in guests yet. I think booting.txt
can be updated when that support is added.
Thanks,
Kristina
On Fri, Mar 24, 2023 at 01:00:43AM +0000, Kristina Martsenko wrote:
> On 17/03/2023 15:07, Catalin Marinas wrote:
> > On Thu, Feb 16, 2023 at 04:00:06PM +0000, Kristina Martsenko wrote:
> >> + For CPUs with Memory Copy and Memory Set instructions (FEAT_MOPS):
> >> +
> >> + - If the kernel is entered at EL1 and EL2 is present:
> >> +
> >> + - HCRX_EL2.MSCEn (bit 11) must be initialised to 0b1.
> >> +
> >> + - HCRX_EL2.MCE2 (bit 10) must be initialised to 0b0.
> >
> > Regarding MCE2, does EL1 actually care if EL2 wants to handle all the
> > memcpy/memset exceptions?
>
> No, EL1 does not need to handle the exceptions itself, but I don't see any
> current use case for allowing EL2 to handle it.
>
> If it was allowed, I think booting.txt would need to specify exactly what Linux
> expects EL2 to do if MCE2 is set (eg, that EL2 handles the exception by
> reformatting registers, modifying single step state, etc).
What I meant is that an EL1 kernel shouldn't care if MCE2 is 0 or 1. We
could clarify that if set to 1, it is expected that the hypervisor
handles the memory copy/set exceptions accordingly.
> > There may even be a valid case to do this at
> > EL2 if you run a guest that uses these instructions but has no clue on
> > how to deal with the specific exception like WrongOption.
>
> Not sure I follow - this series adds the exception handling, so how can a Linux
> guest not know how to handle the exception?
The guest may not always be Linux (e.g. some microkernel) and may not
expect the hardware implementation to change underneath.
> Or do you mean that there may be times when EL1 can't take the exception but
> EL2 may move it to another CPU, and so EL2 would need to handle the exception?
I haven't thought of this but it's actually a good point. Are there any
cases where Linux can't handle a memcpy exception? I guess we need to be
careful with taking such exception in an atomic context (e.g. no
rescheduling on the return path).
> I'm not sure if Linux ever uses mops instructions at times like that.
The compiler may generate a memcpy() call by simply assigning a
structure to another. So we can't control where those instructions are
used.
> Note that this series does not add support for mops in guests yet.
You mean there's no KVM support. But Linux may be run under a different
hypervisor (e.g. Hyper-V) as a guest.
> I think booting.txt can be updated when that support is added.
In booting.txt, when you say the kernel entered at EL1, it implies that
it may be run as a guest under a random hypervisor.
So maybe we should detail the MCE2 requirement a bit, saying that it can
be either 0 or 1 but, for the latter, the hypervisor must handle the
corresponding exceptions.
--
Catalin
On 04/04/2023 11:50, Catalin Marinas wrote:
> On Fri, Mar 24, 2023 at 01:00:43AM +0000, Kristina Martsenko wrote:
>> On 17/03/2023 15:07, Catalin Marinas wrote:
>>> On Thu, Feb 16, 2023 at 04:00:06PM +0000, Kristina Martsenko wrote:
>>>> + For CPUs with Memory Copy and Memory Set instructions (FEAT_MOPS):
>>>> +
>>>> + - If the kernel is entered at EL1 and EL2 is present:
>>>> +
>>>> + - HCRX_EL2.MSCEn (bit 11) must be initialised to 0b1.
>>>> +
>>>> + - HCRX_EL2.MCE2 (bit 10) must be initialised to 0b0.
>>>
>>> Regarding MCE2, does EL1 actually care if EL2 wants to handle all the
>>> memcpy/memset exceptions?
>>
>> Note that this series does not add support for mops in guests yet.
>
> You mean there's no KVM support. But Linux may be run under a different
> hypervisor (e.g. Hyper-V) as a guest.
>
>> I think booting.txt can be updated when that support is added.
>
> In booting.txt, when you say the kernel entered at EL1, it implies that
> it may be run as a guest under a random hypervisor.
>
> So maybe we should detail the MCE2 requirement a bit, saying that it can
> be either 0 or 1 but, for the latter, the hypervisor must handle the
> corresponding exceptions.
That makes sense. I was going to add this, but then realized that MCE2 only
traps memcpy exceptions from EL1, not EL0, so it does not have any effect with
this series. So I think we can just drop the MCE2 requirement entirely for now
and specify the requirement later when we add memcpy instructions into the
kernel itself.
Thanks,
Kristina