With the increase of memory capacity and density, the probability of
memory error increases. The increasing size and density of server RAM
in the data center and cloud have shown increased uncorrectable memory
errors.
Currently, the kernel has a mechanism to recover from hardware memory
errors. This patchset provides an new recovery mechanism.
For arm64, the hardware memory error handling is do_sea() which divided
into two cases:
1. The user state consumed the memory errors, the solution is kill the
user process and isolate the error page.
2. The kernel state consumed the memory errors, the solution is panic.
For case 2, Undifferentiated panic maybe not the optimal choice, it can be
handled better, in some scenarios, we can avoid panic, such as uaccess, if the
uaccess fails due to memory error, only the user process will be affected,
kill the user process and isolate the user page with hardware memory errors
is a better choice.
This patchset can be divided into two parts:
1. Patch 1~4 - do some restructuring to arm64 extable.
2. Patch 5~8 - arm64 add support for machine check safe framework,
then ,add two specific scenarios(uaccess/cow) to machine check safe.
Since V4:
1. According Michael's suggestion, add patch5.
2. According Mark's suggestiog, do some restructuring to arm64
extable, then a new adaptation of machine check safe support is made based
on this.
3. According Mark's suggestion, support machine check safe in do_mte() in
cow scene.
4. In V4, two patches have been merged into -next, so V5 not send these
two patches.
Since V3:
1. According to Robin's suggestion, direct modify user_ldst and
user_ldp in asm-uaccess.h and modify mte.S.
2. Add new macro USER_MC in asm-uaccess.h, used in copy_from_user.S
and copy_to_user.S.
3. According to Robin's suggestion, using micro in copy_page_mc.S to
simplify code.
4. According to KeFeng's suggestion, modify powerpc code in patch1.
5. According to KeFeng's suggestion, modify mm/extable.c and some code
optimization.
Since V2:
1. According to Mark's suggestion, all uaccess can be recovered due to
memory error.
2. Scenario pagecache reading is also supported as part of uaccess
(copy_to_user()) and duplication code problem is also solved.
Thanks for Robin's suggestion.
3. According Mark's suggestion, update commit message of patch 2/5.
4. According Borisllav's suggestion, update commit message of patch 1/5.
Since V1:
1.Consistent with PPC/x86, Using CONFIG_ARCH_HAS_COPY_MC instead of
ARM64_UCE_KERNEL_RECOVERY.
2.Add two new scenes, cow and pagecache reading.
3.Fix two small bug(the first two patch).
V1 in here:
https://lore.kernel.org/lkml/[email protected]/
Tong Tiangen (8):
arm64: extable: add new extable type EX_TYPE_KACCESS_ERR_ZERO support
arm64: extable: make uaaccess helper use extable type
EX_TYPE_UACCESS_ERR_ZERO
arm64: extable: move _cond_extable to _cond_uaccess_extable
arm64: extable: cleanup redundant extable type EX_TYPE_FIXUP
Add generic fallback version of copy_mc_to_user()
arm64: add support for machine check error safe
arm64: add uaccess to machine check safe
arm64: add cow to machine check safe
arch/arm64/Kconfig | 1 +
arch/arm64/include/asm/asm-extable.h | 45 +++++++++----
arch/arm64/include/asm/asm-uaccess.h | 12 ++--
arch/arm64/include/asm/assembler.h | 4 +-
arch/arm64/include/asm/extable.h | 1 +
arch/arm64/include/asm/mte.h | 4 ++
arch/arm64/include/asm/page.h | 10 +++
arch/arm64/include/asm/uaccess.h | 94 ++++++++++++++--------------
arch/arm64/lib/Makefile | 2 +
arch/arm64/lib/copy_from_user.S | 8 +--
arch/arm64/lib/copy_page_mc.S | 82 ++++++++++++++++++++++++
arch/arm64/lib/copy_to_user.S | 8 +--
arch/arm64/lib/mte.S | 19 ++++++
arch/arm64/mm/copypage.c | 41 +++++++++---
arch/arm64/mm/extable.c | 22 ++++++-
arch/arm64/mm/fault.c | 27 +++++++-
arch/powerpc/include/asm/uaccess.h | 1 +
arch/x86/include/asm/uaccess.h | 1 +
include/linux/highmem.h | 8 +++
include/linux/uaccess.h | 9 +++
mm/memory.c | 2 +-
21 files changed, 314 insertions(+), 87 deletions(-)
create mode 100644 arch/arm64/lib/copy_page_mc.S
--
2.25.1
Currently, extable type EX_TYPE_FIXUP is no place to use, We can safely
remove it.
Suggested-by: Mark Rutland <[email protected]>
Signed-off-by: Tong Tiangen <[email protected]>
---
arch/arm64/include/asm/asm-extable.h | 20 ++++----------------
arch/arm64/mm/extable.c | 9 ---------
2 files changed, 4 insertions(+), 25 deletions(-)
diff --git a/arch/arm64/include/asm/asm-extable.h b/arch/arm64/include/asm/asm-extable.h
index d01bd94cc4c2..1f2974467273 100644
--- a/arch/arm64/include/asm/asm-extable.h
+++ b/arch/arm64/include/asm/asm-extable.h
@@ -3,11 +3,10 @@
#define __ASM_ASM_EXTABLE_H
#define EX_TYPE_NONE 0
-#define EX_TYPE_FIXUP 1
-#define EX_TYPE_BPF 2
-#define EX_TYPE_UACCESS_ERR_ZERO 3
-#define EX_TYPE_LOAD_UNALIGNED_ZEROPAD 4
-#define EX_TYPE_KACCESS_ERR_ZERO 5
+#define EX_TYPE_BPF 1
+#define EX_TYPE_UACCESS_ERR_ZERO 2
+#define EX_TYPE_LOAD_UNALIGNED_ZEROPAD 3
+#define EX_TYPE_KACCESS_ERR_ZERO 4
#ifdef __ASSEMBLY__
@@ -20,14 +19,6 @@
.short (data); \
.popsection;
-/*
- * Create an exception table entry for `insn`, which will branch to `fixup`
- * when an unhandled fault is taken.
- */
- .macro _asm_extable, insn, fixup
- __ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_FIXUP, 0)
- .endm
-
/*
* Create an exception table entry for uaccess `insn`, which will branch to `fixup`
* when an unhandled fault is taken.
@@ -62,9 +53,6 @@
".short (" data ")\n" \
".popsection\n"
-#define _ASM_EXTABLE(insn, fixup) \
- __ASM_EXTABLE_RAW(#insn, #fixup, __stringify(EX_TYPE_FIXUP), "0")
-
#define EX_DATA_REG_ERR_SHIFT 0
#define EX_DATA_REG_ERR GENMASK(4, 0)
#define EX_DATA_REG_ZERO_SHIFT 5
diff --git a/arch/arm64/mm/extable.c b/arch/arm64/mm/extable.c
index 056591e5ca80..228d681a8715 100644
--- a/arch/arm64/mm/extable.c
+++ b/arch/arm64/mm/extable.c
@@ -16,13 +16,6 @@ get_ex_fixup(const struct exception_table_entry *ex)
return ((unsigned long)&ex->fixup + ex->fixup);
}
-static bool ex_handler_fixup(const struct exception_table_entry *ex,
- struct pt_regs *regs)
-{
- regs->pc = get_ex_fixup(ex);
- return true;
-}
-
static bool ex_handler_uaccess_err_zero(const struct exception_table_entry *ex,
struct pt_regs *regs)
{
@@ -72,8 +65,6 @@ bool fixup_exception(struct pt_regs *regs)
return false;
switch (ex->type) {
- case EX_TYPE_FIXUP:
- return ex_handler_fixup(ex, regs);
case EX_TYPE_BPF:
return ex_handler_bpf(ex, regs);
case EX_TYPE_UACCESS_ERR_ZERO:
--
2.25.1
x86/powerpc has it's implementation of copy_mc_to_user(), we add generic
fallback in include/linux/uaccess.h prepare for other architechures to
enable CONFIG_ARCH_HAS_COPY_MC.
Signed-off-by: Tong Tiangen <[email protected]>
Acked-by: Michael Ellerman <[email protected]>
---
arch/powerpc/include/asm/uaccess.h | 1 +
arch/x86/include/asm/uaccess.h | 1 +
include/linux/uaccess.h | 9 +++++++++
3 files changed, 11 insertions(+)
diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h
index 9b82b38ff867..58dbe8e2e318 100644
--- a/arch/powerpc/include/asm/uaccess.h
+++ b/arch/powerpc/include/asm/uaccess.h
@@ -358,6 +358,7 @@ copy_mc_to_user(void __user *to, const void *from, unsigned long n)
return n;
}
+#define copy_mc_to_user copy_mc_to_user
#endif
extern long __copy_from_user_flushcache(void *dst, const void __user *src,
diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
index 35f222aa66bf..b7b1aca5d6cd 100644
--- a/arch/x86/include/asm/uaccess.h
+++ b/arch/x86/include/asm/uaccess.h
@@ -512,6 +512,7 @@ copy_mc_to_kernel(void *to, const void *from, unsigned len);
unsigned long __must_check
copy_mc_to_user(void *to, const void *from, unsigned len);
+#define copy_mc_to_user copy_mc_to_user
#endif
/*
diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h
index 5a328cf02b75..07e9faeb14b5 100644
--- a/include/linux/uaccess.h
+++ b/include/linux/uaccess.h
@@ -174,6 +174,15 @@ copy_mc_to_kernel(void *dst, const void *src, size_t cnt)
}
#endif
+#ifndef copy_mc_to_user
+static inline unsigned long __must_check
+copy_mc_to_user(void *dst, const void *src, size_t cnt)
+{
+ check_object_size(src, cnt, true);
+ return raw_copy_to_user(dst, src, cnt);
+}
+#endif
+
static __always_inline void pagefault_disabled_inc(void)
{
current->pagefault_disabled++;
--
2.25.1
On Sat, May 28, 2022 at 06:50:52AM +0000, Tong Tiangen wrote:
> Currently, extable type EX_TYPE_FIXUP is no place to use, We can safely
> remove it.
>
> Suggested-by: Mark Rutland <[email protected]>
> Signed-off-by: Tong Tiangen <[email protected]>
Acked-by: Mark Rutland <[email protected]>
Mark.
> ---
> arch/arm64/include/asm/asm-extable.h | 20 ++++----------------
> arch/arm64/mm/extable.c | 9 ---------
> 2 files changed, 4 insertions(+), 25 deletions(-)
>
> diff --git a/arch/arm64/include/asm/asm-extable.h b/arch/arm64/include/asm/asm-extable.h
> index d01bd94cc4c2..1f2974467273 100644
> --- a/arch/arm64/include/asm/asm-extable.h
> +++ b/arch/arm64/include/asm/asm-extable.h
> @@ -3,11 +3,10 @@
> #define __ASM_ASM_EXTABLE_H
>
> #define EX_TYPE_NONE 0
> -#define EX_TYPE_FIXUP 1
> -#define EX_TYPE_BPF 2
> -#define EX_TYPE_UACCESS_ERR_ZERO 3
> -#define EX_TYPE_LOAD_UNALIGNED_ZEROPAD 4
> -#define EX_TYPE_KACCESS_ERR_ZERO 5
> +#define EX_TYPE_BPF 1
> +#define EX_TYPE_UACCESS_ERR_ZERO 2
> +#define EX_TYPE_LOAD_UNALIGNED_ZEROPAD 3
> +#define EX_TYPE_KACCESS_ERR_ZERO 4
>
> #ifdef __ASSEMBLY__
>
> @@ -20,14 +19,6 @@
> .short (data); \
> .popsection;
>
> -/*
> - * Create an exception table entry for `insn`, which will branch to `fixup`
> - * when an unhandled fault is taken.
> - */
> - .macro _asm_extable, insn, fixup
> - __ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_FIXUP, 0)
> - .endm
> -
> /*
> * Create an exception table entry for uaccess `insn`, which will branch to `fixup`
> * when an unhandled fault is taken.
> @@ -62,9 +53,6 @@
> ".short (" data ")\n" \
> ".popsection\n"
>
> -#define _ASM_EXTABLE(insn, fixup) \
> - __ASM_EXTABLE_RAW(#insn, #fixup, __stringify(EX_TYPE_FIXUP), "0")
> -
> #define EX_DATA_REG_ERR_SHIFT 0
> #define EX_DATA_REG_ERR GENMASK(4, 0)
> #define EX_DATA_REG_ZERO_SHIFT 5
> diff --git a/arch/arm64/mm/extable.c b/arch/arm64/mm/extable.c
> index 056591e5ca80..228d681a8715 100644
> --- a/arch/arm64/mm/extable.c
> +++ b/arch/arm64/mm/extable.c
> @@ -16,13 +16,6 @@ get_ex_fixup(const struct exception_table_entry *ex)
> return ((unsigned long)&ex->fixup + ex->fixup);
> }
>
> -static bool ex_handler_fixup(const struct exception_table_entry *ex,
> - struct pt_regs *regs)
> -{
> - regs->pc = get_ex_fixup(ex);
> - return true;
> -}
> -
> static bool ex_handler_uaccess_err_zero(const struct exception_table_entry *ex,
> struct pt_regs *regs)
> {
> @@ -72,8 +65,6 @@ bool fixup_exception(struct pt_regs *regs)
> return false;
>
> switch (ex->type) {
> - case EX_TYPE_FIXUP:
> - return ex_handler_fixup(ex, regs);
> case EX_TYPE_BPF:
> return ex_handler_bpf(ex, regs);
> case EX_TYPE_UACCESS_ERR_ZERO:
> --
> 2.25.1
>