With the increase of memory capacity and density, the probability of
memory error increases. The increasing size and density of server RAM
in the data center and cloud have shown increased uncorrectable memory
errors.
Currently, the kernel has a mechanism to recover from hardware memory
errors. This patchset provides an new recovery mechanism.
For arm64, the hardware memory error handling is do_sea() which divided
into two cases:
1. The user state consumed the memory errors, the solution is kill the
user process and isolate the error page.
2. The kernel state consumed the memory errors, the solution is panic.
For case 2, Undifferentiated panic maybe not the optimal choice, it can be
handled better, in some scenes, we can avoid panic, such as uaccess, if the
uaccess fails due to memory error, only the user process will be affected,
kill the user process and isolate the user page with hardware memory errors
is a better choice.
This patchset can be divided into three parts:
1. Patch 0/1/4 - make some minor fixes to the associated code.
2. Patch 3 - arm64 add support for machine check safe framework.
3. Pathc 5/6/7 - arm64 add uaccess and cow to machine check safe.
Since V4:
1. According to Robin's suggestion, direct modify user_ldst and
user_ldp in asm-uaccess.h and modify mte.S.
2. Add new macro USER_MC in asm-uaccess.h, used in copy_from_user.S
and copy_to_user.S.
3. According to Robin's suggestion, using micro in copy_page_mc.S to
simplify code.
4. According to KeFeng's suggestion, modify powerpc code in patch1.
5. According to KeFeng's suggestion, modify mm/extable.c and some code
optimization.
Since V3:
1. According to Mark's suggestion, all uaccess can be recovered due to
memory error.
2. Scenario pagecache reading is also supported as part of uaccess
(copy_to_user()) and duplication code problem is also solved.
Thanks for Robin's suggestion.
3. According Mark's suggestion, update commit message of patch 2/5.
4. According Borisllav's suggestion, update commit message of patch 1/5.
Since V2:
1.Consistent with PPC/x86, Using CONFIG_ARCH_HAS_COPY_MC instead of
ARM64_UCE_KERNEL_RECOVERY.
2.Add two new scenes, cow and pagecache reading.
3.Fix two small bug(the first two patch).
V1 in here:
https://lore.kernel.org/lkml/[email protected]/
Robin Murphy (1):
arm64: mte: Clean up user tag accessors
Tong Tiangen (6):
x86, powerpc: fix function define in copy_mc_to_user
arm64: fix types in copy_highpage()
arm64: add support for machine check error safe
arm64: add copy_{to, from}_user to machine check safe
arm64: add {get, put}_user to machine check safe
arm64: add cow to machine check safe
arch/arm64/Kconfig | 1 +
arch/arm64/include/asm/asm-extable.h | 33 +++++++++++
arch/arm64/include/asm/asm-uaccess.h | 15 +++--
arch/arm64/include/asm/extable.h | 1 +
arch/arm64/include/asm/page.h | 10 ++++
arch/arm64/include/asm/uaccess.h | 4 +-
arch/arm64/lib/Makefile | 2 +
arch/arm64/lib/copy_from_user.S | 18 +++---
arch/arm64/lib/copy_page_mc.S | 86 ++++++++++++++++++++++++++++
arch/arm64/lib/copy_to_user.S | 18 +++---
arch/arm64/lib/mte.S | 4 +-
arch/arm64/mm/copypage.c | 36 ++++++++++--
arch/arm64/mm/extable.c | 33 +++++++++++
arch/arm64/mm/fault.c | 27 ++++++++-
arch/powerpc/include/asm/uaccess.h | 1 +
arch/x86/include/asm/uaccess.h | 1 +
include/linux/highmem.h | 8 +++
include/linux/uaccess.h | 9 +++
mm/memory.c | 2 +-
19 files changed, 278 insertions(+), 31 deletions(-)
create mode 100644 arch/arm64/lib/copy_page_mc.S
--
2.25.1
Add copy_{to, from}_user() to machine check safe.
If copy fail due to hardware memory error, only the relevant processes are
affected, so killing the user process and isolate the user page with
hardware memory errors is a more reasonable choice than kernel panic.
Add new extable type EX_TYPE_UACCESS_MC which can be used for uaccess that
can be recovered from hardware memory errors.
The x16 register is used to save the fixup type in copy_xxx_user which
used extable type EX_TYPE_UACCESS_MC.
Signed-off-by: Tong Tiangen <[email protected]>
---
arch/arm64/include/asm/asm-extable.h | 14 ++++++++++++++
arch/arm64/include/asm/asm-uaccess.h | 15 ++++++++++-----
arch/arm64/lib/copy_from_user.S | 18 +++++++++++-------
arch/arm64/lib/copy_to_user.S | 18 +++++++++++-------
arch/arm64/mm/extable.c | 18 ++++++++++++++----
5 files changed, 60 insertions(+), 23 deletions(-)
diff --git a/arch/arm64/include/asm/asm-extable.h b/arch/arm64/include/asm/asm-extable.h
index c39f2437e08e..75b2c00e9523 100644
--- a/arch/arm64/include/asm/asm-extable.h
+++ b/arch/arm64/include/asm/asm-extable.h
@@ -2,12 +2,18 @@
#ifndef __ASM_ASM_EXTABLE_H
#define __ASM_ASM_EXTABLE_H
+#define FIXUP_TYPE_NORMAL 0
+#define FIXUP_TYPE_MC 1
+
#define EX_TYPE_NONE 0
#define EX_TYPE_FIXUP 1
#define EX_TYPE_BPF 2
#define EX_TYPE_UACCESS_ERR_ZERO 3
#define EX_TYPE_LOAD_UNALIGNED_ZEROPAD 4
+/* _MC indicates that can fixup from machine check errors */
+#define EX_TYPE_UACCESS_MC 5
+
#ifdef __ASSEMBLY__
#define __ASM_EXTABLE_RAW(insn, fixup, type, data) \
@@ -27,6 +33,14 @@
__ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_FIXUP, 0)
.endm
+/*
+ * Create an exception table entry for `insn`, which will branch to `fixup`
+ * when an unhandled fault(include sea fault) is taken.
+ */
+ .macro _asm_extable_uaccess_mc, insn, fixup
+ __ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_UACCESS_MC, 0)
+ .endm
+
/*
* Create an exception table entry for `insn` if `fixup` is provided. Otherwise
* do nothing.
diff --git a/arch/arm64/include/asm/asm-uaccess.h b/arch/arm64/include/asm/asm-uaccess.h
index 0557af834e03..6c23c138e1fc 100644
--- a/arch/arm64/include/asm/asm-uaccess.h
+++ b/arch/arm64/include/asm/asm-uaccess.h
@@ -63,6 +63,11 @@ alternative_else_nop_endif
9999: x; \
_asm_extable 9999b, l
+
+#define USER_MC(l, x...) \
+9999: x; \
+ _asm_extable_uaccess_mc 9999b, l
+
/*
* Generate the assembly for LDTR/STTR with exception table entries.
* This is complicated as there is no post-increment or pair versions of the
@@ -73,8 +78,8 @@ alternative_else_nop_endif
8889: ldtr \reg2, [\addr, #8];
add \addr, \addr, \post_inc;
- _asm_extable 8888b,\l;
- _asm_extable 8889b,\l;
+ _asm_extable_uaccess_mc 8888b, \l;
+ _asm_extable_uaccess_mc 8889b, \l;
.endm
.macro user_stp l, reg1, reg2, addr, post_inc
@@ -82,14 +87,14 @@ alternative_else_nop_endif
8889: sttr \reg2, [\addr, #8];
add \addr, \addr, \post_inc;
- _asm_extable 8888b,\l;
- _asm_extable 8889b,\l;
+ _asm_extable_uaccess_mc 8888b,\l;
+ _asm_extable_uaccess_mc 8889b,\l;
.endm
.macro user_ldst l, inst, reg, addr, post_inc
8888: \inst \reg, [\addr];
add \addr, \addr, \post_inc;
- _asm_extable 8888b,\l;
+ _asm_extable_uaccess_mc 8888b, \l;
.endm
#endif
diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S
index 34e317907524..480cc5ac0a8d 100644
--- a/arch/arm64/lib/copy_from_user.S
+++ b/arch/arm64/lib/copy_from_user.S
@@ -25,7 +25,7 @@
.endm
.macro strb1 reg, ptr, val
- strb \reg, [\ptr], \val
+ USER_MC(9998f, strb \reg, [\ptr], \val)
.endm
.macro ldrh1 reg, ptr, val
@@ -33,7 +33,7 @@
.endm
.macro strh1 reg, ptr, val
- strh \reg, [\ptr], \val
+ USER_MC(9998f, strh \reg, [\ptr], \val)
.endm
.macro ldr1 reg, ptr, val
@@ -41,7 +41,7 @@
.endm
.macro str1 reg, ptr, val
- str \reg, [\ptr], \val
+ USER_MC(9998f, str \reg, [\ptr], \val)
.endm
.macro ldp1 reg1, reg2, ptr, val
@@ -49,11 +49,12 @@
.endm
.macro stp1 reg1, reg2, ptr, val
- stp \reg1, \reg2, [\ptr], \val
+ USER_MC(9998f, stp \reg1, \reg2, [\ptr], \val)
.endm
-end .req x5
-srcin .req x15
+end .req x5
+srcin .req x15
+fixup_type .req x16
SYM_FUNC_START(__arch_copy_from_user)
add end, x0, x2
mov srcin, x1
@@ -62,7 +63,10 @@ SYM_FUNC_START(__arch_copy_from_user)
ret
// Exception fixups
-9997: cmp dst, dstin
+ // x16: fixup type written by ex_handler_uaccess_mc
+9997: cmp fixup_type, #FIXUP_TYPE_MC
+ b.eq 9998f
+ cmp dst, dstin
b.ne 9998f
// Before being absolutely sure we couldn't copy anything, try harder
USER(9998f, ldtrb tmp1w, [srcin])
diff --git a/arch/arm64/lib/copy_to_user.S b/arch/arm64/lib/copy_to_user.S
index 802231772608..021a7d27b3a4 100644
--- a/arch/arm64/lib/copy_to_user.S
+++ b/arch/arm64/lib/copy_to_user.S
@@ -20,7 +20,7 @@
* x0 - bytes not copied
*/
.macro ldrb1 reg, ptr, val
- ldrb \reg, [\ptr], \val
+ USER_MC(9998f, ldrb \reg, [\ptr], \val)
.endm
.macro strb1 reg, ptr, val
@@ -28,7 +28,7 @@
.endm
.macro ldrh1 reg, ptr, val
- ldrh \reg, [\ptr], \val
+ USER_MC(9998f, ldrh \reg, [\ptr], \val)
.endm
.macro strh1 reg, ptr, val
@@ -36,7 +36,7 @@
.endm
.macro ldr1 reg, ptr, val
- ldr \reg, [\ptr], \val
+ USER_MC(9998f, ldr \reg, [\ptr], \val)
.endm
.macro str1 reg, ptr, val
@@ -44,15 +44,16 @@
.endm
.macro ldp1 reg1, reg2, ptr, val
- ldp \reg1, \reg2, [\ptr], \val
+ USER_MC(9998f, ldp \reg1, \reg2, [\ptr], \val)
.endm
.macro stp1 reg1, reg2, ptr, val
user_stp 9997f, \reg1, \reg2, \ptr, \val
.endm
-end .req x5
-srcin .req x15
+end .req x5
+srcin .req x15
+fixup_type .req x16
SYM_FUNC_START(__arch_copy_to_user)
add end, x0, x2
mov srcin, x1
@@ -61,7 +62,10 @@ SYM_FUNC_START(__arch_copy_to_user)
ret
// Exception fixups
-9997: cmp dst, dstin
+ // x16: fixup type written by ex_handler_uaccess_mc
+9997: cmp fixup_type, #FIXUP_TYPE_MC
+ b.eq 9998f
+ cmp dst, dstin
b.ne 9998f
// Before being absolutely sure we couldn't copy anything, try harder
ldrb tmp1w, [srcin]
diff --git a/arch/arm64/mm/extable.c b/arch/arm64/mm/extable.c
index 4f0083a550d4..525876c3ebf4 100644
--- a/arch/arm64/mm/extable.c
+++ b/arch/arm64/mm/extable.c
@@ -24,6 +24,14 @@ static bool ex_handler_fixup(const struct exception_table_entry *ex,
return true;
}
+static bool ex_handler_uaccess_type(const struct exception_table_entry *ex,
+ struct pt_regs *regs,
+ unsigned long fixup_type)
+{
+ regs->regs[16] = fixup_type;
+ return ex_handler_fixup(ex, regs);
+}
+
static bool ex_handler_uaccess_err_zero(const struct exception_table_entry *ex,
struct pt_regs *regs)
{
@@ -75,6 +83,8 @@ bool fixup_exception(struct pt_regs *regs)
switch (ex->type) {
case EX_TYPE_FIXUP:
return ex_handler_fixup(ex, regs);
+ case EX_TYPE_UACCESS_MC:
+ return ex_handler_uaccess_type(ex, regs, FIXUP_TYPE_NORMAL);
case EX_TYPE_BPF:
return ex_handler_bpf(ex, regs);
case EX_TYPE_UACCESS_ERR_ZERO:
@@ -94,10 +104,10 @@ bool fixup_exception_mc(struct pt_regs *regs)
if (!ex)
return false;
- /*
- * This is not complete, More Machine check safe extable type can
- * be processed here.
- */
+ switch (ex->type) {
+ case EX_TYPE_UACCESS_MC:
+ return ex_handler_uaccess_type(ex, regs, FIXUP_TYPE_MC);
+ }
return false;
}
--
2.25.1
From: Robin Murphy <[email protected]>
Invoking user_ldst to explicitly add a post-increment of 0 is silly.
Just use a normal USER() annotation and save the redundant instruction.
Signed-off-by: Robin Murphy <[email protected]>
Reviewed-by: Tong Tiangen <[email protected]>
---
arch/arm64/lib/mte.S | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/lib/mte.S b/arch/arm64/lib/mte.S
index 8590af3c98c0..eeb9e45bcce8 100644
--- a/arch/arm64/lib/mte.S
+++ b/arch/arm64/lib/mte.S
@@ -93,7 +93,7 @@ SYM_FUNC_START(mte_copy_tags_from_user)
mov x3, x1
cbz x2, 2f
1:
- user_ldst 2f, ldtrb, w4, x1, 0
+USER(2f, ldtrb w4, [x1])
lsl x4, x4, #MTE_TAG_SHIFT
stg x4, [x0], #MTE_GRANULE_SIZE
add x1, x1, #1
@@ -120,7 +120,7 @@ SYM_FUNC_START(mte_copy_tags_to_user)
1:
ldg x4, [x1]
ubfx x4, x4, #MTE_TAG_SHIFT, #MTE_TAG_SIZE
- user_ldst 2f, sttrb, w4, x0, 0
+USER(2f, sttrb w4, [x0])
add x0, x0, #1
add x1, x1, #MTE_GRANULE_SIZE
subs x2, x2, #1
--
2.25.1
Hi Mark, James, Robin, kindly ping...
Thanks.
在 2022/4/20 11:04, Tong Tiangen 写道:
> With the increase of memory capacity and density, the probability of
> memory error increases. The increasing size and density of server RAM
> in the data center and cloud have shown increased uncorrectable memory
> errors.
>
> Currently, the kernel has a mechanism to recover from hardware memory
> errors. This patchset provides an new recovery mechanism.
>
> For arm64, the hardware memory error handling is do_sea() which divided
> into two cases:
> 1. The user state consumed the memory errors, the solution is kill the
> user process and isolate the error page.
> 2. The kernel state consumed the memory errors, the solution is panic.
>
> For case 2, Undifferentiated panic maybe not the optimal choice, it can be
> handled better, in some scenes, we can avoid panic, such as uaccess, if the
> uaccess fails due to memory error, only the user process will be affected,
> kill the user process and isolate the user page with hardware memory errors
> is a better choice.
>
> This patchset can be divided into three parts:
> 1. Patch 0/1/4 - make some minor fixes to the associated code.
> 2. Patch 3 - arm64 add support for machine check safe framework.
> 3. Pathc 5/6/7 - arm64 add uaccess and cow to machine check safe.
>
> Since V4:
> 1. According to Robin's suggestion, direct modify user_ldst and
> user_ldp in asm-uaccess.h and modify mte.S.
> 2. Add new macro USER_MC in asm-uaccess.h, used in copy_from_user.S
> and copy_to_user.S.
> 3. According to Robin's suggestion, using micro in copy_page_mc.S to
> simplify code.
> 4. According to KeFeng's suggestion, modify powerpc code in patch1.
> 5. According to KeFeng's suggestion, modify mm/extable.c and some code
> optimization.
>
> Since V3:
> 1. According to Mark's suggestion, all uaccess can be recovered due to
> memory error.
> 2. Scenario pagecache reading is also supported as part of uaccess
> (copy_to_user()) and duplication code problem is also solved.
> Thanks for Robin's suggestion.
> 3. According Mark's suggestion, update commit message of patch 2/5.
> 4. According Borisllav's suggestion, update commit message of patch 1/5.
>
> Since V2:
> 1.Consistent with PPC/x86, Using CONFIG_ARCH_HAS_COPY_MC instead of
> ARM64_UCE_KERNEL_RECOVERY.
> 2.Add two new scenes, cow and pagecache reading.
> 3.Fix two small bug(the first two patch).
>
> V1 in here:
> https://lore.kernel.org/lkml/[email protected]/
>
> Robin Murphy (1):
> arm64: mte: Clean up user tag accessors
>
> Tong Tiangen (6):
> x86, powerpc: fix function define in copy_mc_to_user
> arm64: fix types in copy_highpage()
> arm64: add support for machine check error safe
> arm64: add copy_{to, from}_user to machine check safe
> arm64: add {get, put}_user to machine check safe
> arm64: add cow to machine check safe
>
> arch/arm64/Kconfig | 1 +
> arch/arm64/include/asm/asm-extable.h | 33 +++++++++++
> arch/arm64/include/asm/asm-uaccess.h | 15 +++--
> arch/arm64/include/asm/extable.h | 1 +
> arch/arm64/include/asm/page.h | 10 ++++
> arch/arm64/include/asm/uaccess.h | 4 +-
> arch/arm64/lib/Makefile | 2 +
> arch/arm64/lib/copy_from_user.S | 18 +++---
> arch/arm64/lib/copy_page_mc.S | 86 ++++++++++++++++++++++++++++
> arch/arm64/lib/copy_to_user.S | 18 +++---
> arch/arm64/lib/mte.S | 4 +-
> arch/arm64/mm/copypage.c | 36 ++++++++++--
> arch/arm64/mm/extable.c | 33 +++++++++++
> arch/arm64/mm/fault.c | 27 ++++++++-
> arch/powerpc/include/asm/uaccess.h | 1 +
> arch/x86/include/asm/uaccess.h | 1 +
> include/linux/highmem.h | 8 +++
> include/linux/uaccess.h | 9 +++
> mm/memory.c | 2 +-
> 19 files changed, 278 insertions(+), 31 deletions(-)
> create mode 100644 arch/arm64/lib/copy_page_mc.S
>
On Wed, Apr 20, 2022 at 03:04:15AM +0000, Tong Tiangen wrote:
> Add copy_{to, from}_user() to machine check safe.
>
> If copy fail due to hardware memory error, only the relevant processes are
> affected, so killing the user process and isolate the user page with
> hardware memory errors is a more reasonable choice than kernel panic.
Just to make sure I understand - we can only recover if the fault is in
a user page. That is, for a copy_from_user(), we can only handle the
faults in the source address, not the destination.
> diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S
> index 34e317907524..480cc5ac0a8d 100644
> --- a/arch/arm64/lib/copy_from_user.S
> +++ b/arch/arm64/lib/copy_from_user.S
> @@ -25,7 +25,7 @@
> .endm
>
> .macro strb1 reg, ptr, val
> - strb \reg, [\ptr], \val
> + USER_MC(9998f, strb \reg, [\ptr], \val)
> .endm
So if I got the above correctly, why do we need an exception table entry
for the store to the kernel address?
--
Catalin
在 2022/5/4 18:26, Catalin Marinas 写道:
> On Wed, Apr 20, 2022 at 03:04:15AM +0000, Tong Tiangen wrote:
>> Add copy_{to, from}_user() to machine check safe.
>>
>> If copy fail due to hardware memory error, only the relevant processes are
>> affected, so killing the user process and isolate the user page with
>> hardware memory errors is a more reasonable choice than kernel panic.
>
> Just to make sure I understand - we can only recover if the fault is in
> a user page. That is, for a copy_from_user(), we can only handle the
> faults in the source address, not the destination.
At the beginning, I also thought we can only recover if the fault is in
a user page.
After discussion with a Mark[1], I think no matter user page or kernel
page, as long as it is triggered by the user process, only related
processes will be affected. According to this
understanding, it seems that all uaccess can be recovered.
[1]https://patchwork.kernel.org/project/linux-arm-kernel/patch/[email protected]/
Thanks,
Tong.
>
>> diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S
>> index 34e317907524..480cc5ac0a8d 100644
>> --- a/arch/arm64/lib/copy_from_user.S
>> +++ b/arch/arm64/lib/copy_from_user.S
>> @@ -25,7 +25,7 @@
>> .endm
>>
>> .macro strb1 reg, ptr, val
>> - strb \reg, [\ptr], \val
>> + USER_MC(9998f, strb \reg, [\ptr], \val)
>> .endm
>
> So if I got the above correctly, why do we need an exception table entry
> for the store to the kernel address?
>
On Wed, 20 Apr 2022 03:04:11 +0000, Tong Tiangen wrote:
> With the increase of memory capacity and density, the probability of
> memory error increases. The increasing size and density of server RAM
> in the data center and cloud have shown increased uncorrectable memory
> errors.
>
> Currently, the kernel has a mechanism to recover from hardware memory
> errors. This patchset provides an new recovery mechanism.
>
> [...]
Applied to arm64 (for-next/misc), thanks!
[2/7] arm64: fix types in copy_highpage()
https://git.kernel.org/arm64/c/921d161f15d6
--
Catalin
在 2022/5/5 21:41, Catalin Marinas 写道:
> On Thu, May 05, 2022 at 02:39:43PM +0800, Tong Tiangen wrote:
>> 在 2022/5/4 18:26, Catalin Marinas 写道:
>>> On Wed, Apr 20, 2022 at 03:04:15AM +0000, Tong Tiangen wrote:
>>>> Add copy_{to, from}_user() to machine check safe.
>>>>
>>>> If copy fail due to hardware memory error, only the relevant processes are
>>>> affected, so killing the user process and isolate the user page with
>>>> hardware memory errors is a more reasonable choice than kernel panic.
>>>
>>> Just to make sure I understand - we can only recover if the fault is in
>>> a user page. That is, for a copy_from_user(), we can only handle the
>>> faults in the source address, not the destination.
>>
>> At the beginning, I also thought we can only recover if the fault is in a
>> user page.
>> After discussion with a Mark[1], I think no matter user page or kernel page,
>> as long as it is triggered by the user process, only related processes will
>> be affected. According to this
>> understanding, it seems that all uaccess can be recovered.
>>
>> [1]https://patchwork.kernel.org/project/linux-arm-kernel/patch/[email protected]/
>
> We can indeed safely skip this copy and return an error just like
> pretending there was a user page fault. However, my point was more
> around the "isolate the user page with hardware memory errors". If the
> fault is on a kernel address, there's not much you can do about. You'll
> likely trigger it later when you try to access that address (maybe it
> was freed and re-allocated). Do we hope we won't get the same error
> again on that kernel address?
I think the page with memory error will be isolated by memory_failure(),
generally, isolation will succeed, if isolate failure(we need to find
out why), then maybe the same error will trigger it later.
Thanks.
>
On Thu, May 05, 2022 at 02:39:43PM +0800, Tong Tiangen wrote:
> 在 2022/5/4 18:26, Catalin Marinas 写道:
> > On Wed, Apr 20, 2022 at 03:04:15AM +0000, Tong Tiangen wrote:
> > > Add copy_{to, from}_user() to machine check safe.
> > >
> > > If copy fail due to hardware memory error, only the relevant processes are
> > > affected, so killing the user process and isolate the user page with
> > > hardware memory errors is a more reasonable choice than kernel panic.
> >
> > Just to make sure I understand - we can only recover if the fault is in
> > a user page. That is, for a copy_from_user(), we can only handle the
> > faults in the source address, not the destination.
>
> At the beginning, I also thought we can only recover if the fault is in a
> user page.
> After discussion with a Mark[1], I think no matter user page or kernel page,
> as long as it is triggered by the user process, only related processes will
> be affected. According to this
> understanding, it seems that all uaccess can be recovered.
>
> [1]https://patchwork.kernel.org/project/linux-arm-kernel/patch/[email protected]/
We can indeed safely skip this copy and return an error just like
pretending there was a user page fault. However, my point was more
around the "isolate the user page with hardware memory errors". If the
fault is on a kernel address, there's not much you can do about. You'll
likely trigger it later when you try to access that address (maybe it
was freed and re-allocated). Do we hope we won't get the same error
again on that kernel address?
--
Catalin
On Wed, Apr 20, 2022 at 03:04:16AM +0000, Tong Tiangen wrote:
> From: Robin Murphy <[email protected]>
>
> Invoking user_ldst to explicitly add a post-increment of 0 is silly.
> Just use a normal USER() annotation and save the redundant instruction.
>
> Signed-off-by: Robin Murphy <[email protected]>
> Reviewed-by: Tong Tiangen <[email protected]>
When posting someone else's patch, you need to add your own
Signed-off-by tag. Please see:
https://www.kernel.org/doc/html/latest/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin
That said, the patch itself looks sane, and matches its original posting
at:
https://lore.kernel.org/linux-arm-kernel/[email protected]/
So:
Acked-by: Mark Rutland <[email protected]>
Catalin, are you happy to pick up this patch as a cleanup?
Thanks,
Mark.
> ---
> arch/arm64/lib/mte.S | 4 ++--
> 1 file changed, 2 insertions(+), 2 deletions(-)
>
> diff --git a/arch/arm64/lib/mte.S b/arch/arm64/lib/mte.S
> index 8590af3c98c0..eeb9e45bcce8 100644
> --- a/arch/arm64/lib/mte.S
> +++ b/arch/arm64/lib/mte.S
> @@ -93,7 +93,7 @@ SYM_FUNC_START(mte_copy_tags_from_user)
> mov x3, x1
> cbz x2, 2f
> 1:
> - user_ldst 2f, ldtrb, w4, x1, 0
> +USER(2f, ldtrb w4, [x1])
> lsl x4, x4, #MTE_TAG_SHIFT
> stg x4, [x0], #MTE_GRANULE_SIZE
> add x1, x1, #1
> @@ -120,7 +120,7 @@ SYM_FUNC_START(mte_copy_tags_to_user)
> 1:
> ldg x4, [x1]
> ubfx x4, x4, #MTE_TAG_SHIFT, #MTE_TAG_SIZE
> - user_ldst 2f, sttrb, w4, x0, 0
> +USER(2f, sttrb w4, [x0])
> add x0, x0, #1
> add x1, x1, #MTE_GRANULE_SIZE
> subs x2, x2, #1
> --
> 2.25.1
>
On Wed, Apr 20, 2022 at 03:04:15AM +0000, Tong Tiangen wrote:
> Add copy_{to, from}_user() to machine check safe.
>
> If copy fail due to hardware memory error, only the relevant processes are
> affected, so killing the user process and isolate the user page with
> hardware memory errors is a more reasonable choice than kernel panic.
>
> Add new extable type EX_TYPE_UACCESS_MC which can be used for uaccess that
> can be recovered from hardware memory errors.
I don't understand why we need this.
If we apply EX_TYPE_UACCESS consistently to *all* user accesses, and
*only* to user accesses, that would *always* indicate that we can
recover, and that seems much simpler to deal with.
Today we use EX_TYPE_UACCESS_ERR_ZERO for kernel accesses in a couple of
cases, which we should clean up, and we user EX_TYPE_FIXUP for a couple
of user accesses, but those could easily be converted over.
> The x16 register is used to save the fixup type in copy_xxx_user which
> used extable type EX_TYPE_UACCESS_MC.
Why x16?
How is this intended to be consumed, and why is that behaviour different
from any *other* fault?
Mark.
> Signed-off-by: Tong Tiangen <[email protected]>
> ---
> arch/arm64/include/asm/asm-extable.h | 14 ++++++++++++++
> arch/arm64/include/asm/asm-uaccess.h | 15 ++++++++++-----
> arch/arm64/lib/copy_from_user.S | 18 +++++++++++-------
> arch/arm64/lib/copy_to_user.S | 18 +++++++++++-------
> arch/arm64/mm/extable.c | 18 ++++++++++++++----
> 5 files changed, 60 insertions(+), 23 deletions(-)
>
> diff --git a/arch/arm64/include/asm/asm-extable.h b/arch/arm64/include/asm/asm-extable.h
> index c39f2437e08e..75b2c00e9523 100644
> --- a/arch/arm64/include/asm/asm-extable.h
> +++ b/arch/arm64/include/asm/asm-extable.h
> @@ -2,12 +2,18 @@
> #ifndef __ASM_ASM_EXTABLE_H
> #define __ASM_ASM_EXTABLE_H
>
> +#define FIXUP_TYPE_NORMAL 0
> +#define FIXUP_TYPE_MC 1
> +
> #define EX_TYPE_NONE 0
> #define EX_TYPE_FIXUP 1
> #define EX_TYPE_BPF 2
> #define EX_TYPE_UACCESS_ERR_ZERO 3
> #define EX_TYPE_LOAD_UNALIGNED_ZEROPAD 4
>
> +/* _MC indicates that can fixup from machine check errors */
> +#define EX_TYPE_UACCESS_MC 5
> +
> #ifdef __ASSEMBLY__
>
> #define __ASM_EXTABLE_RAW(insn, fixup, type, data) \
> @@ -27,6 +33,14 @@
> __ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_FIXUP, 0)
> .endm
>
> +/*
> + * Create an exception table entry for `insn`, which will branch to `fixup`
> + * when an unhandled fault(include sea fault) is taken.
> + */
> + .macro _asm_extable_uaccess_mc, insn, fixup
> + __ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_UACCESS_MC, 0)
> + .endm
> +
> /*
> * Create an exception table entry for `insn` if `fixup` is provided. Otherwise
> * do nothing.
> diff --git a/arch/arm64/include/asm/asm-uaccess.h b/arch/arm64/include/asm/asm-uaccess.h
> index 0557af834e03..6c23c138e1fc 100644
> --- a/arch/arm64/include/asm/asm-uaccess.h
> +++ b/arch/arm64/include/asm/asm-uaccess.h
> @@ -63,6 +63,11 @@ alternative_else_nop_endif
> 9999: x; \
> _asm_extable 9999b, l
>
> +
> +#define USER_MC(l, x...) \
> +9999: x; \
> + _asm_extable_uaccess_mc 9999b, l
> +
> /*
> * Generate the assembly for LDTR/STTR with exception table entries.
> * This is complicated as there is no post-increment or pair versions of the
> @@ -73,8 +78,8 @@ alternative_else_nop_endif
> 8889: ldtr \reg2, [\addr, #8];
> add \addr, \addr, \post_inc;
>
> - _asm_extable 8888b,\l;
> - _asm_extable 8889b,\l;
> + _asm_extable_uaccess_mc 8888b, \l;
> + _asm_extable_uaccess_mc 8889b, \l;
> .endm
>
> .macro user_stp l, reg1, reg2, addr, post_inc
> @@ -82,14 +87,14 @@ alternative_else_nop_endif
> 8889: sttr \reg2, [\addr, #8];
> add \addr, \addr, \post_inc;
>
> - _asm_extable 8888b,\l;
> - _asm_extable 8889b,\l;
> + _asm_extable_uaccess_mc 8888b,\l;
> + _asm_extable_uaccess_mc 8889b,\l;
> .endm
>
> .macro user_ldst l, inst, reg, addr, post_inc
> 8888: \inst \reg, [\addr];
> add \addr, \addr, \post_inc;
>
> - _asm_extable 8888b,\l;
> + _asm_extable_uaccess_mc 8888b, \l;
> .endm
> #endif
> diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S
> index 34e317907524..480cc5ac0a8d 100644
> --- a/arch/arm64/lib/copy_from_user.S
> +++ b/arch/arm64/lib/copy_from_user.S
> @@ -25,7 +25,7 @@
> .endm
>
> .macro strb1 reg, ptr, val
> - strb \reg, [\ptr], \val
> + USER_MC(9998f, strb \reg, [\ptr], \val)
> .endm
>
> .macro ldrh1 reg, ptr, val
> @@ -33,7 +33,7 @@
> .endm
>
> .macro strh1 reg, ptr, val
> - strh \reg, [\ptr], \val
> + USER_MC(9998f, strh \reg, [\ptr], \val)
> .endm
>
> .macro ldr1 reg, ptr, val
> @@ -41,7 +41,7 @@
> .endm
>
> .macro str1 reg, ptr, val
> - str \reg, [\ptr], \val
> + USER_MC(9998f, str \reg, [\ptr], \val)
> .endm
>
> .macro ldp1 reg1, reg2, ptr, val
> @@ -49,11 +49,12 @@
> .endm
>
> .macro stp1 reg1, reg2, ptr, val
> - stp \reg1, \reg2, [\ptr], \val
> + USER_MC(9998f, stp \reg1, \reg2, [\ptr], \val)
> .endm
>
> -end .req x5
> -srcin .req x15
> +end .req x5
> +srcin .req x15
> +fixup_type .req x16
> SYM_FUNC_START(__arch_copy_from_user)
> add end, x0, x2
> mov srcin, x1
> @@ -62,7 +63,10 @@ SYM_FUNC_START(__arch_copy_from_user)
> ret
>
> // Exception fixups
> -9997: cmp dst, dstin
> + // x16: fixup type written by ex_handler_uaccess_mc
> +9997: cmp fixup_type, #FIXUP_TYPE_MC
> + b.eq 9998f
> + cmp dst, dstin
> b.ne 9998f
> // Before being absolutely sure we couldn't copy anything, try harder
> USER(9998f, ldtrb tmp1w, [srcin])
> diff --git a/arch/arm64/lib/copy_to_user.S b/arch/arm64/lib/copy_to_user.S
> index 802231772608..021a7d27b3a4 100644
> --- a/arch/arm64/lib/copy_to_user.S
> +++ b/arch/arm64/lib/copy_to_user.S
> @@ -20,7 +20,7 @@
> * x0 - bytes not copied
> */
> .macro ldrb1 reg, ptr, val
> - ldrb \reg, [\ptr], \val
> + USER_MC(9998f, ldrb \reg, [\ptr], \val)
> .endm
>
> .macro strb1 reg, ptr, val
> @@ -28,7 +28,7 @@
> .endm
>
> .macro ldrh1 reg, ptr, val
> - ldrh \reg, [\ptr], \val
> + USER_MC(9998f, ldrh \reg, [\ptr], \val)
> .endm
>
> .macro strh1 reg, ptr, val
> @@ -36,7 +36,7 @@
> .endm
>
> .macro ldr1 reg, ptr, val
> - ldr \reg, [\ptr], \val
> + USER_MC(9998f, ldr \reg, [\ptr], \val)
> .endm
>
> .macro str1 reg, ptr, val
> @@ -44,15 +44,16 @@
> .endm
>
> .macro ldp1 reg1, reg2, ptr, val
> - ldp \reg1, \reg2, [\ptr], \val
> + USER_MC(9998f, ldp \reg1, \reg2, [\ptr], \val)
> .endm
>
> .macro stp1 reg1, reg2, ptr, val
> user_stp 9997f, \reg1, \reg2, \ptr, \val
> .endm
>
> -end .req x5
> -srcin .req x15
> +end .req x5
> +srcin .req x15
> +fixup_type .req x16
> SYM_FUNC_START(__arch_copy_to_user)
> add end, x0, x2
> mov srcin, x1
> @@ -61,7 +62,10 @@ SYM_FUNC_START(__arch_copy_to_user)
> ret
>
> // Exception fixups
> -9997: cmp dst, dstin
> + // x16: fixup type written by ex_handler_uaccess_mc
> +9997: cmp fixup_type, #FIXUP_TYPE_MC
> + b.eq 9998f
> + cmp dst, dstin
> b.ne 9998f
> // Before being absolutely sure we couldn't copy anything, try harder
> ldrb tmp1w, [srcin]
> diff --git a/arch/arm64/mm/extable.c b/arch/arm64/mm/extable.c
> index 4f0083a550d4..525876c3ebf4 100644
> --- a/arch/arm64/mm/extable.c
> +++ b/arch/arm64/mm/extable.c
> @@ -24,6 +24,14 @@ static bool ex_handler_fixup(const struct exception_table_entry *ex,
> return true;
> }
>
> +static bool ex_handler_uaccess_type(const struct exception_table_entry *ex,
> + struct pt_regs *regs,
> + unsigned long fixup_type)
> +{
> + regs->regs[16] = fixup_type;
> + return ex_handler_fixup(ex, regs);
> +}
> +
> static bool ex_handler_uaccess_err_zero(const struct exception_table_entry *ex,
> struct pt_regs *regs)
> {
> @@ -75,6 +83,8 @@ bool fixup_exception(struct pt_regs *regs)
> switch (ex->type) {
> case EX_TYPE_FIXUP:
> return ex_handler_fixup(ex, regs);
> + case EX_TYPE_UACCESS_MC:
> + return ex_handler_uaccess_type(ex, regs, FIXUP_TYPE_NORMAL);
> case EX_TYPE_BPF:
> return ex_handler_bpf(ex, regs);
> case EX_TYPE_UACCESS_ERR_ZERO:
> @@ -94,10 +104,10 @@ bool fixup_exception_mc(struct pt_regs *regs)
> if (!ex)
> return false;
>
> - /*
> - * This is not complete, More Machine check safe extable type can
> - * be processed here.
> - */
> + switch (ex->type) {
> + case EX_TYPE_UACCESS_MC:
> + return ex_handler_uaccess_type(ex, regs, FIXUP_TYPE_MC);
> + }
>
> return false;
> }
> --
> 2.25.1
>
On Wed, 20 Apr 2022 03:04:11 +0000, Tong Tiangen wrote:
> With the increase of memory capacity and density, the probability of
> memory error increases. The increasing size and density of server RAM
> in the data center and cloud have shown increased uncorrectable memory
> errors.
>
> Currently, the kernel has a mechanism to recover from hardware memory
> errors. This patchset provides an new recovery mechanism.
>
> [...]
Applied to arm64 (for-next/misc), thanks!
[5/7] arm64: mte: Clean up user tag accessors
https://git.kernel.org/arm64/c/b4d6bb38f9dc
--
Catalin
在 2022/5/13 23:31, Mark Rutland 写道:
> On Wed, Apr 20, 2022 at 03:04:15AM +0000, Tong Tiangen wrote:
>> Add copy_{to, from}_user() to machine check safe.
>>
>> If copy fail due to hardware memory error, only the relevant processes are
>> affected, so killing the user process and isolate the user page with
>> hardware memory errors is a more reasonable choice than kernel panic.
>>
>> Add new extable type EX_TYPE_UACCESS_MC which can be used for uaccess that
>> can be recovered from hardware memory errors.
>
> I don't understand why we need this.
>
> If we apply EX_TYPE_UACCESS consistently to *all* user accesses, and
> *only* to user accesses, that would *always* indicate that we can
> recover, and that seems much simpler to deal with.
>
> Today we use EX_TYPE_UACCESS_ERR_ZERO for kernel accesses in a couple of
> cases, which we should clean up, and we user EX_TYPE_FIXUP for a couple
> of user accesses, but those could easily be converted over.
>
>> The x16 register is used to save the fixup type in copy_xxx_user which
>> used extable type EX_TYPE_UACCESS_MC.
This is dicussed on patch patch 3/7.
>
> Why x16?
>
> How is this intended to be consumed, and why is that behaviour different
> from any *other* fault?
>
> Mark.
This is to distinguish EX_TYPE_FIXUP, if this exception is triggered,
in fixup processing, it is needed to copy by byte, but if exception is
triggered by machine check, the data does not need to be copied again.
So we need one place to store exception type, Therefore, X16 that is not
currently used in copy_from/to_user is selected.
Maybe better to use exception_table_entry->data to pass the register
that needs to be set?
Thanks,
Tong.
>
>> Signed-off-by: Tong Tiangen <[email protected]>
>> ---
>> arch/arm64/include/asm/asm-extable.h | 14 ++++++++++++++
>> arch/arm64/include/asm/asm-uaccess.h | 15 ++++++++++-----
>> arch/arm64/lib/copy_from_user.S | 18 +++++++++++-------
>> arch/arm64/lib/copy_to_user.S | 18 +++++++++++-------
>> arch/arm64/mm/extable.c | 18 ++++++++++++++----
>> 5 files changed, 60 insertions(+), 23 deletions(-)
>>
>> diff --git a/arch/arm64/include/asm/asm-extable.h b/arch/arm64/include/asm/asm-extable.h
>> index c39f2437e08e..75b2c00e9523 100644
>> --- a/arch/arm64/include/asm/asm-extable.h
>> +++ b/arch/arm64/include/asm/asm-extable.h
>> @@ -2,12 +2,18 @@
>> #ifndef __ASM_ASM_EXTABLE_H
>> #define __ASM_ASM_EXTABLE_H
>>
>> +#define FIXUP_TYPE_NORMAL 0
>> +#define FIXUP_TYPE_MC 1
>> +
>> #define EX_TYPE_NONE 0
>> #define EX_TYPE_FIXUP 1
>> #define EX_TYPE_BPF 2
>> #define EX_TYPE_UACCESS_ERR_ZERO 3
>> #define EX_TYPE_LOAD_UNALIGNED_ZEROPAD 4
>>
>> +/* _MC indicates that can fixup from machine check errors */
>> +#define EX_TYPE_UACCESS_MC 5
>> +
>> #ifdef __ASSEMBLY__
>>
>> #define __ASM_EXTABLE_RAW(insn, fixup, type, data) \
>> @@ -27,6 +33,14 @@
>> __ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_FIXUP, 0)
>> .endm
>>
>> +/*
>> + * Create an exception table entry for `insn`, which will branch to `fixup`
>> + * when an unhandled fault(include sea fault) is taken.
>> + */
>> + .macro _asm_extable_uaccess_mc, insn, fixup
>> + __ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_UACCESS_MC, 0)
>> + .endm
>> +
>> /*
>> * Create an exception table entry for `insn` if `fixup` is provided. Otherwise
>> * do nothing.
>> diff --git a/arch/arm64/include/asm/asm-uaccess.h b/arch/arm64/include/asm/asm-uaccess.h
>> index 0557af834e03..6c23c138e1fc 100644
>> --- a/arch/arm64/include/asm/asm-uaccess.h
>> +++ b/arch/arm64/include/asm/asm-uaccess.h
>> @@ -63,6 +63,11 @@ alternative_else_nop_endif
>> 9999: x; \
>> _asm_extable 9999b, l
>>
>> +
>> +#define USER_MC(l, x...) \
>> +9999: x; \
>> + _asm_extable_uaccess_mc 9999b, l
>> +
>> /*
>> * Generate the assembly for LDTR/STTR with exception table entries.
>> * This is complicated as there is no post-increment or pair versions of the
>> @@ -73,8 +78,8 @@ alternative_else_nop_endif
>> 8889: ldtr \reg2, [\addr, #8];
>> add \addr, \addr, \post_inc;
>>
>> - _asm_extable 8888b,\l;
>> - _asm_extable 8889b,\l;
>> + _asm_extable_uaccess_mc 8888b, \l;
>> + _asm_extable_uaccess_mc 8889b, \l;
>> .endm
>>
>> .macro user_stp l, reg1, reg2, addr, post_inc
>> @@ -82,14 +87,14 @@ alternative_else_nop_endif
>> 8889: sttr \reg2, [\addr, #8];
>> add \addr, \addr, \post_inc;
>>
>> - _asm_extable 8888b,\l;
>> - _asm_extable 8889b,\l;
>> + _asm_extable_uaccess_mc 8888b,\l;
>> + _asm_extable_uaccess_mc 8889b,\l;
>> .endm
>>
>> .macro user_ldst l, inst, reg, addr, post_inc
>> 8888: \inst \reg, [\addr];
>> add \addr, \addr, \post_inc;
>>
>> - _asm_extable 8888b,\l;
>> + _asm_extable_uaccess_mc 8888b, \l;
>> .endm
>> #endif
>> diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S
>> index 34e317907524..480cc5ac0a8d 100644
>> --- a/arch/arm64/lib/copy_from_user.S
>> +++ b/arch/arm64/lib/copy_from_user.S
>> @@ -25,7 +25,7 @@
>> .endm
>>
>> .macro strb1 reg, ptr, val
>> - strb \reg, [\ptr], \val
>> + USER_MC(9998f, strb \reg, [\ptr], \val)
>> .endm
>>
>> .macro ldrh1 reg, ptr, val
>> @@ -33,7 +33,7 @@
>> .endm
>>
>> .macro strh1 reg, ptr, val
>> - strh \reg, [\ptr], \val
>> + USER_MC(9998f, strh \reg, [\ptr], \val)
>> .endm
>>
>> .macro ldr1 reg, ptr, val
>> @@ -41,7 +41,7 @@
>> .endm
>>
>> .macro str1 reg, ptr, val
>> - str \reg, [\ptr], \val
>> + USER_MC(9998f, str \reg, [\ptr], \val)
>> .endm
>>
>> .macro ldp1 reg1, reg2, ptr, val
>> @@ -49,11 +49,12 @@
>> .endm
>>
>> .macro stp1 reg1, reg2, ptr, val
>> - stp \reg1, \reg2, [\ptr], \val
>> + USER_MC(9998f, stp \reg1, \reg2, [\ptr], \val)
>> .endm
>>
>> -end .req x5
>> -srcin .req x15
>> +end .req x5
>> +srcin .req x15
>> +fixup_type .req x16
>> SYM_FUNC_START(__arch_copy_from_user)
>> add end, x0, x2
>> mov srcin, x1
>> @@ -62,7 +63,10 @@ SYM_FUNC_START(__arch_copy_from_user)
>> ret
>>
>> // Exception fixups
>> -9997: cmp dst, dstin
>> + // x16: fixup type written by ex_handler_uaccess_mc
>> +9997: cmp fixup_type, #FIXUP_TYPE_MC
>> + b.eq 9998f
>> + cmp dst, dstin
>> b.ne 9998f
>> // Before being absolutely sure we couldn't copy anything, try harder
>> USER(9998f, ldtrb tmp1w, [srcin])
>> diff --git a/arch/arm64/lib/copy_to_user.S b/arch/arm64/lib/copy_to_user.S
>> index 802231772608..021a7d27b3a4 100644
>> --- a/arch/arm64/lib/copy_to_user.S
>> +++ b/arch/arm64/lib/copy_to_user.S
>> @@ -20,7 +20,7 @@
>> * x0 - bytes not copied
>> */
>> .macro ldrb1 reg, ptr, val
>> - ldrb \reg, [\ptr], \val
>> + USER_MC(9998f, ldrb \reg, [\ptr], \val)
>> .endm
>>
>> .macro strb1 reg, ptr, val
>> @@ -28,7 +28,7 @@
>> .endm
>>
>> .macro ldrh1 reg, ptr, val
>> - ldrh \reg, [\ptr], \val
>> + USER_MC(9998f, ldrh \reg, [\ptr], \val)
>> .endm
>>
>> .macro strh1 reg, ptr, val
>> @@ -36,7 +36,7 @@
>> .endm
>>
>> .macro ldr1 reg, ptr, val
>> - ldr \reg, [\ptr], \val
>> + USER_MC(9998f, ldr \reg, [\ptr], \val)
>> .endm
>>
>> .macro str1 reg, ptr, val
>> @@ -44,15 +44,16 @@
>> .endm
>>
>> .macro ldp1 reg1, reg2, ptr, val
>> - ldp \reg1, \reg2, [\ptr], \val
>> + USER_MC(9998f, ldp \reg1, \reg2, [\ptr], \val)
>> .endm
>>
>> .macro stp1 reg1, reg2, ptr, val
>> user_stp 9997f, \reg1, \reg2, \ptr, \val
>> .endm
>>
>> -end .req x5
>> -srcin .req x15
>> +end .req x5
>> +srcin .req x15
>> +fixup_type .req x16
>> SYM_FUNC_START(__arch_copy_to_user)
>> add end, x0, x2
>> mov srcin, x1
>> @@ -61,7 +62,10 @@ SYM_FUNC_START(__arch_copy_to_user)
>> ret
>>
>> // Exception fixups
>> -9997: cmp dst, dstin
>> + // x16: fixup type written by ex_handler_uaccess_mc
>> +9997: cmp fixup_type, #FIXUP_TYPE_MC
>> + b.eq 9998f
>> + cmp dst, dstin
>> b.ne 9998f
>> // Before being absolutely sure we couldn't copy anything, try harder
>> ldrb tmp1w, [srcin]
>> diff --git a/arch/arm64/mm/extable.c b/arch/arm64/mm/extable.c
>> index 4f0083a550d4..525876c3ebf4 100644
>> --- a/arch/arm64/mm/extable.c
>> +++ b/arch/arm64/mm/extable.c
>> @@ -24,6 +24,14 @@ static bool ex_handler_fixup(const struct exception_table_entry *ex,
>> return true;
>> }
>>
>> +static bool ex_handler_uaccess_type(const struct exception_table_entry *ex,
>> + struct pt_regs *regs,
>> + unsigned long fixup_type)
>> +{
>> + regs->regs[16] = fixup_type;
>> + return ex_handler_fixup(ex, regs);
>> +}
>> +
>> static bool ex_handler_uaccess_err_zero(const struct exception_table_entry *ex,
>> struct pt_regs *regs)
>> {
>> @@ -75,6 +83,8 @@ bool fixup_exception(struct pt_regs *regs)
>> switch (ex->type) {
>> case EX_TYPE_FIXUP:
>> return ex_handler_fixup(ex, regs);
>> + case EX_TYPE_UACCESS_MC:
>> + return ex_handler_uaccess_type(ex, regs, FIXUP_TYPE_NORMAL);
>> case EX_TYPE_BPF:
>> return ex_handler_bpf(ex, regs);
>> case EX_TYPE_UACCESS_ERR_ZERO:
>> @@ -94,10 +104,10 @@ bool fixup_exception_mc(struct pt_regs *regs)
>> if (!ex)
>> return false;
>>
>> - /*
>> - * This is not complete, More Machine check safe extable type can
>> - * be processed here.
>> - */
>> + switch (ex->type) {
>> + case EX_TYPE_UACCESS_MC:
>> + return ex_handler_uaccess_type(ex, regs, FIXUP_TYPE_MC);
>> + }
>>
>> return false;
>> }
>> --
>> 2.25.1
>>
> .