rth wrote a patch back in 2016 that uses gcc's address space machinery
to improve code generation for percpu accesses. Ingo asked for some
minor changes to be made, but Richard didn't respond. While looking at
something related, I came across them, and asked Richard's permission
to resurrect and resend the patch, with the changes Ingo asked for at
the time.
The important information about why we want these patch is in 1/2.
Richard Henderson (2):
x86: Use named address spaces in asm/percpu.h
x86: Enable named address spaces for percpu data
arch/x86/include/asm/percpu.h | 147 ++++++++++++++++++++--------------
1 file changed, 88 insertions(+), 59 deletions(-)
--
2.18.0
From: Richard Henderson <[email protected]>
GCC 6 adds support for __seg_fs and __seg_gs as named address spaces,
producing the obvious segment overrides for objects so marked.
Exposing the memory reference allows slightly better code generation
in some cases (and in others, merely affects the scheduling). E.g.:
[1]
- mov %gs:0x0(%rip),%eax
- R_X86_64_PC32 context_tracking+0x4
- cmp $0x1,%eax
+ cmpl $0x1,%gs:0x0(%rip)
+ R_X86_64_PC32 context_tracking+0x3
[2]
- mov %gs:0x0(%rip),%ebx
- R_X86_64_PC32 cpu_number-0x4
- movslq %ebx,%rax
+ movslq %gs:0x0(%rip),%rax
+ R_X86_64_PC32 cpu_number-0x4
[3]
- mov %gs:0x0(%rip),%rdx
- R_X86_64_PC32 cpu_info+0x20
- test $0x1000000,%edx
+ testb $0x1,%gs:0x0(%rip)
+ R_X86_64_PC32 cpu_info+0x22
[4]
- mov $0x0,%rax
- R_X86_64_32S __uv_hub_info
- mov %rax,%rcx
- add %gs:0x0(%rip),%rcx
- R_X86_64_PC32 this_cpu_off-0x4
- movzbl 0x15(%rcx),%ecx
...
- mov %rax,%rdx
- add %gs:0x0(%rip),%rdx
- R_X86_64_PC32 this_cpu_off-0x4
- or (%rdx),%rcx
+ mov %gs:0x0(%rip),%r9
+ R_X86_64_PC32 this_cpu_off-0x4
+ mov $0x0,%rax
+ R_X86_64_32S __uv_hub_info
...
+ movzbl 0x15(%rax,%r9,1),%ecx
...
+ or (%rax,%r9,1),%rdx
The final vmlinux text size is reduced by about 5k for a standard
Fedora configure.
Signed-off-by: Richard Henderson <[email protected]>
[changes as requested by Ingo]
Signed-off-by: Matthew Wilcox <[email protected]>
---
arch/x86/include/asm/percpu.h | 141 ++++++++++++++++++++--------------
1 file changed, 82 insertions(+), 59 deletions(-)
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index e9202a0de8f0..30a08d0d95ee 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -47,6 +47,19 @@
#ifdef CONFIG_SMP
#define __percpu_prefix "%%"__stringify(__percpu_seg)":"
+
+#ifdef __percpu_addrspace
+/* Produce an address-space lvalue for VAR. */
+#define __percpu_as(VAR) \
+ (*(typeof(VAR) __kernel __force __percpu_addrspace *)(uintptr_t)&(VAR))
+
+/*
+ * We cannot allow __my_cpu_offset to recurse through this_cpu_read, as
+ * this will change based on CONFIG_X86_64, with which games are played
+ * in 32-bit compatibility files.
+ */
+#define __my_cpu_offset (__percpu_as(this_cpu_off) + 0)
+#else
#define __my_cpu_offset this_cpu_read(this_cpu_off)
/*
@@ -61,9 +74,11 @@
: "m" (this_cpu_off), "0" (ptr)); \
(typeof(*(ptr)) __kernel __force *)tcp_ptr__; \
})
+#endif /* __percpu_addrspace */
#else
#define __percpu_prefix ""
-#endif
+#undef __percpu_addrspace
+#endif /* SMP */
#define __percpu_arg(x) __percpu_prefix "%" #x
@@ -87,7 +102,14 @@
* don't give an lvalue though). */
extern void __bad_percpu_size(void);
-#define percpu_to_op(op, var, val) \
+#ifdef __percpu_addrspace
+#define percpu_to_op(op, cop, var, val) \
+do { \
+ typeof(var) pto_tmp__ = (val); \
+ __percpu_as(var) cop pto_tmp__; \
+} while (0)
+#else
+#define percpu_to_op(op, cop, var, val) \
do { \
typedef typeof(var) pto_T__; \
if (0) { \
@@ -119,11 +141,15 @@ do { \
default: __bad_percpu_size(); \
} \
} while (0)
+#endif /* __percpu_addrspace */
/*
* Generate a percpu add to memory instruction and optimize code
* if one is added or subtracted.
*/
+#ifdef __percpu_addrspace
+#define percpu_add_op(var, val) percpu_to_op("add", +=, var, val)
+#else
#define percpu_add_op(var, val) \
do { \
typedef typeof(var) pao_T__; \
@@ -179,7 +205,9 @@ do { \
default: __bad_percpu_size(); \
} \
} while (0)
+#endif /* __percpu_addrspace */
+/* ??? Note that percpu_from_op is only ever used with mov. */
#define percpu_from_op(op, var) \
({ \
typeof(var) pfo_ret__; \
@@ -238,35 +266,19 @@ do { \
pfo_ret__; \
})
-#define percpu_unary_op(op, var) \
-({ \
- switch (sizeof(var)) { \
- case 1: \
- asm(op "b "__percpu_arg(0) \
- : "+m" (var)); \
- break; \
- case 2: \
- asm(op "w "__percpu_arg(0) \
- : "+m" (var)); \
- break; \
- case 4: \
- asm(op "l "__percpu_arg(0) \
- : "+m" (var)); \
- break; \
- case 8: \
- asm(op "q "__percpu_arg(0) \
- : "+m" (var)); \
- break; \
- default: __bad_percpu_size(); \
- } \
-})
-
/*
* Add return operation
*/
+#ifdef __percpu_addrspace
+#define percpu_add_return_op(var, val) \
+({ \
+ typeof(var) pto_tmp__ = (val); \
+ __percpu_as(var) += pto_tmp__; \
+})
+#else
#define percpu_add_return_op(var, val) \
({ \
- typeof(var) paro_ret__ = val; \
+ typeof(var) paro_ret__ = (val); \
switch (sizeof(var)) { \
case 1: \
asm("xaddb %0, "__percpu_arg(1) \
@@ -293,6 +305,7 @@ do { \
paro_ret__ += val; \
paro_ret__; \
})
+#endif /* __percpu_addrspace */
/*
* xchg is implemented using cmpxchg without a lock prefix. xchg is
@@ -391,41 +404,47 @@ do { \
*/
#define this_cpu_read_stable(var) percpu_stable_op("mov", var)
+#ifdef __percpu_addrspace
+#define raw_cpu_read_1(pcp) ({ __percpu_as(pcp); })
+#define raw_cpu_read_2(pcp) ({ __percpu_as(pcp); })
+#define raw_cpu_read_4(pcp) ({ __percpu_as(pcp); })
+#else
#define raw_cpu_read_1(pcp) percpu_from_op("mov", pcp)
#define raw_cpu_read_2(pcp) percpu_from_op("mov", pcp)
#define raw_cpu_read_4(pcp) percpu_from_op("mov", pcp)
+#endif
-#define raw_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val)
-#define raw_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val)
-#define raw_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val)
+#define raw_cpu_write_1(pcp, val) percpu_to_op("mov", =, (pcp), val)
+#define raw_cpu_write_2(pcp, val) percpu_to_op("mov", =, (pcp), val)
+#define raw_cpu_write_4(pcp, val) percpu_to_op("mov", =, (pcp), val)
#define raw_cpu_add_1(pcp, val) percpu_add_op((pcp), val)
#define raw_cpu_add_2(pcp, val) percpu_add_op((pcp), val)
#define raw_cpu_add_4(pcp, val) percpu_add_op((pcp), val)
-#define raw_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val)
-#define raw_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val)
-#define raw_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val)
-#define raw_cpu_or_1(pcp, val) percpu_to_op("or", (pcp), val)
-#define raw_cpu_or_2(pcp, val) percpu_to_op("or", (pcp), val)
-#define raw_cpu_or_4(pcp, val) percpu_to_op("or", (pcp), val)
+#define raw_cpu_and_1(pcp, val) percpu_to_op("and", &=, (pcp), val)
+#define raw_cpu_and_2(pcp, val) percpu_to_op("and", &=, (pcp), val)
+#define raw_cpu_and_4(pcp, val) percpu_to_op("and", &=, (pcp), val)
+#define raw_cpu_or_1(pcp, val) percpu_to_op("or", |=, (pcp), val)
+#define raw_cpu_or_2(pcp, val) percpu_to_op("or", |=, (pcp), val)
+#define raw_cpu_or_4(pcp, val) percpu_to_op("or", |=, (pcp), val)
#define raw_cpu_xchg_1(pcp, val) percpu_xchg_op(pcp, val)
#define raw_cpu_xchg_2(pcp, val) percpu_xchg_op(pcp, val)
#define raw_cpu_xchg_4(pcp, val) percpu_xchg_op(pcp, val)
-#define this_cpu_read_1(pcp) percpu_from_op("mov", pcp)
-#define this_cpu_read_2(pcp) percpu_from_op("mov", pcp)
-#define this_cpu_read_4(pcp) percpu_from_op("mov", pcp)
-#define this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val)
-#define this_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val)
-#define this_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val)
-#define this_cpu_add_1(pcp, val) percpu_add_op((pcp), val)
-#define this_cpu_add_2(pcp, val) percpu_add_op((pcp), val)
-#define this_cpu_add_4(pcp, val) percpu_add_op((pcp), val)
-#define this_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val)
-#define this_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val)
-#define this_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val)
-#define this_cpu_or_1(pcp, val) percpu_to_op("or", (pcp), val)
-#define this_cpu_or_2(pcp, val) percpu_to_op("or", (pcp), val)
-#define this_cpu_or_4(pcp, val) percpu_to_op("or", (pcp), val)
+#define this_cpu_read_1(pcp) raw_cpu_read_1(pcp)
+#define this_cpu_read_2(pcp) raw_cpu_read_2(pcp)
+#define this_cpu_read_4(pcp) raw_cpu_read_4(pcp)
+#define this_cpu_write_1(pcp, val) raw_cpu_write_1(pcp, val)
+#define this_cpu_write_2(pcp, val) raw_cpu_write_2(pcp, val)
+#define this_cpu_write_4(pcp, val) raw_cpu_write_4(pcp, val)
+#define this_cpu_add_1(pcp, val) raw_cpu_add_1(pcp, val)
+#define this_cpu_add_2(pcp, val) raw_cpu_add_2(pcp, val)
+#define this_cpu_add_4(pcp, val) raw_cpu_add_4(pcp, val)
+#define this_cpu_and_1(pcp, val) raw_cpu_and_1(pcp, val)
+#define this_cpu_and_2(pcp, val) raw_cpu_and_2(pcp, val)
+#define this_cpu_and_4(pcp, val) raw_cpu_and_4(pcp, val)
+#define this_cpu_or_1(pcp, val) raw_cpu_or_1(pcp, val)
+#define this_cpu_or_2(pcp, val) raw_cpu_or_2(pcp, val)
+#define this_cpu_or_4(pcp, val) raw_cpu_or_4(pcp, val)
#define this_cpu_xchg_1(pcp, nval) percpu_xchg_op(pcp, nval)
#define this_cpu_xchg_2(pcp, nval) percpu_xchg_op(pcp, nval)
#define this_cpu_xchg_4(pcp, nval) percpu_xchg_op(pcp, nval)
@@ -466,20 +485,24 @@ do { \
* 32 bit must fall back to generic operations.
*/
#ifdef CONFIG_X86_64
+#ifdef __percpu_addrspace
+#define raw_cpu_read_8(pcp) ({ __percpu_as(pcp); })
+#else
#define raw_cpu_read_8(pcp) percpu_from_op("mov", pcp)
-#define raw_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val)
-#define raw_cpu_add_8(pcp, val) percpu_add_op((pcp), val)
-#define raw_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val)
-#define raw_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val)
+#endif
+#define raw_cpu_write_8(pcp, val) percpu_to_op("mov", =, (pcp), val)
+#define raw_cpu_add_8(pcp, val) percpu_add_op(pcp, val)
+#define raw_cpu_and_8(pcp, val) percpu_to_op("and", &=, (pcp), val)
+#define raw_cpu_or_8(pcp, val) percpu_to_op("or", |=, (pcp), val)
#define raw_cpu_add_return_8(pcp, val) percpu_add_return_op(pcp, val)
#define raw_cpu_xchg_8(pcp, nval) percpu_xchg_op(pcp, nval)
#define raw_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
-#define this_cpu_read_8(pcp) percpu_from_op("mov", pcp)
-#define this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val)
-#define this_cpu_add_8(pcp, val) percpu_add_op((pcp), val)
-#define this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val)
-#define this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val)
+#define this_cpu_read_8(pcp) raw_cpu_read_8(pcp)
+#define this_cpu_write_8(pcp, val) raw_cpu_write_8(pcp, val)
+#define this_cpu_add_8(pcp, val) raw_cpu_add_8(pcp, val)
+#define this_cpu_and_8(pcp, val) raw_cpu_and_8(pcp, val)
+#define this_cpu_or_8(pcp, val) raw_cpu_or_8(pcp, val)
#define this_cpu_add_return_8(pcp, val) percpu_add_return_op(pcp, val)
#define this_cpu_xchg_8(pcp, nval) percpu_xchg_op(pcp, nval)
#define this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
--
2.18.0
From: Richard Henderson <[email protected]>
For ease of disabling this feature, split the actual enabling into its
own patch.
Signed-off-by: Richard Henderson <[email protected]>
Signed-off-by: Matthew Wilcox <[email protected]>
---
arch/x86/include/asm/percpu.h | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 30a08d0d95ee..f3c5eebcbc86 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -5,9 +5,15 @@
#ifdef CONFIG_X86_64
#define __percpu_seg gs
#define __percpu_mov_op movq
+# ifdef __SEG_GS
+# define __percpu_addrspace __seg_gs
+# endif
#else
#define __percpu_seg fs
#define __percpu_mov_op movl
+# ifdef __SEG_FS
+# define __percpu_addrspace __seg_fs
+# endif
#endif
#ifdef __ASSEMBLY__
--
2.18.0
On 09/12/2018 07:44 AM, Matthew Wilcox wrote:
> rth wrote a patch back in 2016 that uses gcc's address space machinery
> to improve code generation for percpu accesses. Ingo asked for some
> minor changes to be made, but Richard didn't respond. While looking at
> something related, I came across them, and asked Richard's permission
> to resurrect and resend the patch, with the changes Ingo asked for at
> the time.
>
> The important information about why we want these patch is in 1/2.
>
> Richard Henderson (2):
> x86: Use named address spaces in asm/percpu.h
> x86: Enable named address spaces for percpu data
>
> arch/x86/include/asm/percpu.h | 147 ++++++++++++++++++++--------------
> 1 file changed, 88 insertions(+), 59 deletions(-)
Thanks for resurrecting this, Willy. I'd totally forgotten about it.
The adjustments you made look fine.
r~
* Richard Henderson <[email protected]> wrote:
> On 09/12/2018 07:44 AM, Matthew Wilcox wrote:
> > rth wrote a patch back in 2016 that uses gcc's address space machinery
> > to improve code generation for percpu accesses. Ingo asked for some
> > minor changes to be made, but Richard didn't respond. While looking at
> > something related, I came across them, and asked Richard's permission
> > to resurrect and resend the patch, with the changes Ingo asked for at
> > the time.
> >
> > The important information about why we want these patch is in 1/2.
> >
> > Richard Henderson (2):
> > x86: Use named address spaces in asm/percpu.h
> > x86: Enable named address spaces for percpu data
> >
> > arch/x86/include/asm/percpu.h | 147 ++++++++++++++++++++--------------
> > 1 file changed, 88 insertions(+), 59 deletions(-)
>
> Thanks for resurrecting this, Willy. I'd totally forgotten about it.
> The adjustments you made look fine.
Not all GCC versions seem to like this series:
In file included from arch/x86/mm/tlb.c:12:0:
arch/x86/mm/tlb.c: In function ‘switch_mm_irqs_off’:
./arch/x86/include/asm/mmu_context.h:210:28: internal compiler error: Segmentation fault
#define switch_mm_irqs_off switch_mm_irqs_off
arch/x86/mm/tlb.c:183:6: note: in expansion of macro ‘switch_mm_irqs_off’
void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
Please submit a full bug report,
with preprocessed source if appropriate.
See <file:///usr/share/doc/gcc-7/README.Bugs> for instructions.
gcc version 7.3.0 (Ubuntu 7.3.0-16ubuntu3)
Thanks,
Ingo
* Ingo Molnar <[email protected]> wrote:
> Not all GCC versions seem to like this series:
>
> In file included from arch/x86/mm/tlb.c:12:0:
> arch/x86/mm/tlb.c: In function ‘switch_mm_irqs_off’:
> ./arch/x86/include/asm/mmu_context.h:210:28: internal compiler error: Segmentation fault
> #define switch_mm_irqs_off switch_mm_irqs_off
> arch/x86/mm/tlb.c:183:6: note: in expansion of macro ‘switch_mm_irqs_off’
> void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
> Please submit a full bug report,
> with preprocessed source if appropriate.
> See <file:///usr/share/doc/gcc-7/README.Bugs> for instructions.
>
> gcc version 7.3.0 (Ubuntu 7.3.0-16ubuntu3)
That's with a 32-bit kernel, config attached.
Thanks,
Ingo