Inspired by Benjamin Herrenschmidt, this patch series try to reduce the
cpu/mmu feature checking overhead by using jump label. The following is
the difference of the run path of cpu_has_feature between before and after
applying these patches:
before after
addis r10,r2,1 b xxx
addi r9,r10,-2280 b xxx (This will also be omitted if the
ld r9,0(r9) feature is not set)
ld r9,16(r9)
rldicl. r8,r9,55,63
beq c000000000037c94
This patch series passed the build test for almost all the defconfig of ppc.
There does have some broken for some configs. But they are not related to this
change. This also passed allyesconfig for x86. Boot test on p2020rdb and
p5020ds boards.
Kevin Hao (5):
jump_label: factor out the base part of jump_label.h to a separate
file
jump_label: also include linux/atomic.h when jump label is enabled
powerpc: move the cpu_has_feature to a separate file
powerpc: use the jump label for cpu_has_feature
powerpc: use jump label for mmu_has_feature
arch/powerpc/include/asm/cacheflush.h | 1 +
arch/powerpc/include/asm/cpufeatures.h | 42 ++++++++++
arch/powerpc/include/asm/cputable.h | 8 --
arch/powerpc/include/asm/cputime.h | 1 +
arch/powerpc/include/asm/dbell.h | 1 +
arch/powerpc/include/asm/dcr-native.h | 1 +
arch/powerpc/include/asm/mman.h | 1 +
arch/powerpc/include/asm/mmu.h | 19 +++++
arch/powerpc/include/asm/time.h | 1 +
arch/powerpc/kernel/align.c | 1 +
arch/powerpc/kernel/cputable.c | 43 ++++++++++
arch/powerpc/kernel/irq.c | 1 +
arch/powerpc/kernel/process.c | 1 +
arch/powerpc/kernel/setup-common.c | 1 +
arch/powerpc/kernel/setup_32.c | 1 +
arch/powerpc/kernel/smp.c | 1 +
arch/powerpc/oprofile/op_model_rs64.c | 1 +
arch/powerpc/platforms/cell/pervasive.c | 1 +
arch/powerpc/xmon/ppc-dis.c | 1 +
include/linux/jump_label.h | 132 +----------------------------
include/linux/jump_label_base.h | 142 ++++++++++++++++++++++++++++++++
21 files changed, 263 insertions(+), 138 deletions(-)
create mode 100644 arch/powerpc/include/asm/cpufeatures.h
create mode 100644 include/linux/jump_label_base.h
--
1.8.3.1
We plan to use the jump label in the cpu/mmu feature check on ppc.
This will need to include the jump_label.h in several very basic header
files of ppc which seems to be included by most of the other head
files implicitly or explicitly. But in the current jump_label.h,
it also include the "linux/workqueue.h" and this will cause recursive
inclusion. In order to fix this, we choose to factor out the base
part of jump_label.h to a separate header file and we can include
that file instead of jump_label.h to avoid the recursive inclusion.
No functional change.
Signed-off-by: Kevin Hao <[email protected]>
---
include/linux/jump_label.h | 132 +------------------------------------
include/linux/jump_label_base.h | 142 ++++++++++++++++++++++++++++++++++++++++
2 files changed, 144 insertions(+), 130 deletions(-)
create mode 100644 include/linux/jump_label_base.h
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index 0976fc4..14bae65 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -46,20 +46,11 @@
*
*/
-#include <linux/types.h>
-#include <linux/compiler.h>
#include <linux/workqueue.h>
+#include <linux/jump_label_base.h>
-#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
-struct static_key {
- atomic_t enabled;
-/* Set lsb bit to 1 if branch is default true, 0 ot */
- struct jump_entry *entries;
-#ifdef CONFIG_MODULES
- struct static_key_mod *next;
-#endif
-};
+#ifdef HAVE_JUMP_LABEL
struct static_key_deferred {
struct static_key key;
@@ -67,145 +58,26 @@ struct static_key_deferred {
struct delayed_work work;
};
-# include <asm/jump_label.h>
-# define HAVE_JUMP_LABEL
-#endif /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */
-
-enum jump_label_type {
- JUMP_LABEL_DISABLE = 0,
- JUMP_LABEL_ENABLE,
-};
-
-struct module;
-
-#ifdef HAVE_JUMP_LABEL
-
-#define JUMP_LABEL_TRUE_BRANCH 1UL
-
-static
-inline struct jump_entry *jump_label_get_entries(struct static_key *key)
-{
- return (struct jump_entry *)((unsigned long)key->entries
- & ~JUMP_LABEL_TRUE_BRANCH);
-}
-
-static inline bool jump_label_get_branch_default(struct static_key *key)
-{
- if ((unsigned long)key->entries & JUMP_LABEL_TRUE_BRANCH)
- return true;
- return false;
-}
-
-static __always_inline bool static_key_false(struct static_key *key)
-{
- return arch_static_branch(key);
-}
-
-static __always_inline bool static_key_true(struct static_key *key)
-{
- return !static_key_false(key);
-}
-
-extern struct jump_entry __start___jump_table[];
-extern struct jump_entry __stop___jump_table[];
-
-extern void jump_label_init(void);
-extern void jump_label_lock(void);
-extern void jump_label_unlock(void);
-extern void arch_jump_label_transform(struct jump_entry *entry,
- enum jump_label_type type);
-extern void arch_jump_label_transform_static(struct jump_entry *entry,
- enum jump_label_type type);
-extern int jump_label_text_reserved(void *start, void *end);
-extern void static_key_slow_inc(struct static_key *key);
-extern void static_key_slow_dec(struct static_key *key);
extern void static_key_slow_dec_deferred(struct static_key_deferred *key);
-extern void jump_label_apply_nops(struct module *mod);
extern void
jump_label_rate_limit(struct static_key_deferred *key, unsigned long rl);
-#define STATIC_KEY_INIT_TRUE ((struct static_key) \
- { .enabled = ATOMIC_INIT(1), .entries = (void *)1 })
-#define STATIC_KEY_INIT_FALSE ((struct static_key) \
- { .enabled = ATOMIC_INIT(0), .entries = (void *)0 })
-
#else /* !HAVE_JUMP_LABEL */
-#include <linux/atomic.h>
-
-struct static_key {
- atomic_t enabled;
-};
-
-static __always_inline void jump_label_init(void)
-{
-}
-
struct static_key_deferred {
struct static_key key;
};
-static __always_inline bool static_key_false(struct static_key *key)
-{
- if (unlikely(atomic_read(&key->enabled)) > 0)
- return true;
- return false;
-}
-
-static __always_inline bool static_key_true(struct static_key *key)
-{
- if (likely(atomic_read(&key->enabled)) > 0)
- return true;
- return false;
-}
-
-static inline void static_key_slow_inc(struct static_key *key)
-{
- atomic_inc(&key->enabled);
-}
-
-static inline void static_key_slow_dec(struct static_key *key)
-{
- atomic_dec(&key->enabled);
-}
-
static inline void static_key_slow_dec_deferred(struct static_key_deferred *key)
{
static_key_slow_dec(&key->key);
}
-static inline int jump_label_text_reserved(void *start, void *end)
-{
- return 0;
-}
-
-static inline void jump_label_lock(void) {}
-static inline void jump_label_unlock(void) {}
-
-static inline int jump_label_apply_nops(struct module *mod)
-{
- return 0;
-}
-
static inline void
jump_label_rate_limit(struct static_key_deferred *key,
unsigned long rl)
{
}
-#define STATIC_KEY_INIT_TRUE ((struct static_key) \
- { .enabled = ATOMIC_INIT(1) })
-#define STATIC_KEY_INIT_FALSE ((struct static_key) \
- { .enabled = ATOMIC_INIT(0) })
-
#endif /* HAVE_JUMP_LABEL */
-
-#define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
-#define jump_label_enabled static_key_enabled
-
-static inline bool static_key_enabled(struct static_key *key)
-{
- return (atomic_read(&key->enabled) > 0);
-}
-
#endif /* _LINUX_JUMP_LABEL_H */
diff --git a/include/linux/jump_label_base.h b/include/linux/jump_label_base.h
new file mode 100644
index 0000000..20df08f
--- /dev/null
+++ b/include/linux/jump_label_base.h
@@ -0,0 +1,142 @@
+#ifndef _LINUX_JUMP_LABEL_BASE_H
+#define _LINUX_JUMP_LABEL_BASE_H
+
+
+#include <linux/types.h>
+#include <linux/compiler.h>
+
+#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
+
+struct static_key {
+ atomic_t enabled;
+/* Set lsb bit to 1 if branch is default true, 0 ot */
+ struct jump_entry *entries;
+#ifdef CONFIG_MODULES
+ struct static_key_mod *next;
+#endif
+};
+
+# include <asm/jump_label.h>
+# define HAVE_JUMP_LABEL
+#endif /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */
+
+enum jump_label_type {
+ JUMP_LABEL_DISABLE = 0,
+ JUMP_LABEL_ENABLE,
+};
+
+struct module;
+
+#ifdef HAVE_JUMP_LABEL
+
+#define JUMP_LABEL_TRUE_BRANCH 1UL
+
+static
+inline struct jump_entry *jump_label_get_entries(struct static_key *key)
+{
+ return (struct jump_entry *)((unsigned long)key->entries
+ & ~JUMP_LABEL_TRUE_BRANCH);
+}
+
+static inline bool jump_label_get_branch_default(struct static_key *key)
+{
+ if ((unsigned long)key->entries & JUMP_LABEL_TRUE_BRANCH)
+ return true;
+ return false;
+}
+
+static __always_inline bool static_key_false(struct static_key *key)
+{
+ return arch_static_branch(key);
+}
+
+static __always_inline bool static_key_true(struct static_key *key)
+{
+ return !static_key_false(key);
+}
+
+extern struct jump_entry __start___jump_table[];
+extern struct jump_entry __stop___jump_table[];
+
+extern void jump_label_init(void);
+extern void jump_label_lock(void);
+extern void jump_label_unlock(void);
+extern void arch_jump_label_transform(struct jump_entry *entry,
+ enum jump_label_type type);
+extern void arch_jump_label_transform_static(struct jump_entry *entry,
+ enum jump_label_type type);
+extern int jump_label_text_reserved(void *start, void *end);
+extern void static_key_slow_inc(struct static_key *key);
+extern void static_key_slow_dec(struct static_key *key);
+extern void jump_label_apply_nops(struct module *mod);
+
+#define STATIC_KEY_INIT_TRUE ((struct static_key) \
+ { .enabled = ATOMIC_INIT(1), .entries = (void *)1 })
+#define STATIC_KEY_INIT_FALSE ((struct static_key) \
+ { .enabled = ATOMIC_INIT(0), .entries = (void *)0 })
+
+#else /* !HAVE_JUMP_LABEL */
+
+#include <linux/atomic.h>
+
+struct static_key {
+ atomic_t enabled;
+};
+
+static __always_inline void jump_label_init(void)
+{
+}
+
+static __always_inline bool static_key_false(struct static_key *key)
+{
+ if (unlikely(atomic_read(&key->enabled)) > 0)
+ return true;
+ return false;
+}
+
+static __always_inline bool static_key_true(struct static_key *key)
+{
+ if (likely(atomic_read(&key->enabled)) > 0)
+ return true;
+ return false;
+}
+
+static inline void static_key_slow_inc(struct static_key *key)
+{
+ atomic_inc(&key->enabled);
+}
+
+static inline void static_key_slow_dec(struct static_key *key)
+{
+ atomic_dec(&key->enabled);
+}
+
+static inline int jump_label_text_reserved(void *start, void *end)
+{
+ return 0;
+}
+
+static inline void jump_label_lock(void) {}
+static inline void jump_label_unlock(void) {}
+
+static inline int jump_label_apply_nops(struct module *mod)
+{
+ return 0;
+}
+
+#define STATIC_KEY_INIT_TRUE ((struct static_key) \
+ { .enabled = ATOMIC_INIT(1) })
+#define STATIC_KEY_INIT_FALSE ((struct static_key) \
+ { .enabled = ATOMIC_INIT(0) })
+
+#endif /* HAVE_JUMP_LABEL */
+
+#define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
+#define jump_label_enabled static_key_enabled
+
+static inline bool static_key_enabled(struct static_key *key)
+{
+ return (atomic_read(&key->enabled) > 0);
+}
+
+#endif /* _LINUX_JUMP_LABEL_BASE_H */
--
1.8.3.1
The struct static_key will have a atomic_t type member no matter
whether jump label is enabled or not. We would include linux/atomic.h
when jump label is not enabled. But it also does make sense to include
this header file when jump label is enabled.
Signed-off-by: Kevin Hao <[email protected]>
---
include/linux/jump_label_base.h | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/include/linux/jump_label_base.h b/include/linux/jump_label_base.h
index 20df08f..d5c8f4b 100644
--- a/include/linux/jump_label_base.h
+++ b/include/linux/jump_label_base.h
@@ -5,6 +5,8 @@
#include <linux/types.h>
#include <linux/compiler.h>
+#include <linux/atomic.h>
+
#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
struct static_key {
@@ -77,8 +79,6 @@ extern void jump_label_apply_nops(struct module *mod);
#else /* !HAVE_JUMP_LABEL */
-#include <linux/atomic.h>
-
struct static_key {
atomic_t enabled;
};
--
1.8.3.1
2013-08-25 15:15+0800, Kevin Hao:
> We plan to use the jump label in the cpu/mmu feature check on ppc.
> This will need to include the jump_label.h in several very basic header
> files of ppc which seems to be included by most of the other head
> files implicitly or explicitly. But in the current jump_label.h,
> it also include the "linux/workqueue.h" and this will cause recursive
> inclusion. In order to fix this, we choose to factor out the base
> part of jump_label.h to a separate header file and we can include
> that file instead of jump_label.h to avoid the recursive inclusion.
> No functional change.
"linux/workqueue.h" was included because of deferred keys and they are
split into "linux/jump_label_ratelimit.h" to solve the same problem in
paravirt ticket spinlock series.
(still in -next: 851cf6e7 jump_label: Split jumplabel ratelimit)
On Fri, Aug 30, 2013 at 06:37:33PM +0200, Radim Krčmář wrote:
> 2013-08-25 15:15+0800, Kevin Hao:
> > We plan to use the jump label in the cpu/mmu feature check on ppc.
> > This will need to include the jump_label.h in several very basic header
> > files of ppc which seems to be included by most of the other head
> > files implicitly or explicitly. But in the current jump_label.h,
> > it also include the "linux/workqueue.h" and this will cause recursive
> > inclusion. In order to fix this, we choose to factor out the base
> > part of jump_label.h to a separate header file and we can include
> > that file instead of jump_label.h to avoid the recursive inclusion.
> > No functional change.
>
> "linux/workqueue.h" was included because of deferred keys and they are
> split into "linux/jump_label_ratelimit.h" to solve the same problem in
> paravirt ticket spinlock series.
> (still in -next: 851cf6e7 jump_label: Split jumplabel ratelimit)
OK. I will respin a new version based on this patch. Thanks for the
information Radim.
Thanks,
Kevin