Hi,
Renames 'static_branch()' -> very_unlikely(), hopefully, to be more intuitive
as to what jump labels is about. I'm also introducing 'very_likely()', as
the analogue to very_unlikely(). Patch is against the -tip perf branch.
Thanks,
-Jason
Jason Baron (10):
jump label: Add a WARN() if jump label key count goes negative
jump label: fix compiler warning
jump label: introduce very_unlikely()
jump label: introduce very_likely()
perf: update to use very_unlikely()
tracepoints: update to use very_unlikely()
sched: update to use very_[un]likely()
kvm: update to use very_unlikely()
net: update to use very_unlikely()
jump label: Add docs better explaining the whole jump label mechanism
Documentation/jump-label.txt | 258 ++++++++++++++++++++++++++++++++++++++++++
arch/x86/kvm/mmu_audit.c | 2 +-
include/linux/jump_label.h | 99 +++++++++++++---
include/linux/netfilter.h | 2 +-
include/linux/perf_event.h | 6 +-
include/linux/tracepoint.h | 4 +-
include/net/sock.h | 2 +-
kernel/jump_label.c | 79 ++++++++-----
kernel/sched/core.c | 12 +-
kernel/sched/fair.c | 2 +-
kernel/sched/sched.h | 4 +-
kernel/tracepoint.c | 12 +-
net/core/dev.c | 8 +-
13 files changed, 416 insertions(+), 74 deletions(-)
create mode 100644 Documentation/jump-label.txt
--
1.7.7.5
While cross-compiling on sparc64, I found:
kernel/jump_label.c: In function 'jump_label_update':
kernel/jump_label.c:447:40: warning: cast from pointer to integer of different size [-Wpointer-to-int-cast]
Fix by casting to 'unsigned long'.
Signed-off-by: Jason Baron <[email protected]>
---
kernel/jump_label.c | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index ed9654f..543782e 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -424,7 +424,7 @@ static void jump_label_update(struct jump_label_key *key, int enable)
struct jump_entry *entry = key->entries, *stop = __stop___jump_table;
#ifdef CONFIG_MODULES
- struct module *mod = __module_address((jump_label_t)key);
+ struct module *mod = __module_address((unsigned long)key);
__jump_label_mod_update(key, enable);
--
1.7.7.5
The 'static_branch()' construct can be thought of as stronger
form of 'unlikely()'. Thus, let's rename 'static_branch()' ->
'very_unlikely()'. Leave around 'static_branch()' for a while
until all users are converted, but mark it as deprecated.
It would be nice to locate 'very_unlikely()' in include/linux/compiler.h,
where 'unlikely()' is defined, but I'm afraid there are too
many additional headers that I'd have to pull into compiler.h.
Signed-off-by: Jason Baron <[email protected]>
---
include/linux/jump_label.h | 14 ++++++++++++++
1 files changed, 14 insertions(+), 0 deletions(-)
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index f7c6958..563c781 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -72,6 +72,12 @@ struct module;
#define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL}
#endif
+static __always_inline bool very_unlikely(struct jump_label_key *key)
+{
+ return arch_static_branch(key);
+}
+
+/* Deprecated. Please use 'very_unlikely() instead. */
static __always_inline bool static_branch(struct jump_label_key *key)
{
return arch_static_branch(key);
@@ -114,6 +120,14 @@ struct jump_label_key_deferred {
struct jump_label_key key;
};
+static __always_inline bool very_unlikely(struct jump_label_key *key)
+{
+ if (unlikely(atomic_read(&key->enabled)))
+ return true;
+ return false;
+}
+
+/* Deprecated. Please use 'very_unlikely() instead. */
static __always_inline bool static_branch(struct jump_label_key *key)
{
if (unlikely(atomic_read(&key->enabled)))
--
1.7.7.5
The current very_unlikely() construct, assumes that the branch will be
disabled by default. This means we have a single no-op in the straight line
path, and when the branch is made true we patch the no-op with a jump.
There are cases (sched feat code), where we want the branch to default to
true, so that the straight line code is the true branch, and the false
branch is enabled via a jump.
In order to implement this while having, jump_label_inc(), and
jump_label_dec() retain their current meaning, we have to store the
initial branch state. I'm using the lowest bit of the 'entries'
pointer in the jump_label_key struct, since this points to memory
that has to be aligned to the arch pointer size. We could have stored this
in the 'struct jump_entry' data structure, but I wanted to avoid
adding additional space overhead.
Thus, the new API is initialized as:
struct jump_label_key true_key = JUMP_LABEL_INIT_TRUE;
or
struct jump_label_key false_key = JUMP_LABEL_INIT_FALSE;
Leaving out an initialization, defaults to false, as in:
struct jump_label_key uninitialized_key;
Then, for the branches we have:
very_unlikely(&false_key);
or
very_likely(&true_key);
And finally, jump_label_inc(&key), jump_label_dec(&key) are unchanged -
'jump_label_inc()' means 'make true', and jump_label_dec()' means make false,
with the expected increment/decrement counting.
Thus, you must use 'true_key' with a 'very_likely()' branch, and a
'false_key' or an 'uninitialized_key' with a 'very_unlikely()' branch.
Mixing different branches with the same jump_label_key is not allowed.
I've left the old static_branch(), (which is the same as the new
very_unlikely()), at least until we've converted over all in-tree and
pending users.
Signed-off-by: Jason Baron <[email protected]>
[ Simplified code using jump_label_type(); removed CONFIG_MODULE
special casing, since C99 initialization inits all unmentioned
members to 0; obfuscated key->entries initialization. ]
Signed-off-by: Peter Zijlstra <[email protected]>
Link: http://lkml.kernel.org/r/742699c7e855a61f881c501ea415a43a5da9cd04.1324493360.git.jbaron@redhat.com
---
include/linux/jump_label.h | 89 ++++++++++++++++++++++++++++++++++----------
kernel/jump_label.c | 72 ++++++++++++++++++++++-------------
2 files changed, 115 insertions(+), 46 deletions(-)
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index 563c781..fdf6fff 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -9,10 +9,10 @@
*
* Jump labels provide an interface to generate dynamic branches using
* self-modifying code. Assuming toolchain and architecture support the result
- * of a "if (static_branch(&key))" statement is a unconditional branch (which
+ * of a "if (very_unlikely(&key))" statement is a unconditional branch (which
* defaults to false - and the true block is placed out of line).
*
- * However at runtime we can change the 'static' branch target using
+ * However at runtime we can change the branch target using
* jump_label_{inc,dec}(). These function as a 'reference' count on the key
* object and for as long as there are references all branches referring to
* that particular key will point to the (out of line) true block.
@@ -31,7 +31,21 @@
*
* Lacking toolchain and or architecture support, it falls back to a simple
* conditional branch.
- */
+ *
+ * struct jump_label_key my_key = JUMP_LABEL_INIT_TRUE;
+ *
+ * if (very_likely(&my_key)) {
+ * }
+ *
+ * will result in the true case being in-line and starts the key with a single
+ * reference. Mixing very_likely() and very_unlikely() on the same key is not
+ * allowed.
+ *
+ * Not initializing the key (static data is initialized to 0s anyway) is the
+ * same as using JUMP_LABEL_INIT_FALSE and very_unlikely() is
+ * equivalent with static_branch().
+ *
+*/
#include <linux/types.h>
#include <linux/compiler.h>
@@ -41,6 +55,7 @@
struct jump_label_key {
atomic_t enabled;
+/* Set lsb bit to 1 if branch is default true, 0 ot */
struct jump_entry *entries;
#ifdef CONFIG_MODULES
struct jump_label_mod *next;
@@ -66,17 +81,32 @@ struct module;
#ifdef HAVE_JUMP_LABEL
-#ifdef CONFIG_MODULES
-#define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL, NULL}
-#else
-#define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL}
-#endif
+#define JUMP_LABEL_TRUE_BRANCH 1UL
+
+static
+inline struct jump_entry *jump_label_get_entries(struct jump_label_key *key)
+{
+ return (struct jump_entry *)((unsigned long)key->entries
+ & ~JUMP_LABEL_TRUE_BRANCH);
+}
+
+static inline bool jump_label_get_branch_default(struct jump_label_key *key)
+{
+ if ((unsigned long)key->entries & JUMP_LABEL_TRUE_BRANCH)
+ return true;
+ return false;
+}
static __always_inline bool very_unlikely(struct jump_label_key *key)
{
return arch_static_branch(key);
}
+static __always_inline bool very_likely(struct jump_label_key *key)
+{
+ return !very_unlikely(key);
+}
+
/* Deprecated. Please use 'very_unlikely() instead. */
static __always_inline bool static_branch(struct jump_label_key *key)
{
@@ -97,17 +127,20 @@ extern int jump_label_text_reserved(void *start, void *end);
extern void jump_label_inc(struct jump_label_key *key);
extern void jump_label_dec(struct jump_label_key *key);
extern void jump_label_dec_deferred(struct jump_label_key_deferred *key);
-extern bool jump_label_enabled(struct jump_label_key *key);
+extern bool jump_label_true(struct jump_label_key *key);
extern void jump_label_apply_nops(struct module *mod);
-extern void jump_label_rate_limit(struct jump_label_key_deferred *key,
- unsigned long rl);
+extern void
+jump_label_rate_limit(struct jump_label_key_deferred *key, unsigned long rl);
+
+#define JUMP_LABEL_INIT_TRUE ((struct jump_label_key) \
+ { .enabled = ATOMIC_INIT(1), .entries = (void *)1 })
+#define JUMP_LABEL_INIT_FALSE ((struct jump_label_key) \
+ { .enabled = ATOMIC_INIT(0), .entries = (void *)0 })
#else /* !HAVE_JUMP_LABEL */
#include <linux/atomic.h>
-#define JUMP_LABEL_INIT {ATOMIC_INIT(0)}
-
struct jump_label_key {
atomic_t enabled;
};
@@ -122,7 +155,14 @@ struct jump_label_key_deferred {
static __always_inline bool very_unlikely(struct jump_label_key *key)
{
- if (unlikely(atomic_read(&key->enabled)))
+ if (unlikely(atomic_read(&key->enabled)) > 0)
+ return true;
+ return false;
+}
+
+static __always_inline bool very_likely(struct jump_label_key *key)
+{
+ if (likely(atomic_read(&key->enabled)) > 0)
return true;
return false;
}
@@ -130,7 +170,7 @@ static __always_inline bool very_unlikely(struct jump_label_key *key)
/* Deprecated. Please use 'very_unlikely() instead. */
static __always_inline bool static_branch(struct jump_label_key *key)
{
- if (unlikely(atomic_read(&key->enabled)))
+ if (unlikely(atomic_read(&key->enabled)) > 0)
return true;
return false;
}
@@ -158,9 +198,9 @@ static inline int jump_label_text_reserved(void *start, void *end)
static inline void jump_label_lock(void) {}
static inline void jump_label_unlock(void) {}
-static inline bool jump_label_enabled(struct jump_label_key *key)
+static inline bool jump_label_true(struct jump_label_key *key)
{
- return !!atomic_read(&key->enabled);
+ return (atomic_read(&key->enabled) > 0);
}
static inline int jump_label_apply_nops(struct module *mod)
@@ -168,13 +208,22 @@ static inline int jump_label_apply_nops(struct module *mod)
return 0;
}
-static inline void jump_label_rate_limit(struct jump_label_key_deferred *key,
+static inline void
+jump_label_rate_limit(struct jump_label_key_deferred *key,
unsigned long rl)
{
}
+
+#define JUMP_LABEL_INIT_TRUE ((struct jump_label_key) \
+ { .enabled = ATOMIC_INIT(1) })
+#define JUMP_LABEL_INIT_FALSE ((struct jump_label_key) \
+ { .enabled = ATOMIC_INIT(0) })
+
#endif /* HAVE_JUMP_LABEL */
-#define jump_label_key_enabled ((struct jump_label_key){ .enabled = ATOMIC_INIT(1), })
-#define jump_label_key_disabled ((struct jump_label_key){ .enabled = ATOMIC_INIT(0), })
+#define jump_label_key_enabled JUMP_LABEL_INIT_TRUE
+#define jump_label_key_disabled JUMP_LABEL_INIT_FALSE
+#define JUMP_LABEL_INIT JUMP_LABEL_INIT_FALSE
+#define jump_label_enabled jump_label_true
#endif /* _LINUX_JUMP_LABEL_H */
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 543782e..2b55284 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -29,10 +29,11 @@ void jump_label_unlock(void)
mutex_unlock(&jump_label_mutex);
}
-bool jump_label_enabled(struct jump_label_key *key)
+bool jump_label_true(struct jump_label_key *key)
{
- return !!atomic_read(&key->enabled);
+ return (atomic_read(&key->enabled) > 0);
}
+EXPORT_SYMBOL_GPL(jump_label_true);
static int jump_label_cmp(const void *a, const void *b)
{
@@ -66,8 +67,12 @@ void jump_label_inc(struct jump_label_key *key)
return;
jump_label_lock();
- if (atomic_read(&key->enabled) == 0)
- jump_label_update(key, JUMP_LABEL_ENABLE);
+ if (atomic_read(&key->enabled) == 0) {
+ if (!jump_label_get_branch_default(key))
+ jump_label_update(key, JUMP_LABEL_ENABLE);
+ else
+ jump_label_update(key, JUMP_LABEL_DISABLE);
+ }
atomic_inc(&key->enabled);
jump_label_unlock();
}
@@ -85,12 +90,14 @@ static void __jump_label_dec(struct jump_label_key *key,
if (rate_limit) {
atomic_inc(&key->enabled);
schedule_delayed_work(work, rate_limit);
- } else
- jump_label_update(key, JUMP_LABEL_DISABLE);
-
+ } else {
+ if (!jump_label_get_branch_default(key))
+ jump_label_update(key, JUMP_LABEL_DISABLE);
+ else
+ jump_label_update(key, JUMP_LABEL_ENABLE);
+ }
jump_label_unlock();
}
-EXPORT_SYMBOL_GPL(jump_label_dec);
static void jump_label_update_timeout(struct work_struct *work)
{
@@ -103,12 +110,13 @@ void jump_label_dec(struct jump_label_key *key)
{
__jump_label_dec(key, 0, NULL);
}
+EXPORT_SYMBOL_GPL(jump_label_dec);
void jump_label_dec_deferred(struct jump_label_key_deferred *key)
{
__jump_label_dec(&key->key, key->timeout, &key->work);
}
-
+EXPORT_SYMBOL_GPL(jump_label_dec_deferred);
void jump_label_rate_limit(struct jump_label_key_deferred *key,
unsigned long rl)
@@ -170,6 +178,17 @@ static void __jump_label_update(struct jump_label_key *key,
}
}
+static enum jump_label_type jump_label_type(struct jump_label_key *key)
+{
+ bool true_branch = jump_label_get_branch_default(key);
+ bool state = jump_label_true(key);
+
+ if ((!true_branch && state) || (true_branch && !state))
+ return JUMP_LABEL_ENABLE;
+
+ return JUMP_LABEL_DISABLE;
+}
+
void __init jump_label_init(void)
{
struct jump_entry *iter_start = __start___jump_table;
@@ -184,13 +203,15 @@ void __init jump_label_init(void)
struct jump_label_key *iterk;
iterk = (struct jump_label_key *)(unsigned long)iter->key;
- arch_jump_label_transform_static(iter, jump_label_enabled(iterk) ?
- JUMP_LABEL_ENABLE : JUMP_LABEL_DISABLE);
+ arch_jump_label_transform_static(iter, jump_label_type(iterk));
if (iterk == key)
continue;
key = iterk;
- key->entries = iter;
+ /*
+ * Set key->entries to iter, but preserve JUMP_LABEL_TRUE_BRANCH.
+ */
+ *((unsigned long *)&key->entries) += (unsigned long)iter;
#ifdef CONFIG_MODULES
key->next = NULL;
#endif
@@ -254,11 +275,7 @@ void jump_label_apply_nops(struct module *mod)
return;
for (iter = iter_start; iter < iter_stop; iter++) {
- struct jump_label_key *iterk;
-
- iterk = (struct jump_label_key *)(unsigned long)iter->key;
- arch_jump_label_transform_static(iter, jump_label_enabled(iterk) ?
- JUMP_LABEL_ENABLE : JUMP_LABEL_DISABLE);
+ arch_jump_label_transform_static(iter, JUMP_LABEL_DISABLE);
}
}
@@ -277,28 +294,30 @@ static int jump_label_add_module(struct module *mod)
jump_label_sort_entries(iter_start, iter_stop);
for (iter = iter_start; iter < iter_stop; iter++) {
- if (iter->key == (jump_label_t)(unsigned long)key)
- continue;
+ struct jump_label_key *iterk;
- key = (struct jump_label_key *)(unsigned long)iter->key;
+ iterk = (struct jump_label_key *)(unsigned long)iter->key;
+ if (iterk == key)
+ continue;
+ key = iterk;
if (__module_address(iter->key) == mod) {
- atomic_set(&key->enabled, 0);
- key->entries = iter;
+ /*
+ * Set key->entries to iter, but preserve JUMP_LABEL_TRUE_BRANCH.
+ */
+ *((unsigned long *)&key->entries) += (unsigned long)iter;
key->next = NULL;
continue;
}
-
jlm = kzalloc(sizeof(struct jump_label_mod), GFP_KERNEL);
if (!jlm)
return -ENOMEM;
-
jlm->mod = mod;
jlm->entries = iter;
jlm->next = key->next;
key->next = jlm;
- if (jump_label_enabled(key))
+ if (jump_label_type(key) == JUMP_LABEL_ENABLE)
__jump_label_update(key, iter, iter_stop, JUMP_LABEL_ENABLE);
}
@@ -421,7 +440,8 @@ int jump_label_text_reserved(void *start, void *end)
static void jump_label_update(struct jump_label_key *key, int enable)
{
- struct jump_entry *entry = key->entries, *stop = __stop___jump_table;
+ struct jump_entry *stop = __stop___jump_table;
+ struct jump_entry *entry = jump_label_get_entries(key);
#ifdef CONFIG_MODULES
struct module *mod = __module_address((unsigned long)key);
--
1.7.7.5
Update tracepoins with api change: static_branch() -> very_unlikely().
Cc: Steven Rostedt <[email protected]>
Cc: Frederic Weisbecker <[email protected]>
Cc: Mathieu Desnoyers <[email protected]>
Signed-off-by: Jason Baron <[email protected]>
---
include/linux/tracepoint.h | 4 ++--
kernel/tracepoint.c | 12 ++++++------
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h
index df0a779..282cf59 100644
--- a/include/linux/tracepoint.h
+++ b/include/linux/tracepoint.h
@@ -143,7 +143,7 @@ static inline void tracepoint_synchronize_unregister(void)
extern struct tracepoint __tracepoint_##name; \
static inline void trace_##name(proto) \
{ \
- if (static_branch(&__tracepoint_##name.key)) \
+ if (very_unlikely(&__tracepoint_##name.key)) \
__DO_TRACE(&__tracepoint_##name, \
TP_PROTO(data_proto), \
TP_ARGS(data_args), \
@@ -176,7 +176,7 @@ static inline void tracepoint_synchronize_unregister(void)
__attribute__((section("__tracepoints_strings"))) = #name; \
struct tracepoint __tracepoint_##name \
__attribute__((section("__tracepoints"))) = \
- { __tpstrtab_##name, JUMP_LABEL_INIT, reg, unreg, NULL };\
+ { __tpstrtab_##name, JUMP_LABEL_INIT_FALSE, reg, unreg, NULL };\
static struct tracepoint * const __tracepoint_ptr_##name __used \
__attribute__((section("__tracepoints_ptrs"))) = \
&__tracepoint_##name;
diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c
index f1539de..ad32493 100644
--- a/kernel/tracepoint.c
+++ b/kernel/tracepoint.c
@@ -256,9 +256,9 @@ static void set_tracepoint(struct tracepoint_entry **entry,
{
WARN_ON(strcmp((*entry)->name, elem->name) != 0);
- if (elem->regfunc && !jump_label_enabled(&elem->key) && active)
+ if (elem->regfunc && !jump_label_true(&elem->key) && active)
elem->regfunc();
- else if (elem->unregfunc && jump_label_enabled(&elem->key) && !active)
+ else if (elem->unregfunc && jump_label_true(&elem->key) && !active)
elem->unregfunc();
/*
@@ -269,9 +269,9 @@ static void set_tracepoint(struct tracepoint_entry **entry,
* is used.
*/
rcu_assign_pointer(elem->funcs, (*entry)->funcs);
- if (active && !jump_label_enabled(&elem->key))
+ if (active && !jump_label_true(&elem->key))
jump_label_inc(&elem->key);
- else if (!active && jump_label_enabled(&elem->key))
+ else if (!active && jump_label_true(&elem->key))
jump_label_dec(&elem->key);
}
@@ -283,10 +283,10 @@ static void set_tracepoint(struct tracepoint_entry **entry,
*/
static void disable_tracepoint(struct tracepoint *elem)
{
- if (elem->unregfunc && jump_label_enabled(&elem->key))
+ if (elem->unregfunc && jump_label_true(&elem->key))
elem->unregfunc();
- if (jump_label_enabled(&elem->key))
+ if (jump_label_true(&elem->key))
jump_label_dec(&elem->key);
rcu_assign_pointer(elem->funcs, NULL);
}
--
1.7.7.5
Add better documentation for jump labels.
Signed-off-by: Jason Baron <[email protected]>
---
Documentation/jump-label.txt | 258 ++++++++++++++++++++++++++++++++++++++++++
1 files changed, 258 insertions(+), 0 deletions(-)
create mode 100644 Documentation/jump-label.txt
diff --git a/Documentation/jump-label.txt b/Documentation/jump-label.txt
new file mode 100644
index 0000000..210b760
--- /dev/null
+++ b/Documentation/jump-label.txt
@@ -0,0 +1,258 @@
+ Jump Label
+ ----------
+
+By: Jason Baron <[email protected]>
+
+
+1) Motivation
+
+
+Currently, tracepoints are implemented using a conditional. The conditional
+check requires checking a global variable for each tracepoint. Although
+the overhead of this check is small, it increases when the memory cache comes
+under pressure (memory cache lines for these global variables may be shared
+with other memory accesses). As we increase the number of tracepoints in the
+kernel this overhead may become more of an issue. In addition, tracepoints are
+often dormant (disabled) and provide no direct kernel functionality. Thus, it
+is highly desirable to reduce their impact as much as possible. Although
+tracepoints are the original motivation for this work, other kernel code paths
+should be able to make use of the jump label optimization.
+
+
+2) Solution
+
+
+gcc (v4.5) adds a new 'asm goto' statement that allows branching to a label:
+
+http://gcc.gnu.org/ml/gcc-patches/2009-07/msg01556.html
+
+Using the 'asm goto', we can create branches that are either taken or not taken
+by default, without the need to check memory. Then, at run-time, we can patch
+the branch site to change the branch direction.
+
+For example, if we have a simple branch that is disabled by default:
+
+ if (very_unlikely(&jump_key))
+ printk("I am the true branch\n");
+
+Thus, by default the 'printk' will not be emitted. And the code generated will
+consist of a single atomic 'no-op' instruction (5 bytes on x86), in the
+straight-line code path. When the branch is 'flipped', we will patch the 'no-op'
+in the straight-line codepath with a 'jump' instruction to the out-of-line true
+branch. Thus, changing branch direction is expensive but branch selection is
+basically 'free'. That is the basic tradeoff of this optimization.
+
+
+3) Jump label API, usage and examples:
+
+
+In order to use a jump label you much first define a key:
+
+ struct jump_label_key jump_key;
+
+Which is initialized as:
+
+ struct jump_label_key jump_key = JUMP_LABEL_INIT_TRUE;
+
+or:
+
+ struct jump_label_Key jump_key = JUMP_LABEL_INIT_FALSE;
+
+If the key is not initialized, it is default false. The
+'struct jump_label_key', must be a 'global'. That is, it can't be allocated on
+the stack or dynamically allocated at run-time.
+
+The key is then used in code as:
+
+ if (very_unlikely(&jump_key))
+ do unlikely code
+ else
+ do likely code
+
+Or:
+
+ if (very_likely(&jump_key))
+ do likely code
+ else
+ do unlikely code
+
+A key that is initialized via 'JUMP_LABEL_INIT_FALSE', must be used in a
+'very_unlikely()' construct. Likewise, a key initialized via
+'JUMP_LABEL_INIT_TRUE' must be used in a 'very_likely()' construct.
+A single key can be used in many branches, but all the branches must match
+the way that the key has been initialized.
+
+The branch(es) can then be switched via:
+
+ jump_label_inc(&jump_key);
+ jump_label_dec(&jump_key);
+
+Thus, 'jump_label_inc()' means 'make the branch true', and
+'jump_label_dec()' means 'make the the branch false' with appropriate reference
+counting. For example, if the key is initialized true, a jump_label_dec(), will
+switch the branch to false. And a subsequent jump_label_inc(), will change
+the branch back to true. Likewise, if the key is initialized false, a
+'jump_label_inc()', will change the branch to true. And then a
+'jump_label_dec()', will again make the branch false.
+
+An example usage in the kernel is the implementation of tracepoints:
+
+ static inline void trace_##name(proto) \
+ { \
+ if (very_unlikely(&__tracepoint_##name.key)) \
+ __DO_TRACE(&__tracepoint_##name, \
+ TP_PROTO(data_proto), \
+ TP_ARGS(data_args), \
+ TP_CONDITION(cond)); \
+ }
+
+Tracepoints are disabled by default, and can be placed in performance critical
+pieces of the kernel. Thus, by using a jump labels, the tracepoints can have
+absolutely minimal impact when not in use.
+
+
+4) Architecture interface
+
+
+There are a few functions and macros that architectures must implement in order
+to take advantage of this optimization. If there is no architecture support, we
+simply fall back to a traditional, load, test, and jump sequence.
+
+* select HAVE_ARCH_JUMP_LABEL, see: arch/x86/Kconfig
+
+* #define JUMP_LABEL_NOP_SIZE, see: arch/x86/include/asm/jump_label.h
+
+* __always_inline bool arch_static_branch(struct jump_label_key *key), see:
+ arch/x86/include/asm/jump_label.h
+
+* void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type),
+ see: arch/x86/kernel/jump_label.c
+
+* __init_or_module void arch_jump_label_transform_static(struct jump_entry *entry, enum jump_label_type type),
+ see: arch/x86/kernel/jump_label.c
+
+
+* struct jump_entry, see: arch/x86/include/asm/jump_label.h
+
+
+5) Jump label analysis, results (x86_64):
+
+
+As an example, let's add the following branch to 'getppid()', such that the
+system call now looks like:
+
+SYSCALL_DEFINE0(getppid)
+{
+ int pid;
+
++ if (very_unlikely(&key))
++ printk("I am the true branch\n");
+
+ rcu_read_lock();
+ pid = task_tgid_vnr(rcu_dereference(current->real_parent));
+ rcu_read_unlock();
+
+ return pid;
+}
+
+The resulting instructions with jump labels is:
+
+ffffffff81044290 <sys_getppid>:
+ffffffff81044290: 55 push %rbp
+ffffffff81044291: 48 89 e5 mov %rsp,%rbp
+ffffffff81044294: e9 00 00 00 00 jmpq ffffffff81044299 <sys_getppid+0x9>
+ffffffff81044299: 65 48 8b 04 25 c0 b6 mov %gs:0xb6c0,%rax
+ffffffff810442a0: 00 00
+ffffffff810442a2: 48 8b 80 80 02 00 00 mov 0x280(%rax),%rax
+ffffffff810442a9: 48 8b 80 b0 02 00 00 mov 0x2b0(%rax),%rax
+ffffffff810442b0: 48 8b b8 e8 02 00 00 mov 0x2e8(%rax),%rdi
+ffffffff810442b7: e8 f4 d9 00 00 callq ffffffff81051cb0 <pid_vnr>
+ffffffff810442bc: 5d pop %rbp
+ffffffff810442bd: 48 98 cltq
+ffffffff810442bf: c3 retq
+ffffffff810442c0: 48 c7 c7 e3 54 98 81 mov $0xffffffff819854e3,%rdi
+ffffffff810442c7: 31 c0 xor %eax,%eax
+ffffffff810442c9: e8 71 13 6d 00 callq ffffffff8171563f <printk>
+ffffffff810442ce: eb c9 jmp ffffffff81044299 <sys_getppid+0x9>
+
+Without jump labels it looks like:
+
+ffffffff810441f0 <sys_getppid>:
+ffffffff810441f0: 8b 05 8a 52 d8 00 mov 0xd8528a(%rip),%eax # ffffffff81dc9480 <key>
+ffffffff810441f6: 55 push %rbp
+ffffffff810441f7: 48 89 e5 mov %rsp,%rbp
+ffffffff810441fa: 85 c0 test %eax,%eax
+ffffffff810441fc: 75 27 jne ffffffff81044225 <sys_getppid+0x35>
+ffffffff810441fe: 65 48 8b 04 25 c0 b6 mov %gs:0xb6c0,%rax
+ffffffff81044205: 00 00
+ffffffff81044207: 48 8b 80 80 02 00 00 mov 0x280(%rax),%rax
+ffffffff8104420e: 48 8b 80 b0 02 00 00 mov 0x2b0(%rax),%rax
+ffffffff81044215: 48 8b b8 e8 02 00 00 mov 0x2e8(%rax),%rdi
+ffffffff8104421c: e8 2f da 00 00 callq ffffffff81051c50 <pid_vnr>
+ffffffff81044221: 5d pop %rbp
+ffffffff81044222: 48 98 cltq
+ffffffff81044224: c3 retq
+ffffffff81044225: 48 c7 c7 13 53 98 81 mov $0xffffffff81985313,%rdi
+ffffffff8104422c: 31 c0 xor %eax,%eax
+ffffffff8104422e: e8 60 0f 6d 00 callq ffffffff81715193 <printk>
+ffffffff81044233: eb c9 jmp ffffffff810441fe <sys_getppid+0xe>
+ffffffff81044235: 66 66 2e 0f 1f 84 00 data32 nopw %cs:0x0(%rax,%rax,1)
+ffffffff8104423c: 00 00 00 00
+
+Thus, the disable jump label case adds a 'mov', 'test' and 'jne' instruction
+vs. the jump label case just has a 'no-op' or 'jmp 0'. (The jmp 0, is patched
+to a 5 byte atomic no-op instruction at boot-time.) Thus, the disabled jump
+label case adds:
+
+6 (mov) + 2 (test) + 2 (jne) = 10 - 5 (5 byte jump 0) = 5 addition bytes.
+
+If we then include the padding bytes, the jump label code saves, 16 total
+bytes of instruction memory for this small fucntion. In this case the
+non-jump label function is 80 bytes long. Thus, we have have saved 20% of the
+instruction footprint. We can in fact improve this even further, since the
+5-byte no-op really can be a 2-byte no-op since we can reach the branch with
+a 2-byte jmp. However, we have not yet implemented optimal no-op sizes (they
+are currently hard-coded).
+
+Since there are a number of jump labels in the scheduler paths, 'pipe-test',
+found here: http://thread.gmane.org/gmane.linux.kernel/1129232/focus=1129389,
+can be used to show the performance improvement. Testing done on 3.3.0-rc2:
+
+jump label disabled:
+
+ Performance counter stats for 'bash -c /tmp/pipe-test' (50 runs):
+
+ 855.700314 task-clock # 0.534 CPUs utilized ( +- 0.11% )
+ 200,003 context-switches # 0.234 M/sec ( +- 0.00% )
+ 0 CPU-migrations # 0.000 M/sec ( +- 39.58% )
+ 487 page-faults # 0.001 M/sec ( +- 0.02% )
+ 1,474,374,262 cycles # 1.723 GHz ( +- 0.17% )
+ <not supported> stalled-cycles-frontend
+ <not supported> stalled-cycles-backend
+ 1,178,049,567 instructions # 0.80 insns per cycle ( +- 0.06% )
+ 208,368,926 branches # 243.507 M/sec ( +- 0.06% )
+ 5,569,188 branch-misses # 2.67% of all branches ( +- 0.54% )
+
+ 1.601607384 seconds time elapsed ( +- 0.07% )
+
+jump label enabled:
+
+ Performance counter stats for 'bash -c /tmp/pipe-test' (50 runs):
+
+ 841.043185 task-clock # 0.533 CPUs utilized ( +- 0.12% )
+ 200,004 context-switches # 0.238 M/sec ( +- 0.00% )
+ 0 CPU-migrations # 0.000 M/sec ( +- 40.87% )
+ 487 page-faults # 0.001 M/sec ( +- 0.05% )
+ 1,432,559,428 cycles # 1.703 GHz ( +- 0.18% )
+ <not supported> stalled-cycles-frontend
+ <not supported> stalled-cycles-backend
+ 1,175,363,994 instructions # 0.82 insns per cycle ( +- 0.04% )
+ 206,859,359 branches # 245.956 M/sec ( +- 0.04% )
+ 4,884,119 branch-misses # 2.36% of all branches ( +- 0.85% )
+
+ 1.579384366 seconds time elapsed
+
+The percentage of saved branches is .7%, and we've saved 12% on 'branch-misses'.
+This is where we would expect to get the most savings, since this optimization
+is about reducing the number of branches. In addition, we've saved .2% on
+instructions, and 2.8% on cycles and 1.4% on elapsed time.
--
1.7.7.5
Update kvm with api change: static_branch() -> very_unlikely().
Cc: Avi Kivity <[email protected]>
Cc: Xiao Guangrong <[email protected]>
Signed-off-by: Jason Baron <[email protected]>
---
arch/x86/kvm/mmu_audit.c | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/arch/x86/kvm/mmu_audit.c b/arch/x86/kvm/mmu_audit.c
index fe15dcc..7709e02 100644
--- a/arch/x86/kvm/mmu_audit.c
+++ b/arch/x86/kvm/mmu_audit.c
@@ -250,7 +250,7 @@ static void __kvm_mmu_audit(struct kvm_vcpu *vcpu, int point)
static inline void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point)
{
- if (static_branch((&mmu_audit_key)))
+ if (very_unlikely((&mmu_audit_key)))
__kvm_mmu_audit(vcpu, point);
}
--
1.7.7.5
Update scheduler with api change: static_branch() -> very_unlikely().
Cc: Peter Zijlstra <[email protected]>
Cc: Paul Turner <[email protected]>
Cc: Ingo Molnar <[email protected]>
Signed-off-by: Jason Baron <[email protected]>
---
include/linux/jump_label.h | 2 --
kernel/sched/core.c | 12 ++++++------
kernel/sched/fair.c | 2 +-
kernel/sched/sched.h | 4 ++--
4 files changed, 9 insertions(+), 11 deletions(-)
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index fdf6fff..757d8dc 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -221,8 +221,6 @@ jump_label_rate_limit(struct jump_label_key_deferred *key,
#endif /* HAVE_JUMP_LABEL */
-#define jump_label_key_enabled JUMP_LABEL_INIT_TRUE
-#define jump_label_key_disabled JUMP_LABEL_INIT_FALSE
#define JUMP_LABEL_INIT JUMP_LABEL_INIT_FALSE
#define jump_label_enabled jump_label_true
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 5255c9d..a357dbf 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -162,8 +162,8 @@ static int sched_feat_show(struct seq_file *m, void *v)
#ifdef HAVE_JUMP_LABEL
-#define jump_label_key__true jump_label_key_enabled
-#define jump_label_key__false jump_label_key_disabled
+#define jump_label_key__true JUMP_LABEL_INIT_TRUE
+#define jump_label_key__false JUMP_LABEL_INIT_FALSE
#define SCHED_FEAT(name, enabled) \
jump_label_key__##enabled ,
@@ -176,13 +176,13 @@ struct jump_label_key sched_feat_keys[__SCHED_FEAT_NR] = {
static void sched_feat_disable(int i)
{
- if (jump_label_enabled(&sched_feat_keys[i]))
+ if (jump_label_true(&sched_feat_keys[i]))
jump_label_dec(&sched_feat_keys[i]);
}
static void sched_feat_enable(int i)
{
- if (!jump_label_enabled(&sched_feat_keys[i]))
+ if (!jump_label_true(&sched_feat_keys[i]))
jump_label_inc(&sched_feat_keys[i]);
}
#else
@@ -894,7 +894,7 @@ static void update_rq_clock_task(struct rq *rq, s64 delta)
delta -= irq_delta;
#endif
#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
- if (static_branch((¶virt_steal_rq_enabled))) {
+ if (very_unlikely((¶virt_steal_rq_enabled))) {
u64 st;
steal = paravirt_steal_clock(cpu_of(rq));
@@ -2756,7 +2756,7 @@ void account_idle_time(cputime_t cputime)
static __always_inline bool steal_account_process_tick(void)
{
#ifdef CONFIG_PARAVIRT
- if (static_branch(¶virt_steal_enabled)) {
+ if (very_unlikely(¶virt_steal_enabled)) {
u64 steal, st = 0;
steal = paravirt_steal_clock(smp_processor_id());
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 7c6414f..67206ae 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1403,7 +1403,7 @@ static struct jump_label_key __cfs_bandwidth_used;
static inline bool cfs_bandwidth_used(void)
{
- return static_branch(&__cfs_bandwidth_used);
+ return very_unlikely(&__cfs_bandwidth_used);
}
void account_cfs_bandwidth_used(int enabled, int was_enabled)
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 98c0c26..addeb9e 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -632,12 +632,12 @@ enum {
#if defined(CONFIG_SCHED_DEBUG) && defined(HAVE_JUMP_LABEL)
static __always_inline bool static_branch__true(struct jump_label_key *key)
{
- return likely(static_branch(key)); /* Not out of line branch. */
+ return very_likely(key); /* Not out of line branch. */
}
static __always_inline bool static_branch__false(struct jump_label_key *key)
{
- return unlikely(static_branch(key)); /* Out of line branch. */
+ return very_unlikely(key); /* Out of line branch. */
}
#define SCHED_FEAT(name, enabled) \
--
1.7.7.5
Update perf functions with api change: static_branch() -> very_unlikely().
Cc: Peter Zijlstra <[email protected]>
Cc: Paul Mackerras <[email protected]>
Cc: Ingo Molnar <[email protected]>
Cc: Arnaldo Carvalho de Melo <[email protected]>
Signed-off-by: Jason Baron <[email protected]>
---
include/linux/perf_event.h | 6 +++---
1 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 412b790..041d02b 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -1066,7 +1066,7 @@ perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
{
struct pt_regs hot_regs;
- if (static_branch(&perf_swevent_enabled[event_id])) {
+ if (very_unlikely(&perf_swevent_enabled[event_id])) {
if (!regs) {
perf_fetch_caller_regs(&hot_regs);
regs = &hot_regs;
@@ -1080,7 +1080,7 @@ extern struct jump_label_key_deferred perf_sched_events;
static inline void perf_event_task_sched_in(struct task_struct *prev,
struct task_struct *task)
{
- if (static_branch(&perf_sched_events.key))
+ if (very_unlikely(&perf_sched_events.key))
__perf_event_task_sched_in(prev, task);
}
@@ -1089,7 +1089,7 @@ static inline void perf_event_task_sched_out(struct task_struct *prev,
{
perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, NULL, 0);
- if (static_branch(&perf_sched_events.key))
+ if (very_unlikely(&perf_sched_events.key))
__perf_event_task_sched_out(prev, next);
}
--
1.7.7.5
The count on a jump label key should never go negative. Add a WARN() to
check for this condition.
Cc: Gleb Natapov <[email protected]>
Signed-off-by: Jason Baron <[email protected]>
---
kernel/jump_label.c | 5 ++++-
1 files changed, 4 insertions(+), 1 deletions(-)
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 01d3b70..ed9654f 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -76,8 +76,11 @@ EXPORT_SYMBOL_GPL(jump_label_inc);
static void __jump_label_dec(struct jump_label_key *key,
unsigned long rate_limit, struct delayed_work *work)
{
- if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex))
+ if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) {
+ WARN(atomic_read(&key->enabled) < 0,
+ "jump label: negative count!\n");
return;
+ }
if (rate_limit) {
atomic_inc(&key->enabled);
--
1.7.7.5
On 02/21/2012 12:02 PM, Jason Baron wrote:
> Hi,
>
> Renames 'static_branch()' -> very_unlikely(), hopefully, to be more intuitive
> as to what jump labels is about. I'm also introducing 'very_likely()', as
> the analogue to very_unlikely(). Patch is against the -tip perf branch.
>
Erk... I'm not happy about this. very_unlikely() makes it sound like it
operates like unlikely(), which really is not the case. There is a huge
difference in mechanism here as well as usage.
-hpa
Update networking with api change: static_branch() -> very_unlikely().
Cc: Eric Dumazet <[email protected]>
Cc: David S. Miller <[email protected]>
Cc: Glauber Costa <[email protected]>
Signed-off-by: Jason Baron <[email protected]>
---
include/linux/netfilter.h | 2 +-
include/net/sock.h | 2 +-
net/core/dev.c | 8 ++++----
3 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/include/linux/netfilter.h b/include/linux/netfilter.h
index b809265..43d0cc0 100644
--- a/include/linux/netfilter.h
+++ b/include/linux/netfilter.h
@@ -169,7 +169,7 @@ static inline bool nf_hooks_active(u_int8_t pf, unsigned int hook)
{
if (__builtin_constant_p(pf) &&
__builtin_constant_p(hook))
- return static_branch(&nf_hooks_needed[pf][hook]);
+ return very_unlikely(&nf_hooks_needed[pf][hook]);
return !list_empty(&nf_hooks[pf][hook]);
}
diff --git a/include/net/sock.h b/include/net/sock.h
index 91c1c8b..1d16574 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -930,7 +930,7 @@ static inline struct cg_proto *parent_cg_proto(struct proto *proto,
{
return proto->proto_cgroup(parent_mem_cgroup(cg_proto->memcg));
}
-#define mem_cgroup_sockets_enabled static_branch(&memcg_socket_limit_enabled)
+#define mem_cgroup_sockets_enabled very_unlikely(&memcg_socket_limit_enabled)
#else
#define mem_cgroup_sockets_enabled 0
static inline struct cg_proto *parent_cg_proto(struct proto *proto,
diff --git a/net/core/dev.c b/net/core/dev.c
index 115dee1..6503c14 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -1481,12 +1481,12 @@ EXPORT_SYMBOL(net_disable_timestamp);
static inline void net_timestamp_set(struct sk_buff *skb)
{
skb->tstamp.tv64 = 0;
- if (static_branch(&netstamp_needed))
+ if (very_unlikely(&netstamp_needed))
__net_timestamp(skb);
}
#define net_timestamp_check(COND, SKB) \
- if (static_branch(&netstamp_needed)) { \
+ if (very_unlikely(&netstamp_needed)) { \
if ((COND) && !(SKB)->tstamp.tv64) \
__net_timestamp(SKB); \
} \
@@ -2945,7 +2945,7 @@ int netif_rx(struct sk_buff *skb)
trace_netif_rx(skb);
#ifdef CONFIG_RPS
- if (static_branch(&rps_needed)) {
+ if (very_unlikely(&rps_needed)) {
struct rps_dev_flow voidflow, *rflow = &voidflow;
int cpu;
@@ -3309,7 +3309,7 @@ int netif_receive_skb(struct sk_buff *skb)
return NET_RX_SUCCESS;
#ifdef CONFIG_RPS
- if (static_branch(&rps_needed)) {
+ if (very_unlikely(&rps_needed)) {
struct rps_dev_flow voidflow, *rflow = &voidflow;
int cpu, ret;
--
1.7.7.5
On Tue, Feb 21, 2012 at 12:09:20PM -0800, H. Peter Anvin wrote:
> On 02/21/2012 12:02 PM, Jason Baron wrote:
> > Hi,
> >
> > Renames 'static_branch()' -> very_unlikely(), hopefully, to be more intuitive
> > as to what jump labels is about. I'm also introducing 'very_likely()', as
> > the analogue to very_unlikely(). Patch is against the -tip perf branch.
> >
>
> Erk... I'm not happy about this. very_unlikely() makes it sound like it
> operates like unlikely(), which really is not the case. There is a huge
> difference in mechanism here as well as usage.
>
> -hpa
>
The naming discussion really stems from the addition of a default true
branch.
Originally we had 'static_branch()'. Then, in the first RFC introducing
the default true branch, I proposed: 'static_branch_def_false', and
'static_branch_def_true'. Did you like those better?
I'm not really too hung up on the naming, but I did think that
very_[un]likely were an interesting possibility.
Thanks,
-Jason
On Tue, 2012-02-21 at 12:09 -0800, H. Peter Anvin wrote:
> On 02/21/2012 12:02 PM, Jason Baron wrote:
> > Hi,
> >
> > Renames 'static_branch()' -> very_unlikely(), hopefully, to be more intuitive
> > as to what jump labels is about. I'm also introducing 'very_likely()', as
> > the analogue to very_unlikely(). Patch is against the -tip perf branch.
> >
>
> Erk... I'm not happy about this. very_unlikely() makes it sound like it
> operates like unlikely(), which really is not the case. There is a huge
> difference in mechanism here as well as usage.
I agree with Peter.
What about static_branch_true() and static_branch_false().
Or remove the "_branch" part and have static_true() and static_false()?
-- Steve
On Tue, 2012-02-21 at 15:20 -0500, Jason Baron wrote:
> I'm not really too hung up on the naming, but I did think that
> very_[un]likely were an interesting possibility.
The problem comes from what Peter said. They are too similar to
"likely()" and "unlikely()", and can become confusing.
Maybe "static_likely()" and "static_unlikely()" as the word "static" can
imply something strange about these. Or perhaps a "const_likely()"?
Maybe "dynamic_branch_true()" and "dynamic_branch_false()". This may be
the most descriptive.
-- Steve
On 02/21/2012 12:39 PM, Steven Rostedt wrote:
> On Tue, 2012-02-21 at 15:20 -0500, Jason Baron wrote:
>
>> I'm not really too hung up on the naming, but I did think that
>> very_[un]likely were an interesting possibility.
>
> The problem comes from what Peter said. They are too similar to
> "likely()" and "unlikely()", and can become confusing.
>
> Maybe "static_likely()" and "static_unlikely()" as the word "static" can
> imply something strange about these. Or perhaps a "const_likely()"?
>
> Maybe "dynamic_branch_true()" and "dynamic_branch_false()". This may be
> the most descriptive.
>
I thought about this some more, and the very_[un]likely() naming is even
worse than I originally thought: the jump label stuff isn't about the
bias level, but rather if a static decision (on the order or once per
boot) can be made to go one way or the other.
-hpa
On 02/21/2012 12:21 PM, Steven Rostedt wrote:
> On Tue, 2012-02-21 at 12:09 -0800, H. Peter Anvin wrote:
>> On 02/21/2012 12:02 PM, Jason Baron wrote:
>>> Hi,
>>>
>>> Renames 'static_branch()' -> very_unlikely(), hopefully, to be more intuitive
>>> as to what jump labels is about. I'm also introducing 'very_likely()', as
>>> the analogue to very_unlikely(). Patch is against the -tip perf branch.
>>>
>>
>> Erk... I'm not happy about this. very_unlikely() makes it sound like it
>> operates like unlikely(), which really is not the case. There is a huge
>> difference in mechanism here as well as usage.
>
> I agree with Peter.
>
> What about static_branch_true() and static_branch_false().
>
> Or remove the "_branch" part and have static_true() and static_false()?
>
static_true() and static_false() seem good.
-hpa
* Steven Rostedt ([email protected]) wrote:
> On Tue, 2012-02-21 at 15:20 -0500, Jason Baron wrote:
>
> > I'm not really too hung up on the naming, but I did think that
> > very_[un]likely were an interesting possibility.
>
> The problem comes from what Peter said. They are too similar to
> "likely()" and "unlikely()", and can become confusing.
>
> Maybe "static_likely()" and "static_unlikely()" as the word "static" can
> imply something strange about these. Or perhaps a "const_likely()"?
My 2 cents:
static_likely()/static_unlikely() seems to be the less strange
construct names I've seen fly so far. ;-) And they seem to convey the
semantic of static branches and branch "hint" quite well.
Thanks,
Mathieu
>
> Maybe "dynamic_branch_true()" and "dynamic_branch_false()". This may be
> the most descriptive.
>
> -- Steve
>
>
>
--
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com
* H. Peter Anvin ([email protected]) wrote:
> On 02/21/2012 12:39 PM, Steven Rostedt wrote:
> > On Tue, 2012-02-21 at 15:20 -0500, Jason Baron wrote:
> >
> >> I'm not really too hung up on the naming, but I did think that
> >> very_[un]likely were an interesting possibility.
> >
> > The problem comes from what Peter said. They are too similar to
> > "likely()" and "unlikely()", and can become confusing.
> >
> > Maybe "static_likely()" and "static_unlikely()" as the word "static" can
> > imply something strange about these. Or perhaps a "const_likely()"?
> >
> > Maybe "dynamic_branch_true()" and "dynamic_branch_false()". This may be
> > the most descriptive.
> >
>
> I thought about this some more, and the very_[un]likely() naming is even
> worse than I originally thought: the jump label stuff isn't about the
> bias level, but rather if a static decision (on the order or once per
> boot) can be made to go one way or the other.
>
> -hpa
I agree that this decision is taken typically once at boot time, so
claiming it is only a strong compiler bias hint for block placement
would be a lie. However, I think the fact that the fall-through is for
either true or false branch seems to be an implicit bias. Therefore, I
start to like the static_likely()/static_unlikely(), which conveys both
the static nature of the branch, as well as the bias.
Thanks,
Mathieu
--
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com
* H. Peter Anvin <[email protected]> wrote:
> On 02/21/2012 12:02 PM, Jason Baron wrote:
> > Hi,
> >
> > Renames 'static_branch()' -> very_unlikely(), hopefully, to be more intuitive
> > as to what jump labels is about. I'm also introducing 'very_likely()', as
> > the analogue to very_unlikely(). Patch is against the -tip perf branch.
> >
>
> Erk... I'm not happy about this. very_unlikely() makes it
> sound like it operates like unlikely(), which really is not
> the case. There is a huge difference in mechanism here as
> well as usage.
There are three issues why I insisted for Jason to rename these
nonsensically 'jump_label' patterned methods before I send any
more jump label commits to Linus:
The pattern has spread beyond the niche of tracing internals and
nobody seemed to have any second thoughts about the actual
readability of these names. That is a major FAIL and it was my
primary concern.
For those *reading* the code, the similarity of
very_likely()/very_unlikely() to the likely()/unlikely()
patterns is *INTENTIONAL*, as functionally the branch site
itself is very similar to a real branch!
Secondly, for those *writing* such code, you cannot just
'accidentally' change a unlikely() to a very_unlikely() and get
something you didn't ask for ...
It is the modification site that is dissimilar (and which is
slow in the jump label case) - and that is very apparently
dissimilar that it takes some effort to change these flags. If
you write such code you have to add the whole jump label
mechanism to it, which makes it very apparent what is happening
and makes the costs very apparent as well.
Thirdly, the fact that it's a 'jump label' is an
*implementational* detail. On some architectures very_unlikely()
indeed maps to unlikely(), because jump labels have not been
implemented yet. On some architectures very_unlikely() is
implemented via jump labels. On some future architectures it
might be implemented via JIT-ing the target function. (I made
the last one up)
So it makes sense to decouple 'jump labels' from the actual high
level naming of this API.
Anyway, I've Cc:-ed Linus and Andrew, I plan to take the renames
but they can NAK me if doing that is stupid.
Thanks,
Ingo
On 02/21/2012 10:50 PM, Ingo Molnar wrote:
>
> The pattern has spread beyond the niche of tracing internals and
> nobody seemed to have any second thoughts about the actual
> readability of these names. That is a major FAIL and it was my
> primary concern.
>
> For those *reading* the code, the similarity of
> very_likely()/very_unlikely() to the likely()/unlikely()
> patterns is *INTENTIONAL*, as functionally the branch site
> itself is very similar to a real branch!
>
> Secondly, for those *writing* such code, you cannot just
> 'accidentally' change a unlikely() to a very_unlikely() and get
> something you didn't ask for ...
>
> It is the modification site that is dissimilar (and which is
> slow in the jump label case) - and that is very apparently
> dissimilar that it takes some effort to change these flags. If
> you write such code you have to add the whole jump label
> mechanism to it, which makes it very apparent what is happening
> and makes the costs very apparent as well.
>
> Thirdly, the fact that it's a 'jump label' is an
> *implementational* detail. On some architectures very_unlikely()
> indeed maps to unlikely(), because jump labels have not been
> implemented yet. On some architectures very_unlikely() is
> implemented via jump labels. On some future architectures it
> might be implemented via JIT-ing the target function. (I made
> the last one up)
>
> So it makes sense to decouple 'jump labels' from the actual high
> level naming of this API.
>
> Anyway, I've Cc:-ed Linus and Andrew, I plan to take the renames
> but they can NAK me if doing that is stupid.
>
I have to VEHEMENTLY disagree with you here.
likely()/unlikely() are supposed to be used when it is clear at the time
the code is written which way the branch is biased or which path is
important (unlikely() is typically used for performance-unimportant
bailouts.)
Jump labels are not that way. The key aspect for when the jump label
optimization is relevant is *how often is the direction changed*. It is
perfectly sane for this to be done on a 50:50 branch, as long as the
50:50-ness is settled once for all dring a boot. Consider, for example,
an Intel CPU versus an AMD CPU. Wouldn't you agree that it would be
absolutely ridiculous and downright offensive to have:
if (very_unlikely(cpu_vendor_amd)) {
Yet this is a *fantastic* use of the jump labels, because your CPU
vendor isn't going to change in the middle of execution (hot VM
migrations excepted ;)
So the key aspect of this is the staticness of the conditional, NOT the
degree of bias of the branch. Hence my past insistence on the
"static_branch" name (rather than "jump_label")... the branch part can
be omitted, as an implementation detail, but the staticness of it is its
absolutely key defining characteristic.
-hpa
--
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel. I don't speak on their behalf.
* H. Peter Anvin <[email protected]> wrote:
> On 02/21/2012 10:50 PM, Ingo Molnar wrote:
> >
> > The pattern has spread beyond the niche of tracing internals and
> > nobody seemed to have any second thoughts about the actual
> > readability of these names. That is a major FAIL and it was my
> > primary concern.
> >
> > For those *reading* the code, the similarity of
> > very_likely()/very_unlikely() to the likely()/unlikely()
> > patterns is *INTENTIONAL*, as functionally the branch site
> > itself is very similar to a real branch!
> >
> > Secondly, for those *writing* such code, you cannot just
> > 'accidentally' change a unlikely() to a very_unlikely() and get
> > something you didn't ask for ...
> >
> > It is the modification site that is dissimilar (and which is
> > slow in the jump label case) - and that is very apparently
> > dissimilar that it takes some effort to change these flags. If
> > you write such code you have to add the whole jump label
> > mechanism to it, which makes it very apparent what is happening
> > and makes the costs very apparent as well.
> >
> > Thirdly, the fact that it's a 'jump label' is an
> > *implementational* detail. On some architectures very_unlikely()
> > indeed maps to unlikely(), because jump labels have not been
> > implemented yet. On some architectures very_unlikely() is
> > implemented via jump labels. On some future architectures it
> > might be implemented via JIT-ing the target function. (I made
> > the last one up)
> >
> > So it makes sense to decouple 'jump labels' from the actual high
> > level naming of this API.
> >
> > Anyway, I've Cc:-ed Linus and Andrew, I plan to take the renames
> > but they can NAK me if doing that is stupid.
> >
>
> I have to VEHEMENTLY disagree with you here.
>
> likely()/unlikely() are supposed to be used when it is clear
> at the time the code is written which way the branch is biased
> or which path is important (unlikely() is typically used for
> performance-unimportant bailouts.)
>
> Jump labels are not that way. The key aspect for when the
> jump label optimization is relevant is *how often is the
> direction changed*. It is perfectly sane for this to be done
> on a 50:50 branch, as long as the 50:50-ness is settled once
> for all dring a boot. [...]
Erm, that's not at all true - jump labels have a strong build
time bias/assymetry: the conditional block is moved totally out
of line and the original fall-through code-path is left as
straight a flow as possible.
Just look at the jump label usage site disassembly and check the
historic evolution of this feature in the Git logs:
The *primary* benefit of jump labels was always to the
fall-trough code. We hurt the tracepoint-enabled codepath
*INTENTIONALLY*, to make sure the overwhelmingly common case of
them being off is left alone as much as possible.
> [...] Consider, for example, an Intel CPU versus an AMD CPU.
> Wouldn't you agree that it would be absolutely ridiculous and
> downright offensive to have:
>
> if (very_unlikely(cpu_vendor_amd)) {
>
> Yet this is a *fantastic* use of the jump labels, because your
> CPU vendor isn't going to change in the middle of execution
> (hot VM migrations excepted ;)
No, that condition perfectly validly represents what happens in
reality: that the conditional AMD code is moved *out of line* -
in most cases penalizing it in terms of instruction scheduling,
etc.
There is a fundamental assymetry, and intentionally so. You
*really* have to think what the common case is, and make sure
the build defaults to that. It's not the end of the world to
have it flipped over, but there's costs and those costs are
higher even in the branch path than a regular
likely()/unlikely().
So you are rather wrong about your expectations - I think that
is one more piece of evidence that the naming was less than
optimal.
> So the key aspect of this is the staticness of the
> conditional, NOT the degree of bias of the branch. Hence my
> past insistence on the "static_branch" name (rather than
> "jump_label")... the branch part can be omitted, as an
> implementation detail, but the staticness of it is its
> absolutely key defining characteristic.
I don't think you understand this facility as well as you think
you do.
Thanks,
Ingo
* Steven Rostedt <[email protected]> wrote:
> On Tue, 2012-02-21 at 15:20 -0500, Jason Baron wrote:
>
> > I'm not really too hung up on the naming, but I did think
> > that very_[un]likely were an interesting possibility.
>
> The problem comes from what Peter said. They are too similar
> to "likely()" and "unlikely()", and can become confusing.
See my other mail.
> Maybe "static_likely()" and "static_unlikely()" as the word
> "static" can imply something strange about these. Or perhaps a
> "const_likely()"?
>
> Maybe "dynamic_branch_true()" and "dynamic_branch_false()". This may be
> the most descriptive.
too long.
'static branch' or 'static condition' is not a bad concept, if
people don't find the similarity to 'static' too confusing ;-).
But it is fundamentally mixing execution and *data type* and it
is not conveying the build time bias properly.
So the best high level naming would be something like:
struct static_condition static_flag = STATIC_COND_FALSE;
if (very_unlikely(&static_flag)) {
...
}
...
static_cond_inc(&static_flag);
...
static_cond_dec(&static_flag);
See how *both* the build time bias and the cost of a state
transition is properly conveyed?
I suggested something like this to Jason in the off-list
discusion and it's not fully implemented yet. Let me whip up a
test branch [pun and potential confusion unintended] that shows
it.
Thanks,
Ingo
On 02/21/2012 11:25 PM, Ingo Molnar wrote:
>
> There is a fundamental assymetry, and intentionally so. You
> *really* have to think what the common case is, and make sure
> the build defaults to that. It's not the end of the world to
> have it flipped over, but there's costs and those costs are
> higher even in the branch path than a regular
> likely()/unlikely().
>
No, not really -- it's still an unconditional branch, which means you
will not tax the branch predictor in any way and which can be followed
by the front end without taking a speculation hit. So although there is
an out-of-line penalty -- which you hit for any conditional, after all
you can only have *one* piece of code which is straight line -- it
should be less than for a normal conditional branch.
> So you are rather wrong about your expectations - I think that
> is one more piece of evidence that the naming was less than
> optimal.
>
>> So the key aspect of this is the staticness of the
>> conditional, NOT the degree of bias of the branch. Hence my
>> past insistence on the "static_branch" name (rather than
>> "jump_label")... the branch part can be omitted, as an
>> implementation detail, but the staticness of it is its
>> absolutely key defining characteristic.
>
> I don't think you understand this facility as well as you think
> you do.
Uh, no, I do... see the above, but combine that of course with the sheer
astronomical cost of flipping the conditional.
-hpa
--
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel. I don't speak on their behalf.
* H. Peter Anvin <[email protected]> wrote:
> On 02/21/2012 11:25 PM, Ingo Molnar wrote:
> >
> > There is a fundamental assymetry, and intentionally so. You
> > *really* have to think what the common case is, and make
> > sure the build defaults to that. It's not the end of the
> > world to have it flipped over, but there's costs and those
> > costs are higher even in the branch path than a regular
> > likely()/unlikely().
>
> No, not really -- it's still an unconditional branch, which
> means you will not tax the branch predictor in any way and
> which can be followed by the front end without taking a
> speculation hit. [...]
You are talking about CPU level costs, I am also talking about
costs introduced at build time.
Fact is, jump-label unlikely branches are moved *out of line*:
they are often in unlikely portions of the function (near other
unlikely branches), with instruction cache granularity costs and
potentially higher instruction-cache miss costs attached, etc.
You are missing three important aspects:
Firstly, instead of:
ins1
ins2
ins3
ins4
ins5
ins-compare
ins-branch
ins6
ins7
ins8
ins9
ins10
We have:
ins1
ins2
ins3
ins4
ins5
ins-jump
[ hole ]
ins6
ins7
ins8
ins9
ins10
ins-jump back
Where the 'hole' fragments the instruction cache layout. Given
that most of kernel execution is instruction-cache-cold, the
'straightness' of kernel code matters quite a bit.
Secondly, there's build time instruction scheduling costs as
well: GCC will prefer the likely branch over the unlikely one,
so we might see extra instructions in the out-of-line code:
ins1
ins2
ins3
ins4
ins5
ins-jump
[ hole ]
ins-extra-1
ins-extra-2
ins6
ins7
ins8
ins9
ins10
ins-jump back
In that sense jump labels are unlikely() branches combined with
a patching mechanism.
Thus *both* aspects are important: if a branch is *truly* 50/50
then it's quite possibly *NOT* a correct optimization to use
jump-labels as the 'uncommon' code goes through extra hoops and
fragments out of the fastpath, which in quite many real life
cases can outstrip the advantage of the avoidance of a single
branch ...
Thirdly,
even if it's a correct optimization and both branches happen to
outperform the pre-jump-label version, regardless of the
direction of the jump label flag, it's *STILL* fundamentally
assymetric: due to the hole and due to the possible extra
instructions the out of line code will be slower by a few
instruction and the NOP fall-through will be faster.
This is fundamentally so, and any naming that tries to *hide*
that assymetry and the associated micro-costs is confused.
Thanks,
Ingo
* Ingo Molnar <[email protected]> wrote:
> But it is fundamentally mixing execution and *data type* and
> it is not conveying the build time bias properly.
>
> So the best high level naming would be something like:
>
> struct static_condition static_flag = STATIC_COND_FALSE;
>
>
> if (very_unlikely(&static_flag)) {
> ...
> }
>
> ...
>
> static_cond_inc(&static_flag);
> ...
> static_cond_dec(&static_flag);
Btw., I think the modification path could also carry the high
cost of modification (stopping all cpus, modifying code, etc.).
This could be done via:
static_cond_slow_inc(&static_flag);
...
static_cond_slow_dec(&static_flag);
And if a developer does not notice that 'slow' implies a
performance cost, then he probably would have doubly missed this
aspect of jump_label_inc()/jump_label_dec().
Thanks,
Ingo
Not arguing that, but the static aspect is still key... or people will read it as another version of likely/unlikely. I'd be fine with static_likely/unlikely for example; I wish "static" wasn't such an overloaded word in C but I can't.personally think of a better term.
Ingo Molnar <[email protected]> wrote:
>
>* H. Peter Anvin <[email protected]> wrote:
>
>> On 02/21/2012 11:25 PM, Ingo Molnar wrote:
>> >
>> > There is a fundamental assymetry, and intentionally so. You
>> > *really* have to think what the common case is, and make
>> > sure the build defaults to that. It's not the end of the
>> > world to have it flipped over, but there's costs and those
>> > costs are higher even in the branch path than a regular
>> > likely()/unlikely().
>>
>> No, not really -- it's still an unconditional branch, which
>> means you will not tax the branch predictor in any way and
>> which can be followed by the front end without taking a
>> speculation hit. [...]
>
>You are talking about CPU level costs, I am also talking about
>costs introduced at build time.
>
>Fact is, jump-label unlikely branches are moved *out of line*:
>they are often in unlikely portions of the function (near other
>unlikely branches), with instruction cache granularity costs and
>potentially higher instruction-cache miss costs attached, etc.
>
>You are missing three important aspects:
>
>Firstly, instead of:
>
> ins1
> ins2
> ins3
> ins4
> ins5
> ins-compare
> ins-branch
> ins6
> ins7
> ins8
> ins9
> ins10
>
>We have:
>
> ins1
> ins2
> ins3
> ins4
> ins5
> ins-jump
>
> [ hole ]
>
> ins6
> ins7
> ins8
> ins9
> ins10
> ins-jump back
>
>Where the 'hole' fragments the instruction cache layout. Given
>that most of kernel execution is instruction-cache-cold, the
>'straightness' of kernel code matters quite a bit.
>
>Secondly, there's build time instruction scheduling costs as
>well: GCC will prefer the likely branch over the unlikely one,
>so we might see extra instructions in the out-of-line code:
>
>
> ins1
> ins2
> ins3
> ins4
> ins5
> ins-jump
>
> [ hole ]
>
> ins-extra-1
> ins-extra-2
> ins6
> ins7
> ins8
> ins9
> ins10
> ins-jump back
>
>In that sense jump labels are unlikely() branches combined with
>a patching mechanism.
>
>Thus *both* aspects are important: if a branch is *truly* 50/50
>then it's quite possibly *NOT* a correct optimization to use
>jump-labels as the 'uncommon' code goes through extra hoops and
>fragments out of the fastpath, which in quite many real life
>cases can outstrip the advantage of the avoidance of a single
>branch ...
>
>Thirdly,
>
>even if it's a correct optimization and both branches happen to
>outperform the pre-jump-label version, regardless of the
>direction of the jump label flag, it's *STILL* fundamentally
>assymetric: due to the hole and due to the possible extra
>instructions the out of line code will be slower by a few
>instruction and the NOP fall-through will be faster.
>
>This is fundamentally so, and any naming that tries to *hide*
>that assymetry and the associated micro-costs is confused.
>
>Thanks,
>
> Ingo
--
Sent from my mobile phone. Please excuse my brevity and lack of formatting.
Stupid thought... do we have cases that matter where the bias and default don't agree?
Ingo Molnar <[email protected]> wrote:
>
>* Ingo Molnar <[email protected]> wrote:
>
>> But it is fundamentally mixing execution and *data type* and
>> it is not conveying the build time bias properly.
>>
>> So the best high level naming would be something like:
>>
>> struct static_condition static_flag = STATIC_COND_FALSE;
>>
>>
>> if (very_unlikely(&static_flag)) {
>> ...
>> }
>>
>> ...
>>
>> static_cond_inc(&static_flag);
>> ...
>> static_cond_dec(&static_flag);
>
>Btw., I think the modification path could also carry the high
>cost of modification (stopping all cpus, modifying code, etc.).
>
>This could be done via:
>
> static_cond_slow_inc(&static_flag);
> ...
> static_cond_slow_dec(&static_flag);
>
>And if a developer does not notice that 'slow' implies a
>performance cost, then he probably would have doubly missed this
>aspect of jump_label_inc()/jump_label_dec().
>
>Thanks,
>
> Ingo
--
Sent from my mobile phone. Please excuse my brevity and lack of formatting.
* H. Peter Anvin <[email protected]> wrote:
> Not arguing that, but the static aspect is still key... or
> people will read it as another version of likely/unlikely.
They can *read* it as such, that as is very much intentional!
People reading such code should indeed treat it as a branch
probability attribute (and ignore it in 99.9% of the cases).
The moment they *write* static_cond_slow_inc() in real code
though they will be warned about the speciality and slowness of
the update path. More so than they are warned by the current
jump_label_inc() name, me thinks.
> I'd be fine with static_likely/unlikely for example; I wish
> "static" wasn't such an overloaded word in C but I
> can't.personally think of a better term.
Yeah, we considered static_likely()/unlikely() but it is indeed
overloaded *way* too much, so we went for
very_likely()/very_unlikely() which also fairly conveys the real
meaning at the usage site ...
I think the naming scheme I suggested in the other mail
sufficienty carries both the attribute, bias and update cost
aspects.
Thanks,
Ingo
* H. Peter Anvin <[email protected]> wrote:
> Stupid thought... do we have cases that matter where the bias
> and default don't agree?
Yeah, that was one of my worries about the proposed original
tongue twisters (see Jason's original series: "jump label:
introduce default true branch").
For example could you tell *at a glance* what this does:
+ if (!static_branch_def_false(&perf_sched_events.key))
?
I certainly couldn't, I'd have to consider the '!', that it's a
'static branch' and that it's either 'defined to false' or
'default to false'.
Linguistic and visual barriers all around, and that's for code
that I am intimately familar with ...
The problem with static_branch_def_false/def_true was that the
very intuitively visible bias that we see with
likely()/unlikely() is confused in jump label constructs through
two layers of modifiers. And the fix is so easy, a simple rename
in most cases ;-)
So instead of that, in this series we have:
+ if (very_unlikely(&perf_sched_events.key))
which is a heck of an improvement IMO. I'd still up its
readability a notch, by also signalling the overhead of the
update path by making it:
+ if (very_unlikely(&perf_sched_events.slow_flag))
... but I don't want to be that much of a readability nazi ;-)
Thanks,
Ingo
So here's a boot tested patch on top of Jason's series that does
all the cleanups I talked about and turns jump labels into a
more intuitive to use facility. It should also address the
various misconceptions and confusions that surround jump labels.
Typical usage scenarios:
#include <linux/static_key.h>
struct static_key key = STATIC_KEY_INIT_TRUE;
if (very_unlikely(&key))
do unlikely code
else
do likely code
Or:
if (very_likely(&key))
do likely code
else
do unlikely code
The static key is modified via:
static_key_slow_inc(&key);
...
static_key_slow_dec(&key);
The 'slow' prefix makes it abundantly clear that this is an
expensive operation.
I've updated all in-kernel code to use this everywhere. Note
that I (intentionally) have not pushed through the rename
blindly through to the lowest levels: the actual jump-label
patching arch facility should be named like that, so we want to
decouple jump labels from the static-key facility a bit.
On non-jump-label enabled architectures static keys default to
likely()/unlikely() branches.
Thanks,
Ingo
Signed-off-by: Ingo Molnar <[email protected]>
---
---
Documentation/jump-label.txt | 38 ++++++++---------
arch/ia64/include/asm/paravirt.h | 6 +-
arch/ia64/kernel/paravirt.c | 4 -
arch/mips/include/asm/jump_label.h | 2
arch/powerpc/include/asm/jump_label.h | 2
arch/s390/include/asm/jump_label.h | 2
arch/sparc/include/asm/jump_label.h | 2
arch/x86/include/asm/jump_label.h | 6 +-
arch/x86/include/asm/paravirt.h | 6 +-
arch/x86/kernel/kvm.c | 4 -
arch/x86/kernel/paravirt.c | 4 -
arch/x86/kvm/mmu_audit.c | 6 +-
include/linux/jump_label.h | 76 +++++++++++++++++-----------------
include/linux/netdevice.h | 4 -
include/linux/netfilter.h | 4 -
include/linux/perf_api.h | 2
include/linux/perf_event.h | 6 +-
include/linux/static_key.h | 1
include/linux/tracepoint.h | 6 +-
include/net/sock.h | 4 -
kernel/events/core.c | 16 +++----
kernel/jump_label.c | 72 ++++++++++++++++----------------
kernel/sched/core.c | 14 +++---
kernel/sched/fair.c | 6 +-
kernel/sched/sched.h | 10 ++--
kernel/tracepoint.c | 20 ++++----
net/core/dev.c | 16 +++----
net/core/net-sysfs.c | 4 -
net/core/sock.c | 4 -
net/core/sysctl_net_core.c | 4 -
net/ipv4/tcp_memcontrol.c | 6 +-
net/netfilter/core.c | 6 +-
32 files changed, 182 insertions(+), 181 deletions(-)
Index: linux/Documentation/jump-label.txt
===================================================================
--- linux.orig/Documentation/jump-label.txt
+++ linux/Documentation/jump-label.txt
@@ -32,7 +32,7 @@ the branch site to change the branch dir
For example, if we have a simple branch that is disabled by default:
- if (very_unlikely(&jump_key))
+ if (very_unlikely(&key))
printk("I am the true branch\n");
Thus, by default the 'printk' will not be emitted. And the code generated will
@@ -48,58 +48,58 @@ basically 'free'. That is the basic trad
In order to use a jump label you much first define a key:
- struct jump_label_key jump_key;
+ struct static_key key;
Which is initialized as:
- struct jump_label_key jump_key = JUMP_LABEL_INIT_TRUE;
+ struct static_key key = STATIC_KEY_INIT_TRUE;
or:
- struct jump_label_Key jump_key = JUMP_LABEL_INIT_FALSE;
+ struct static_key key = STATIC_KEY_INIT_FALSE;
If the key is not initialized, it is default false. The
-'struct jump_label_key', must be a 'global'. That is, it can't be allocated on
+'struct static_key', must be a 'global'. That is, it can't be allocated on
the stack or dynamically allocated at run-time.
The key is then used in code as:
- if (very_unlikely(&jump_key))
+ if (very_unlikely(&key))
do unlikely code
else
do likely code
Or:
- if (very_likely(&jump_key))
+ if (very_likely(&key))
do likely code
else
do unlikely code
-A key that is initialized via 'JUMP_LABEL_INIT_FALSE', must be used in a
+A key that is initialized via 'STATIC_KEY_INIT_FALSE', must be used in a
'very_unlikely()' construct. Likewise, a key initialized via
-'JUMP_LABEL_INIT_TRUE' must be used in a 'very_likely()' construct.
+'STATIC_KEY_INIT_TRUE' must be used in a 'very_likely()' construct.
A single key can be used in many branches, but all the branches must match
the way that the key has been initialized.
The branch(es) can then be switched via:
- jump_label_inc(&jump_key);
- jump_label_dec(&jump_key);
+ static_key_slow_inc(&key);
+ static_key_slow_dec(&key);
-Thus, 'jump_label_inc()' means 'make the branch true', and
-'jump_label_dec()' means 'make the the branch false' with appropriate reference
-counting. For example, if the key is initialized true, a jump_label_dec(), will
-switch the branch to false. And a subsequent jump_label_inc(), will change
+Thus, 'static_key_slow_inc()' means 'make the branch true', and
+'static_key_slow_dec()' means 'make the the branch false' with appropriate reference
+counting. For example, if the key is initialized true, a static_key_slow_dec(), will
+switch the branch to false. And a subsequent static_key_slow_inc(), will change
the branch back to true. Likewise, if the key is initialized false, a
-'jump_label_inc()', will change the branch to true. And then a
-'jump_label_dec()', will again make the branch false.
+'static_key_slow_inc()', will change the branch to true. And then a
+'static_key_slow_dec()', will again make the branch false.
An example usage in the kernel is the implementation of tracepoints:
static inline void trace_##name(proto) \
{ \
- if (very_unlikely(&__tracepoint_##name.key)) \
+ if (very_unlikely(&__tracepoint_##name.key)) \
__DO_TRACE(&__tracepoint_##name, \
TP_PROTO(data_proto), \
TP_ARGS(data_args), \
@@ -122,7 +122,7 @@ simply fall back to a traditional, load,
* #define JUMP_LABEL_NOP_SIZE, see: arch/x86/include/asm/jump_label.h
-* __always_inline bool arch_static_branch(struct jump_label_key *key), see:
+* __always_inline bool arch_static_branch(struct static_key *key), see:
arch/x86/include/asm/jump_label.h
* void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type),
Index: linux/arch/ia64/include/asm/paravirt.h
===================================================================
--- linux.orig/arch/ia64/include/asm/paravirt.h
+++ linux/arch/ia64/include/asm/paravirt.h
@@ -281,9 +281,9 @@ paravirt_init_missing_ticks_accounting(i
pv_time_ops.init_missing_ticks_accounting(cpu);
}
-struct jump_label_key;
-extern struct jump_label_key paravirt_steal_enabled;
-extern struct jump_label_key paravirt_steal_rq_enabled;
+struct static_key;
+extern struct static_key paravirt_steal_enabled;
+extern struct static_key paravirt_steal_rq_enabled;
static inline int
paravirt_do_steal_accounting(unsigned long *new_itm)
Index: linux/arch/ia64/kernel/paravirt.c
===================================================================
--- linux.orig/arch/ia64/kernel/paravirt.c
+++ linux/arch/ia64/kernel/paravirt.c
@@ -634,8 +634,8 @@ struct pv_irq_ops pv_irq_ops = {
* pv_time_ops
* time operations
*/
-struct jump_label_key paravirt_steal_enabled;
-struct jump_label_key paravirt_steal_rq_enabled;
+struct static_key paravirt_steal_enabled;
+struct static_key paravirt_steal_rq_enabled;
static int
ia64_native_do_steal_accounting(unsigned long *new_itm)
Index: linux/arch/mips/include/asm/jump_label.h
===================================================================
--- linux.orig/arch/mips/include/asm/jump_label.h
+++ linux/arch/mips/include/asm/jump_label.h
@@ -20,7 +20,7 @@
#define WORD_INSN ".word"
#endif
-static __always_inline bool arch_static_branch(struct jump_label_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key)
{
asm goto("1:\tnop\n\t"
"nop\n\t"
Index: linux/arch/powerpc/include/asm/jump_label.h
===================================================================
--- linux.orig/arch/powerpc/include/asm/jump_label.h
+++ linux/arch/powerpc/include/asm/jump_label.h
@@ -17,7 +17,7 @@
#define JUMP_ENTRY_TYPE stringify_in_c(FTR_ENTRY_LONG)
#define JUMP_LABEL_NOP_SIZE 4
-static __always_inline bool arch_static_branch(struct jump_label_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key)
{
asm goto("1:\n\t"
"nop\n\t"
Index: linux/arch/s390/include/asm/jump_label.h
===================================================================
--- linux.orig/arch/s390/include/asm/jump_label.h
+++ linux/arch/s390/include/asm/jump_label.h
@@ -13,7 +13,7 @@
#define ASM_ALIGN ".balign 4"
#endif
-static __always_inline bool arch_static_branch(struct jump_label_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key)
{
asm goto("0: brcl 0,0\n"
".pushsection __jump_table, \"aw\"\n"
Index: linux/arch/sparc/include/asm/jump_label.h
===================================================================
--- linux.orig/arch/sparc/include/asm/jump_label.h
+++ linux/arch/sparc/include/asm/jump_label.h
@@ -7,7 +7,7 @@
#define JUMP_LABEL_NOP_SIZE 4
-static __always_inline bool arch_static_branch(struct jump_label_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key)
{
asm goto("1:\n\t"
"nop\n\t"
Index: linux/arch/x86/include/asm/jump_label.h
===================================================================
--- linux.orig/arch/x86/include/asm/jump_label.h
+++ linux/arch/x86/include/asm/jump_label.h
@@ -9,12 +9,12 @@
#define JUMP_LABEL_NOP_SIZE 5
-#define JUMP_LABEL_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t"
+#define STATIC_KEY_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t"
-static __always_inline bool arch_static_branch(struct jump_label_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key)
{
asm goto("1:"
- JUMP_LABEL_INITIAL_NOP
+ STATIC_KEY_INITIAL_NOP
".pushsection __jump_table, \"aw\" \n\t"
_ASM_ALIGN "\n\t"
_ASM_PTR "1b, %l[l_yes], %c0 \n\t"
Index: linux/arch/x86/include/asm/paravirt.h
===================================================================
--- linux.orig/arch/x86/include/asm/paravirt.h
+++ linux/arch/x86/include/asm/paravirt.h
@@ -230,9 +230,9 @@ static inline unsigned long long paravir
return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock);
}
-struct jump_label_key;
-extern struct jump_label_key paravirt_steal_enabled;
-extern struct jump_label_key paravirt_steal_rq_enabled;
+struct static_key;
+extern struct static_key paravirt_steal_enabled;
+extern struct static_key paravirt_steal_rq_enabled;
static inline u64 paravirt_steal_clock(int cpu)
{
Index: linux/arch/x86/kernel/kvm.c
===================================================================
--- linux.orig/arch/x86/kernel/kvm.c
+++ linux/arch/x86/kernel/kvm.c
@@ -438,9 +438,9 @@ void __init kvm_guest_init(void)
static __init int activate_jump_labels(void)
{
if (has_steal_clock) {
- jump_label_inc(¶virt_steal_enabled);
+ static_key_slow_inc(¶virt_steal_enabled);
if (steal_acc)
- jump_label_inc(¶virt_steal_rq_enabled);
+ static_key_slow_inc(¶virt_steal_rq_enabled);
}
return 0;
Index: linux/arch/x86/kernel/paravirt.c
===================================================================
--- linux.orig/arch/x86/kernel/paravirt.c
+++ linux/arch/x86/kernel/paravirt.c
@@ -202,8 +202,8 @@ static void native_flush_tlb_single(unsi
__native_flush_tlb_single(addr);
}
-struct jump_label_key paravirt_steal_enabled;
-struct jump_label_key paravirt_steal_rq_enabled;
+struct static_key paravirt_steal_enabled;
+struct static_key paravirt_steal_rq_enabled;
static u64 native_steal_clock(int cpu)
{
Index: linux/arch/x86/kvm/mmu_audit.c
===================================================================
--- linux.orig/arch/x86/kvm/mmu_audit.c
+++ linux/arch/x86/kvm/mmu_audit.c
@@ -234,7 +234,7 @@ static void audit_vcpu_spte(struct kvm_v
}
static bool mmu_audit;
-static struct jump_label_key mmu_audit_key;
+static struct static_key mmu_audit_key;
static void __kvm_mmu_audit(struct kvm_vcpu *vcpu, int point)
{
@@ -259,7 +259,7 @@ static void mmu_audit_enable(void)
if (mmu_audit)
return;
- jump_label_inc(&mmu_audit_key);
+ static_key_slow_inc(&mmu_audit_key);
mmu_audit = true;
}
@@ -268,7 +268,7 @@ static void mmu_audit_disable(void)
if (!mmu_audit)
return;
- jump_label_dec(&mmu_audit_key);
+ static_key_slow_dec(&mmu_audit_key);
mmu_audit = false;
}
Index: linux/include/linux/jump_label.h
===================================================================
--- linux.orig/include/linux/jump_label.h
+++ linux/include/linux/jump_label.h
@@ -13,11 +13,11 @@
* defaults to false - and the true block is placed out of line).
*
* However at runtime we can change the branch target using
- * jump_label_{inc,dec}(). These function as a 'reference' count on the key
+ * static_key_slow_{inc,dec}(). These function as a 'reference' count on the key
* object and for as long as there are references all branches referring to
* that particular key will point to the (out of line) true block.
*
- * Since this relies on modifying code the jump_label_{inc,dec}() functions
+ * Since this relies on modifying code the static_key_slow_{inc,dec}() functions
* must be considered absolute slow paths (machine wide synchronization etc.).
* OTOH, since the affected branches are unconditional their runtime overhead
* will be absolutely minimal, esp. in the default (off) case where the total
@@ -26,13 +26,13 @@
*
* When the control is directly exposed to userspace it is prudent to delay the
* decrement to avoid high frequency code modifications which can (and do)
- * cause significant performance degradation. Struct jump_label_key_deferred and
- * jump_label_dec_deferred() provide for this.
+ * cause significant performance degradation. Struct static_key_deferred and
+ * static_key_slow_dec_deferred() provide for this.
*
* Lacking toolchain and or architecture support, it falls back to a simple
* conditional branch.
*
- * struct jump_label_key my_key = JUMP_LABEL_INIT_TRUE;
+ * struct static_key my_key = STATIC_KEY_INIT_TRUE;
*
* if (very_likely(&my_key)) {
* }
@@ -42,7 +42,7 @@
* allowed.
*
* Not initializing the key (static data is initialized to 0s anyway) is the
- * same as using JUMP_LABEL_INIT_FALSE and very_unlikely() is
+ * same as using STATIC_KEY_INIT_FALSE and very_unlikely() is
* equivalent with static_branch().
*
*/
@@ -53,17 +53,17 @@
#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
-struct jump_label_key {
+struct static_key {
atomic_t enabled;
/* Set lsb bit to 1 if branch is default true, 0 ot */
struct jump_entry *entries;
#ifdef CONFIG_MODULES
- struct jump_label_mod *next;
+ struct static_key_mod *next;
#endif
};
-struct jump_label_key_deferred {
- struct jump_label_key key;
+struct static_key_deferred {
+ struct static_key key;
unsigned long timeout;
struct delayed_work work;
};
@@ -84,31 +84,31 @@ struct module;
#define JUMP_LABEL_TRUE_BRANCH 1UL
static
-inline struct jump_entry *jump_label_get_entries(struct jump_label_key *key)
+inline struct jump_entry *jump_label_get_entries(struct static_key *key)
{
return (struct jump_entry *)((unsigned long)key->entries
& ~JUMP_LABEL_TRUE_BRANCH);
}
-static inline bool jump_label_get_branch_default(struct jump_label_key *key)
+static inline bool jump_label_get_branch_default(struct static_key *key)
{
if ((unsigned long)key->entries & JUMP_LABEL_TRUE_BRANCH)
return true;
return false;
}
-static __always_inline bool very_unlikely(struct jump_label_key *key)
+static __always_inline bool very_unlikely(struct static_key *key)
{
return arch_static_branch(key);
}
-static __always_inline bool very_likely(struct jump_label_key *key)
+static __always_inline bool very_likely(struct static_key *key)
{
return !very_unlikely(key);
}
/* Deprecated. Please use 'very_unlikely() instead. */
-static __always_inline bool static_branch(struct jump_label_key *key)
+static __always_inline bool static_branch(struct static_key *key)
{
return arch_static_branch(key);
}
@@ -124,24 +124,24 @@ extern void arch_jump_label_transform(st
extern void arch_jump_label_transform_static(struct jump_entry *entry,
enum jump_label_type type);
extern int jump_label_text_reserved(void *start, void *end);
-extern void jump_label_inc(struct jump_label_key *key);
-extern void jump_label_dec(struct jump_label_key *key);
-extern void jump_label_dec_deferred(struct jump_label_key_deferred *key);
-extern bool jump_label_true(struct jump_label_key *key);
+extern void static_key_slow_inc(struct static_key *key);
+extern void static_key_slow_dec(struct static_key *key);
+extern void static_key_slow_dec_deferred(struct static_key_deferred *key);
+extern bool static_key_true(struct static_key *key);
extern void jump_label_apply_nops(struct module *mod);
extern void
-jump_label_rate_limit(struct jump_label_key_deferred *key, unsigned long rl);
+jump_label_rate_limit(struct static_key_deferred *key, unsigned long rl);
-#define JUMP_LABEL_INIT_TRUE ((struct jump_label_key) \
+#define STATIC_KEY_INIT_TRUE ((struct static_key) \
{ .enabled = ATOMIC_INIT(1), .entries = (void *)1 })
-#define JUMP_LABEL_INIT_FALSE ((struct jump_label_key) \
+#define STATIC_KEY_INIT_FALSE ((struct static_key) \
{ .enabled = ATOMIC_INIT(0), .entries = (void *)0 })
#else /* !HAVE_JUMP_LABEL */
#include <linux/atomic.h>
-struct jump_label_key {
+struct static_key {
atomic_t enabled;
};
@@ -149,18 +149,18 @@ static __always_inline void jump_label_i
{
}
-struct jump_label_key_deferred {
- struct jump_label_key key;
+struct static_key_deferred {
+ struct static_key key;
};
-static __always_inline bool very_unlikely(struct jump_label_key *key)
+static __always_inline bool very_unlikely(struct static_key *key)
{
if (unlikely(atomic_read(&key->enabled)) > 0)
return true;
return false;
}
-static __always_inline bool very_likely(struct jump_label_key *key)
+static __always_inline bool very_likely(struct static_key *key)
{
if (likely(atomic_read(&key->enabled)) > 0)
return true;
@@ -168,26 +168,26 @@ static __always_inline bool very_likely(
}
/* Deprecated. Please use 'very_unlikely() instead. */
-static __always_inline bool static_branch(struct jump_label_key *key)
+static __always_inline bool static_branch(struct static_key *key)
{
if (unlikely(atomic_read(&key->enabled)) > 0)
return true;
return false;
}
-static inline void jump_label_inc(struct jump_label_key *key)
+static inline void static_key_slow_inc(struct static_key *key)
{
atomic_inc(&key->enabled);
}
-static inline void jump_label_dec(struct jump_label_key *key)
+static inline void static_key_slow_dec(struct static_key *key)
{
atomic_dec(&key->enabled);
}
-static inline void jump_label_dec_deferred(struct jump_label_key_deferred *key)
+static inline void static_key_slow_dec_deferred(struct static_key_deferred *key)
{
- jump_label_dec(&key->key);
+ static_key_slow_dec(&key->key);
}
static inline int jump_label_text_reserved(void *start, void *end)
@@ -198,7 +198,7 @@ static inline int jump_label_text_reserv
static inline void jump_label_lock(void) {}
static inline void jump_label_unlock(void) {}
-static inline bool jump_label_true(struct jump_label_key *key)
+static inline bool static_key_true(struct static_key *key)
{
return (atomic_read(&key->enabled) > 0);
}
@@ -209,19 +209,19 @@ static inline int jump_label_apply_nops(
}
static inline void
-jump_label_rate_limit(struct jump_label_key_deferred *key,
+jump_label_rate_limit(struct static_key_deferred *key,
unsigned long rl)
{
}
-#define JUMP_LABEL_INIT_TRUE ((struct jump_label_key) \
+#define STATIC_KEY_INIT_TRUE ((struct static_key) \
{ .enabled = ATOMIC_INIT(1) })
-#define JUMP_LABEL_INIT_FALSE ((struct jump_label_key) \
+#define STATIC_KEY_INIT_FALSE ((struct static_key) \
{ .enabled = ATOMIC_INIT(0) })
#endif /* HAVE_JUMP_LABEL */
-#define JUMP_LABEL_INIT JUMP_LABEL_INIT_FALSE
-#define jump_label_enabled jump_label_true
+#define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
+#define jump_label_enabled static_key_true
#endif /* _LINUX_JUMP_LABEL_H */
Index: linux/include/linux/netdevice.h
===================================================================
--- linux.orig/include/linux/netdevice.h
+++ linux/include/linux/netdevice.h
@@ -214,8 +214,8 @@ enum {
#include <linux/skbuff.h>
#ifdef CONFIG_RPS
-#include <linux/jump_label.h>
-extern struct jump_label_key rps_needed;
+#include <linux/static_key.h>
+extern struct static_key rps_needed;
#endif
struct neighbour;
Index: linux/include/linux/netfilter.h
===================================================================
--- linux.orig/include/linux/netfilter.h
+++ linux/include/linux/netfilter.h
@@ -163,8 +163,8 @@ extern struct ctl_path nf_net_ipv4_netfi
extern struct list_head nf_hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
#if defined(CONFIG_JUMP_LABEL)
-#include <linux/jump_label.h>
-extern struct jump_label_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
+#include <linux/static_key.h>
+extern struct static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
static inline bool nf_hooks_active(u_int8_t pf, unsigned int hook)
{
if (__builtin_constant_p(pf) &&
Index: linux/include/linux/perf_api.h
===================================================================
--- linux.orig/include/linux/perf_api.h
+++ linux/include/linux/perf_api.h
@@ -29,7 +29,7 @@ static inline void perf_fetch_caller_reg
perf_arch_fetch_caller_regs(regs, ((unsigned long)__builtin_return_address(0)));
}
-extern struct jump_label_key perf_swevent_enabled[];
+extern struct static_key perf_swevent_enabled[];
static __always_inline void
perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
Index: linux/include/linux/perf_event.h
===================================================================
--- linux.orig/include/linux/perf_event.h
+++ linux/include/linux/perf_event.h
@@ -514,7 +514,7 @@ struct perf_guest_info_callbacks {
#include <linux/ftrace.h>
#include <linux/cpu.h>
#include <linux/irq_work.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
#include <linux/atomic.h>
#include <asm/local.h>
@@ -1038,7 +1038,7 @@ static inline int is_software_event(stru
return event->pmu->task_ctx_nr == perf_sw_context;
}
-extern struct jump_label_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
+extern struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
extern void __perf_sw_event(u32, u64, struct pt_regs *, u64);
@@ -1075,7 +1075,7 @@ perf_sw_event(u32 event_id, u64 nr, stru
}
}
-extern struct jump_label_key_deferred perf_sched_events;
+extern struct static_key_deferred perf_sched_events;
static inline void perf_event_task_sched_in(struct task_struct *prev,
struct task_struct *task)
Index: linux/include/linux/static_key.h
===================================================================
--- /dev/null
+++ linux/include/linux/static_key.h
@@ -0,0 +1 @@
+#include <linux/jump_label.h>
Index: linux/include/linux/tracepoint.h
===================================================================
--- linux.orig/include/linux/tracepoint.h
+++ linux/include/linux/tracepoint.h
@@ -17,7 +17,7 @@
#include <linux/errno.h>
#include <linux/types.h>
#include <linux/rcupdate.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
struct module;
struct tracepoint;
@@ -29,7 +29,7 @@ struct tracepoint_func {
struct tracepoint {
const char *name; /* Tracepoint name */
- struct jump_label_key key;
+ struct static_key key;
void (*regfunc)(void);
void (*unregfunc)(void);
struct tracepoint_func __rcu *funcs;
@@ -188,7 +188,7 @@ static inline void tracepoint_synchroniz
__attribute__((section("__tracepoints_strings"))) = #name; \
struct tracepoint __tracepoint_##name \
__attribute__((section("__tracepoints"))) = \
- { __tpstrtab_##name, JUMP_LABEL_INIT_FALSE, reg, unreg, NULL };\
+ { __tpstrtab_##name, STATIC_KEY_INIT_FALSE, reg, unreg, NULL };\
static struct tracepoint * const __tracepoint_ptr_##name __used \
__attribute__((section("__tracepoints_ptrs"))) = \
&__tracepoint_##name;
Index: linux/include/net/sock.h
===================================================================
--- linux.orig/include/net/sock.h
+++ linux/include/net/sock.h
@@ -55,7 +55,7 @@
#include <linux/uaccess.h>
#include <linux/memcontrol.h>
#include <linux/res_counter.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
#include <linux/filter.h>
#include <linux/rculist_nulls.h>
@@ -924,7 +924,7 @@ inline void sk_refcnt_debug_release(cons
#endif /* SOCK_REFCNT_DEBUG */
#if defined(CONFIG_CGROUP_MEM_RES_CTLR_KMEM) && defined(CONFIG_NET)
-extern struct jump_label_key memcg_socket_limit_enabled;
+extern struct static_key memcg_socket_limit_enabled;
static inline struct cg_proto *parent_cg_proto(struct proto *proto,
struct cg_proto *cg_proto)
{
Index: linux/kernel/events/core.c
===================================================================
--- linux.orig/kernel/events/core.c
+++ linux/kernel/events/core.c
@@ -128,7 +128,7 @@ enum event_type_t {
* perf_sched_events : >0 events exist
* perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
*/
-struct jump_label_key_deferred perf_sched_events __read_mostly;
+struct static_key_deferred perf_sched_events __read_mostly;
static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
static atomic_t nr_mmap_events __read_mostly;
@@ -2769,7 +2769,7 @@ static void free_event(struct perf_event
if (!event->parent) {
if (event->attach_state & PERF_ATTACH_TASK)
- jump_label_dec_deferred(&perf_sched_events);
+ static_key_slow_dec_deferred(&perf_sched_events);
if (event->attr.mmap || event->attr.mmap_data)
atomic_dec(&nr_mmap_events);
if (event->attr.comm)
@@ -2780,7 +2780,7 @@ static void free_event(struct perf_event
put_callchain_buffers();
if (is_cgroup_event(event)) {
atomic_dec(&per_cpu(perf_cgroup_events, event->cpu));
- jump_label_dec_deferred(&perf_sched_events);
+ static_key_slow_dec_deferred(&perf_sched_events);
}
}
@@ -4982,7 +4982,7 @@ fail:
return err;
}
-struct jump_label_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
+struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
static void sw_perf_event_destroy(struct perf_event *event)
{
@@ -4990,7 +4990,7 @@ static void sw_perf_event_destroy(struct
WARN_ON(event->parent);
- jump_label_dec(&perf_swevent_enabled[event_id]);
+ static_key_slow_dec(&perf_swevent_enabled[event_id]);
swevent_hlist_put(event);
}
@@ -5020,7 +5020,7 @@ static int perf_swevent_init(struct perf
if (err)
return err;
- jump_label_inc(&perf_swevent_enabled[event_id]);
+ static_key_slow_inc(&perf_swevent_enabled[event_id]);
event->destroy = sw_perf_event_destroy;
}
@@ -5843,7 +5843,7 @@ done:
if (!event->parent) {
if (event->attach_state & PERF_ATTACH_TASK)
- jump_label_inc(&perf_sched_events.key);
+ static_key_slow_inc(&perf_sched_events.key);
if (event->attr.mmap || event->attr.mmap_data)
atomic_inc(&nr_mmap_events);
if (event->attr.comm)
@@ -6081,7 +6081,7 @@ SYSCALL_DEFINE5(perf_event_open,
* - that may need work on context switch
*/
atomic_inc(&per_cpu(perf_cgroup_events, event->cpu));
- jump_label_inc(&perf_sched_events.key);
+ static_key_slow_inc(&perf_sched_events.key);
}
/*
Index: linux/kernel/jump_label.c
===================================================================
--- linux.orig/kernel/jump_label.c
+++ linux/kernel/jump_label.c
@@ -12,7 +12,7 @@
#include <linux/slab.h>
#include <linux/sort.h>
#include <linux/err.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
#ifdef HAVE_JUMP_LABEL
@@ -29,11 +29,11 @@ void jump_label_unlock(void)
mutex_unlock(&jump_label_mutex);
}
-bool jump_label_true(struct jump_label_key *key)
+bool static_key_true(struct static_key *key)
{
return (atomic_read(&key->enabled) > 0);
}
-EXPORT_SYMBOL_GPL(jump_label_true);
+EXPORT_SYMBOL_GPL(static_key_true);
static int jump_label_cmp(const void *a, const void *b)
{
@@ -59,9 +59,9 @@ jump_label_sort_entries(struct jump_entr
sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL);
}
-static void jump_label_update(struct jump_label_key *key, int enable);
+static void jump_label_update(struct static_key *key, int enable);
-void jump_label_inc(struct jump_label_key *key)
+void static_key_slow_inc(struct static_key *key)
{
if (atomic_inc_not_zero(&key->enabled))
return;
@@ -76,9 +76,9 @@ void jump_label_inc(struct jump_label_ke
atomic_inc(&key->enabled);
jump_label_unlock();
}
-EXPORT_SYMBOL_GPL(jump_label_inc);
+EXPORT_SYMBOL_GPL(static_key_slow_inc);
-static void __jump_label_dec(struct jump_label_key *key,
+static void __static_key_slow_dec(struct static_key *key,
unsigned long rate_limit, struct delayed_work *work)
{
if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) {
@@ -101,24 +101,24 @@ static void __jump_label_dec(struct jump
static void jump_label_update_timeout(struct work_struct *work)
{
- struct jump_label_key_deferred *key =
- container_of(work, struct jump_label_key_deferred, work.work);
- __jump_label_dec(&key->key, 0, NULL);
+ struct static_key_deferred *key =
+ container_of(work, struct static_key_deferred, work.work);
+ __static_key_slow_dec(&key->key, 0, NULL);
}
-void jump_label_dec(struct jump_label_key *key)
+void static_key_slow_dec(struct static_key *key)
{
- __jump_label_dec(key, 0, NULL);
+ __static_key_slow_dec(key, 0, NULL);
}
-EXPORT_SYMBOL_GPL(jump_label_dec);
+EXPORT_SYMBOL_GPL(static_key_slow_dec);
-void jump_label_dec_deferred(struct jump_label_key_deferred *key)
+void static_key_slow_dec_deferred(struct static_key_deferred *key)
{
- __jump_label_dec(&key->key, key->timeout, &key->work);
+ __static_key_slow_dec(&key->key, key->timeout, &key->work);
}
-EXPORT_SYMBOL_GPL(jump_label_dec_deferred);
+EXPORT_SYMBOL_GPL(static_key_slow_dec_deferred);
-void jump_label_rate_limit(struct jump_label_key_deferred *key,
+void jump_label_rate_limit(struct static_key_deferred *key,
unsigned long rl)
{
key->timeout = rl;
@@ -161,7 +161,7 @@ void __weak __init_or_module arch_jump_l
arch_jump_label_transform(entry, type);
}
-static void __jump_label_update(struct jump_label_key *key,
+static void __jump_label_update(struct static_key *key,
struct jump_entry *entry,
struct jump_entry *stop, int enable)
{
@@ -178,10 +178,10 @@ static void __jump_label_update(struct j
}
}
-static enum jump_label_type jump_label_type(struct jump_label_key *key)
+static enum jump_label_type jump_label_type(struct static_key *key)
{
bool true_branch = jump_label_get_branch_default(key);
- bool state = jump_label_true(key);
+ bool state = static_key_true(key);
if ((!true_branch && state) || (true_branch && !state))
return JUMP_LABEL_ENABLE;
@@ -193,16 +193,16 @@ void __init jump_label_init(void)
{
struct jump_entry *iter_start = __start___jump_table;
struct jump_entry *iter_stop = __stop___jump_table;
- struct jump_label_key *key = NULL;
+ struct static_key *key = NULL;
struct jump_entry *iter;
jump_label_lock();
jump_label_sort_entries(iter_start, iter_stop);
for (iter = iter_start; iter < iter_stop; iter++) {
- struct jump_label_key *iterk;
+ struct static_key *iterk;
- iterk = (struct jump_label_key *)(unsigned long)iter->key;
+ iterk = (struct static_key *)(unsigned long)iter->key;
arch_jump_label_transform_static(iter, jump_label_type(iterk));
if (iterk == key)
continue;
@@ -221,8 +221,8 @@ void __init jump_label_init(void)
#ifdef CONFIG_MODULES
-struct jump_label_mod {
- struct jump_label_mod *next;
+struct static_key_mod {
+ struct static_key_mod *next;
struct jump_entry *entries;
struct module *mod;
};
@@ -242,9 +242,9 @@ static int __jump_label_mod_text_reserve
start, end);
}
-static void __jump_label_mod_update(struct jump_label_key *key, int enable)
+static void __jump_label_mod_update(struct static_key *key, int enable)
{
- struct jump_label_mod *mod = key->next;
+ struct static_key_mod *mod = key->next;
while (mod) {
struct module *m = mod->mod;
@@ -284,8 +284,8 @@ static int jump_label_add_module(struct
struct jump_entry *iter_start = mod->jump_entries;
struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
struct jump_entry *iter;
- struct jump_label_key *key = NULL;
- struct jump_label_mod *jlm;
+ struct static_key *key = NULL;
+ struct static_key_mod *jlm;
/* if the module doesn't have jump label entries, just return */
if (iter_start == iter_stop)
@@ -294,9 +294,9 @@ static int jump_label_add_module(struct
jump_label_sort_entries(iter_start, iter_stop);
for (iter = iter_start; iter < iter_stop; iter++) {
- struct jump_label_key *iterk;
+ struct static_key *iterk;
- iterk = (struct jump_label_key *)(unsigned long)iter->key;
+ iterk = (struct static_key *)(unsigned long)iter->key;
if (iterk == key)
continue;
@@ -309,7 +309,7 @@ static int jump_label_add_module(struct
key->next = NULL;
continue;
}
- jlm = kzalloc(sizeof(struct jump_label_mod), GFP_KERNEL);
+ jlm = kzalloc(sizeof(struct static_key_mod), GFP_KERNEL);
if (!jlm)
return -ENOMEM;
jlm->mod = mod;
@@ -329,14 +329,14 @@ static void jump_label_del_module(struct
struct jump_entry *iter_start = mod->jump_entries;
struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
struct jump_entry *iter;
- struct jump_label_key *key = NULL;
- struct jump_label_mod *jlm, **prev;
+ struct static_key *key = NULL;
+ struct static_key_mod *jlm, **prev;
for (iter = iter_start; iter < iter_stop; iter++) {
if (iter->key == (jump_label_t)(unsigned long)key)
continue;
- key = (struct jump_label_key *)(unsigned long)iter->key;
+ key = (struct static_key *)(unsigned long)iter->key;
if (__module_address(iter->key) == mod)
continue;
@@ -438,7 +438,7 @@ int jump_label_text_reserved(void *start
return ret;
}
-static void jump_label_update(struct jump_label_key *key, int enable)
+static void jump_label_update(struct static_key *key, int enable)
{
struct jump_entry *stop = __stop___jump_table;
struct jump_entry *entry = jump_label_get_entries(key);
Index: linux/kernel/sched/core.c
===================================================================
--- linux.orig/kernel/sched/core.c
+++ linux/kernel/sched/core.c
@@ -162,13 +162,13 @@ static int sched_feat_show(struct seq_fi
#ifdef HAVE_JUMP_LABEL
-#define jump_label_key__true JUMP_LABEL_INIT_TRUE
-#define jump_label_key__false JUMP_LABEL_INIT_FALSE
+#define jump_label_key__true STATIC_KEY_INIT_TRUE
+#define jump_label_key__false STATIC_KEY_INIT_FALSE
#define SCHED_FEAT(name, enabled) \
jump_label_key__##enabled ,
-struct jump_label_key sched_feat_keys[__SCHED_FEAT_NR] = {
+struct static_key sched_feat_keys[__SCHED_FEAT_NR] = {
#include "features.h"
};
@@ -176,14 +176,14 @@ struct jump_label_key sched_feat_keys[__
static void sched_feat_disable(int i)
{
- if (jump_label_true(&sched_feat_keys[i]))
- jump_label_dec(&sched_feat_keys[i]);
+ if (static_key_true(&sched_feat_keys[i]))
+ static_key_slow_dec(&sched_feat_keys[i]);
}
static void sched_feat_enable(int i)
{
- if (!jump_label_true(&sched_feat_keys[i]))
- jump_label_inc(&sched_feat_keys[i]);
+ if (!static_key_true(&sched_feat_keys[i]))
+ static_key_slow_inc(&sched_feat_keys[i]);
}
#else
static void sched_feat_disable(int i) { };
Index: linux/kernel/sched/fair.c
===================================================================
--- linux.orig/kernel/sched/fair.c
+++ linux/kernel/sched/fair.c
@@ -1399,7 +1399,7 @@ entity_tick(struct cfs_rq *cfs_rq, struc
#ifdef CONFIG_CFS_BANDWIDTH
#ifdef HAVE_JUMP_LABEL
-static struct jump_label_key __cfs_bandwidth_used;
+static struct static_key __cfs_bandwidth_used;
static inline bool cfs_bandwidth_used(void)
{
@@ -1410,9 +1410,9 @@ void account_cfs_bandwidth_used(int enab
{
/* only need to count groups transitioning between enabled/!enabled */
if (enabled && !was_enabled)
- jump_label_inc(&__cfs_bandwidth_used);
+ static_key_slow_inc(&__cfs_bandwidth_used);
else if (!enabled && was_enabled)
- jump_label_dec(&__cfs_bandwidth_used);
+ static_key_slow_dec(&__cfs_bandwidth_used);
}
#else /* HAVE_JUMP_LABEL */
static bool cfs_bandwidth_used(void)
Index: linux/kernel/sched/sched.h
===================================================================
--- linux.orig/kernel/sched/sched.h
+++ linux/kernel/sched/sched.h
@@ -611,7 +611,7 @@ static inline void __set_task_cpu(struct
* Tunables that become constants when CONFIG_SCHED_DEBUG is off:
*/
#ifdef CONFIG_SCHED_DEBUG
-# include <linux/jump_label.h>
+# include <linux/static_key.h>
# define const_debug __read_mostly
#else
# define const_debug const
@@ -630,18 +630,18 @@ enum {
#undef SCHED_FEAT
#if defined(CONFIG_SCHED_DEBUG) && defined(HAVE_JUMP_LABEL)
-static __always_inline bool static_branch__true(struct jump_label_key *key)
+static __always_inline bool static_branch__true(struct static_key *key)
{
return very_likely(key); /* Not out of line branch. */
}
-static __always_inline bool static_branch__false(struct jump_label_key *key)
+static __always_inline bool static_branch__false(struct static_key *key)
{
return very_unlikely(key); /* Out of line branch. */
}
#define SCHED_FEAT(name, enabled) \
-static __always_inline bool static_branch_##name(struct jump_label_key *key) \
+static __always_inline bool static_branch_##name(struct static_key *key) \
{ \
return static_branch__##enabled(key); \
}
@@ -650,7 +650,7 @@ static __always_inline bool static_branc
#undef SCHED_FEAT
-extern struct jump_label_key sched_feat_keys[__SCHED_FEAT_NR];
+extern struct static_key sched_feat_keys[__SCHED_FEAT_NR];
#define sched_feat(x) (static_branch_##x(&sched_feat_keys[__SCHED_FEAT_##x]))
#else /* !(SCHED_DEBUG && HAVE_JUMP_LABEL) */
#define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x))
Index: linux/kernel/tracepoint.c
===================================================================
--- linux.orig/kernel/tracepoint.c
+++ linux/kernel/tracepoint.c
@@ -25,7 +25,7 @@
#include <linux/err.h>
#include <linux/slab.h>
#include <linux/sched.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
extern struct tracepoint * const __start___tracepoints_ptrs[];
extern struct tracepoint * const __stop___tracepoints_ptrs[];
@@ -256,9 +256,9 @@ static void set_tracepoint(struct tracep
{
WARN_ON(strcmp((*entry)->name, elem->name) != 0);
- if (elem->regfunc && !jump_label_true(&elem->key) && active)
+ if (elem->regfunc && !static_key_true(&elem->key) && active)
elem->regfunc();
- else if (elem->unregfunc && jump_label_true(&elem->key) && !active)
+ else if (elem->unregfunc && static_key_true(&elem->key) && !active)
elem->unregfunc();
/*
@@ -269,10 +269,10 @@ static void set_tracepoint(struct tracep
* is used.
*/
rcu_assign_pointer(elem->funcs, (*entry)->funcs);
- if (active && !jump_label_true(&elem->key))
- jump_label_inc(&elem->key);
- else if (!active && jump_label_true(&elem->key))
- jump_label_dec(&elem->key);
+ if (active && !static_key_true(&elem->key))
+ static_key_slow_inc(&elem->key);
+ else if (!active && static_key_true(&elem->key))
+ static_key_slow_dec(&elem->key);
}
/*
@@ -283,11 +283,11 @@ static void set_tracepoint(struct tracep
*/
static void disable_tracepoint(struct tracepoint *elem)
{
- if (elem->unregfunc && jump_label_true(&elem->key))
+ if (elem->unregfunc && static_key_true(&elem->key))
elem->unregfunc();
- if (jump_label_true(&elem->key))
- jump_label_dec(&elem->key);
+ if (static_key_true(&elem->key))
+ static_key_slow_dec(&elem->key);
rcu_assign_pointer(elem->funcs, NULL);
}
Index: linux/net/core/dev.c
===================================================================
--- linux.orig/net/core/dev.c
+++ linux/net/core/dev.c
@@ -134,7 +134,7 @@
#include <linux/inetdevice.h>
#include <linux/cpu_rmap.h>
#include <linux/net_tstamp.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
#include <net/flow_keys.h>
#include "net-sysfs.h"
@@ -1441,11 +1441,11 @@ int call_netdevice_notifiers(unsigned lo
}
EXPORT_SYMBOL(call_netdevice_notifiers);
-static struct jump_label_key netstamp_needed __read_mostly;
+static struct static_key netstamp_needed __read_mostly;
#ifdef HAVE_JUMP_LABEL
-/* We are not allowed to call jump_label_dec() from irq context
+/* We are not allowed to call static_key_slow_dec() from irq context
* If net_disable_timestamp() is called from irq context, defer the
- * jump_label_dec() calls.
+ * static_key_slow_dec() calls.
*/
static atomic_t netstamp_needed_deferred;
#endif
@@ -1457,12 +1457,12 @@ void net_enable_timestamp(void)
if (deferred) {
while (--deferred)
- jump_label_dec(&netstamp_needed);
+ static_key_slow_dec(&netstamp_needed);
return;
}
#endif
WARN_ON(in_interrupt());
- jump_label_inc(&netstamp_needed);
+ static_key_slow_inc(&netstamp_needed);
}
EXPORT_SYMBOL(net_enable_timestamp);
@@ -1474,7 +1474,7 @@ void net_disable_timestamp(void)
return;
}
#endif
- jump_label_dec(&netstamp_needed);
+ static_key_slow_dec(&netstamp_needed);
}
EXPORT_SYMBOL(net_disable_timestamp);
@@ -2660,7 +2660,7 @@ EXPORT_SYMBOL(__skb_get_rxhash);
struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
EXPORT_SYMBOL(rps_sock_flow_table);
-struct jump_label_key rps_needed __read_mostly;
+struct static_key rps_needed __read_mostly;
static struct rps_dev_flow *
set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
Index: linux/net/core/net-sysfs.c
===================================================================
--- linux.orig/net/core/net-sysfs.c
+++ linux/net/core/net-sysfs.c
@@ -608,10 +608,10 @@ static ssize_t store_rps_map(struct netd
spin_unlock(&rps_map_lock);
if (map)
- jump_label_inc(&rps_needed);
+ static_key_slow_inc(&rps_needed);
if (old_map) {
kfree_rcu(old_map, rcu);
- jump_label_dec(&rps_needed);
+ static_key_slow_dec(&rps_needed);
}
free_cpumask_var(mask);
return len;
Index: linux/net/core/sock.c
===================================================================
--- linux.orig/net/core/sock.c
+++ linux/net/core/sock.c
@@ -111,7 +111,7 @@
#include <linux/init.h>
#include <linux/highmem.h>
#include <linux/user_namespace.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
#include <linux/memcontrol.h>
#include <asm/uaccess.h>
@@ -184,7 +184,7 @@ void mem_cgroup_sockets_destroy(struct c
static struct lock_class_key af_family_keys[AF_MAX];
static struct lock_class_key af_family_slock_keys[AF_MAX];
-struct jump_label_key memcg_socket_limit_enabled;
+struct static_key memcg_socket_limit_enabled;
EXPORT_SYMBOL(memcg_socket_limit_enabled);
/*
Index: linux/net/core/sysctl_net_core.c
===================================================================
--- linux.orig/net/core/sysctl_net_core.c
+++ linux/net/core/sysctl_net_core.c
@@ -69,9 +69,9 @@ static int rps_sock_flow_sysctl(ctl_tabl
if (sock_table != orig_sock_table) {
rcu_assign_pointer(rps_sock_flow_table, sock_table);
if (sock_table)
- jump_label_inc(&rps_needed);
+ static_key_slow_inc(&rps_needed);
if (orig_sock_table) {
- jump_label_dec(&rps_needed);
+ static_key_slow_dec(&rps_needed);
synchronize_rcu();
vfree(orig_sock_table);
}
Index: linux/net/ipv4/tcp_memcontrol.c
===================================================================
--- linux.orig/net/ipv4/tcp_memcontrol.c
+++ linux/net/ipv4/tcp_memcontrol.c
@@ -111,7 +111,7 @@ void tcp_destroy_cgroup(struct cgroup *c
val = res_counter_read_u64(&tcp->tcp_memory_allocated, RES_LIMIT);
if (val != RESOURCE_MAX)
- jump_label_dec(&memcg_socket_limit_enabled);
+ static_key_slow_dec(&memcg_socket_limit_enabled);
}
EXPORT_SYMBOL(tcp_destroy_cgroup);
@@ -143,9 +143,9 @@ static int tcp_update_limit(struct mem_c
net->ipv4.sysctl_tcp_mem[i]);
if (val == RESOURCE_MAX && old_lim != RESOURCE_MAX)
- jump_label_dec(&memcg_socket_limit_enabled);
+ static_key_slow_dec(&memcg_socket_limit_enabled);
else if (old_lim == RESOURCE_MAX && val != RESOURCE_MAX)
- jump_label_inc(&memcg_socket_limit_enabled);
+ static_key_slow_inc(&memcg_socket_limit_enabled);
return 0;
}
Index: linux/net/netfilter/core.c
===================================================================
--- linux.orig/net/netfilter/core.c
+++ linux/net/netfilter/core.c
@@ -56,7 +56,7 @@ struct list_head nf_hooks[NFPROTO_NUMPRO
EXPORT_SYMBOL(nf_hooks);
#if defined(CONFIG_JUMP_LABEL)
-struct jump_label_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
+struct static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
EXPORT_SYMBOL(nf_hooks_needed);
#endif
@@ -77,7 +77,7 @@ int nf_register_hook(struct nf_hook_ops
list_add_rcu(®->list, elem->list.prev);
mutex_unlock(&nf_hook_mutex);
#if defined(CONFIG_JUMP_LABEL)
- jump_label_inc(&nf_hooks_needed[reg->pf][reg->hooknum]);
+ static_key_slow_inc(&nf_hooks_needed[reg->pf][reg->hooknum]);
#endif
return 0;
}
@@ -89,7 +89,7 @@ void nf_unregister_hook(struct nf_hook_o
list_del_rcu(®->list);
mutex_unlock(&nf_hook_mutex);
#if defined(CONFIG_JUMP_LABEL)
- jump_label_dec(&nf_hooks_needed[reg->pf][reg->hooknum]);
+ static_key_slow_dec(&nf_hooks_needed[reg->pf][reg->hooknum]);
#endif
synchronize_net();
}
- update the help text as it's not only about trace points anymore
- explain the functional effects of the option prominently,
treating the implementational details as a second level of
information
Additionally we might also consider renaming CONFIG_JUMP_LABEL
to:
CONFIG_OPTIMIZE_STATIC_BRANCHES=y
or so, to make it clearer what it's all really about and makes
it part of the family of similarly named optimizations - but I'm
not too concerned about that, as CONFIG_JUMP_LABEL does not
appear in actual static-key usage sites.
Signed-off-by: Ingo Molnar <[email protected]>
---
arch/Kconfig | 27 +++++++++++++++++++--------
1 files changed, 19 insertions(+), 8 deletions(-)
diff --git a/arch/Kconfig b/arch/Kconfig
index 4f55c73..5b448a7 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -47,18 +47,29 @@ config KPROBES
If in doubt, say "N".
config JUMP_LABEL
- bool "Optimize trace point call sites"
+ bool "Optimize very unlikely/likely branches"
depends on HAVE_ARCH_JUMP_LABEL
help
+ This option enables a transparent branch optimization that
+ makes certain almost-always-true or almost-always-false branch
+ conditions even cheaper to execute within the kernel.
+
+ Certain performance-sensitive kernel code, such as trace points,
+ scheduler functionality, networking code and KVM have such
+ branches and include support for this optimization technique.
+
If it is detected that the compiler has support for "asm goto",
- the kernel will compile trace point locations with just a
- nop instruction. When trace points are enabled, the nop will
- be converted to a jump to the trace function. This technique
- lowers overhead and stress on the branch prediction of the
- processor.
-
- On i386, options added to the compiler flags may increase
- the size of the kernel slightly.
+ the kernel will compile such branches with just a nop
+ instruction. When the condition flag is toggled to true, the
+ nop will be converted to a jump instruction to execute the
+ conditional block of instructions.
+
+ This technique lowers overhead and stress on the branch prediction
+ of the processor and generally makes the kernel faster. The update
+ of the condition is slower, but those are always very rare.
+
+ ( On 32-bit x86, the necessary options added to the compiler
+ flags may increase the size of the kernel slightly. )
config OPTPROBES
def_bool y
I like the blue shed, but anyway...
On Wed, 2012-02-22 at 09:06 +0100, Ingo Molnar wrote:
> * H. Peter Anvin <[email protected]> wrote:
>
> > Not arguing that, but the static aspect is still key... or
> > people will read it as another version of likely/unlikely.
>
> They can *read* it as such, that as is very much intentional!
I mentioned this thread on irc and the first comment I received was:
"is that a new attempt at trying to guide the compiler?"
I personally find the very_unlikely() confusing, but then again I like
the blue shed over the pink one.
-- Steve
* Steven Rostedt <[email protected]> wrote:
> I like the blue shed, but anyway...
>
> On Wed, 2012-02-22 at 09:06 +0100, Ingo Molnar wrote:
> > * H. Peter Anvin <[email protected]> wrote:
> >
> > > Not arguing that, but the static aspect is still key... or
> > > people will read it as another version of likely/unlikely.
> >
> > They can *read* it as such, that as is very much intentional!
>
> I mentioned this thread on irc and the first comment I received was:
>
> "is that a new attempt at trying to guide the compiler?"
It essentially is, implemented partly via compiler help,
combined with runtime code patching, as an extended run-time arm
of the compiler in essence, to make out of line slowpaths even
cheaper to have around - to make the fast-path even faster.
> I personally find the very_unlikely() confusing, but then
> again I like the blue shed over the pink one.
Confusing in what way?
Thanks,
Ingo
On Wed, 2012-02-22 at 14:34 +0100, Ingo Molnar wrote:
> * Steven Rostedt <[email protected]> wrote:
>
> > I like the blue shed, but anyway...
> >
> > On Wed, 2012-02-22 at 09:06 +0100, Ingo Molnar wrote:
> > > * H. Peter Anvin <[email protected]> wrote:
> > >
> > > > Not arguing that, but the static aspect is still key... or
> > > > people will read it as another version of likely/unlikely.
> > >
> > > They can *read* it as such, that as is very much intentional!
> >
> > I mentioned this thread on irc and the first comment I received was:
> >
> > "is that a new attempt at trying to guide the compiler?"
>
> It essentially is, implemented partly via compiler help,
> combined with runtime code patching, as an extended run-time arm
> of the compiler in essence, to make out of line slowpaths even
> cheaper to have around - to make the fast-path even faster.
Actually, I was partly under the impression that we didn't care if it
was in the fast path. Maybe we need three versions. Let me explain.
We have cases where we want it default disabled and the code called when
enabled should be as out of line as possible. Tracing definitely falls
in this pattern. But we could push a "unlikely(static_branch())" for
such a case (or keep it as very_unlikely()).
Then we have cases where it is default enabled, where we can insert the
code in the fast path. Or do we even care how the compiler places it?
Because this leads us to the third use...
The third use is the case were we don't know the branch should be taken
or not until boot. We don't want the compiler to optimize the paths at
all. This example is for things like CPU features or types (as HPA
explained the "if (very_unlikely(cpu_vendor_amd))". This is the category
that we want to have an efficient system for the running hardware. We
can't bias one way or the other at compile time because frankly, we
don't know the answer until run time. This could also be used by modules
that are drivers for several types of hardware, and it can dynamically
change itself to suit the hardware it is driving.
>
> > I personally find the very_unlikely() confusing, but then
> > again I like the blue shed over the pink one.
>
> Confusing in what way?
Because it really just looks like a stronger "unlikely()" and
fundamentally it really isn't. For tracing, sure that can be the way we
look at it, but for the more general case, it is a "We don't know which
way this branch should go most of the time, lets just pick one then
optimize later".
Again, maybe we need a "very_unlikely()" for the tracing case, maybe
even a "very_likely()", but then keep a static_branch() or whatever for
those cases you do not want to optimize at compile time.
-- Steve
* Steven Rostedt ([email protected]) wrote:
[...]
> Actually, I was partly under the impression that we didn't care if it
> was in the fast path. Maybe we need three versions. Let me explain.
>
> We have cases where we want it default disabled and the code called when
> enabled should be as out of line as possible. Tracing definitely falls
> in this pattern. But we could push a "unlikely(static_branch())" for
> such a case (or keep it as very_unlikely()).
>
> Then we have cases where it is default enabled, where we can insert the
> code in the fast path. Or do we even care how the compiler places it?
> Because this leads us to the third use...
>
> The third use is the case were we don't know the branch should be taken
> or not until boot. We don't want the compiler to optimize the paths at
> all. This example is for things like CPU features or types (as HPA
> explained the "if (very_unlikely(cpu_vendor_amd))". This is the category
> that we want to have an efficient system for the running hardware. We
> can't bias one way or the other at compile time because frankly, we
> don't know the answer until run time. This could also be used by modules
> that are drivers for several types of hardware, and it can dynamically
> change itself to suit the hardware it is driving.
>
[...]
One possible naming that might be a good fit:
- read_always_likely()
- read_always_unlikely()
- read_always_branch()
I think it is important to convey both that it is expected to be always
read, pretty much never updated, and the bias, or absence of bias.
I also _like_ to have a relatively long name here, because the update
cost is so high that someone should really think before using this
facility. In my opinion, it's not "just" a stronger likely/unlikely.
Thoughts ?
Thanks,
Mathieu
--
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com
The bike shed is getting really dark.
On Wed, 2012-02-22 at 09:20 -0500, Mathieu Desnoyers wrote:
>
> [...]
> One possible naming that might be a good fit:
>
> - read_always_likely()
> - read_always_unlikely()
> - read_always_branch()
>
> I think it is important to convey both that it is expected to be always
> read, pretty much never updated, and the bias, or absence of bias.
That actually looks even more confusing. "read_always"? What the hell is
that?
>
> I also _like_ to have a relatively long name here, because the update
> cost is so high that someone should really think before using this
> facility. In my opinion, it's not "just" a stronger likely/unlikely.
Then make it what it is (with a long name...)
if (jump_label_likely())
if (jump_label_unlikely())
That's probably the least confusing of the names. And for the cases we
don't care:
if (jump_label_branch())
The above is the most descriptive and I would say the least confusing.
Someone on IRC said that they wish it had jump_label in the name. As
they see there's a "CONFIG_JUMP_LABEL" it tells us where those jump
labels are used.
Need to go and knock down the bike shed now.
-- Steve
* Steven Rostedt <[email protected]> wrote:
> On Wed, 2012-02-22 at 14:34 +0100, Ingo Molnar wrote:
> > * Steven Rostedt <[email protected]> wrote:
> >
> > > I like the blue shed, but anyway...
> > >
> > > On Wed, 2012-02-22 at 09:06 +0100, Ingo Molnar wrote:
> > > > * H. Peter Anvin <[email protected]> wrote:
> > > >
> > > > > Not arguing that, but the static aspect is still key... or
> > > > > people will read it as another version of likely/unlikely.
> > > >
> > > > They can *read* it as such, that as is very much intentional!
> > >
> > > I mentioned this thread on irc and the first comment I received was:
> > >
> > > "is that a new attempt at trying to guide the compiler?"
> >
> > It essentially is, implemented partly via compiler help,
> > combined with runtime code patching, as an extended run-time
> > arm of the compiler in essence, to make out of line
> > slowpaths even cheaper to have around - to make the
> > fast-path even faster.
>
> Actually, I was partly under the impression that we didn't
> care if it was in the fast path. Maybe we need three versions.
> Let me explain.
>
> We have cases where we want it default disabled and the code
> called when enabled should be as out of line as possible.
> Tracing definitely falls in this pattern. But we could push a
> "unlikely(static_branch())" for such a case (or keep it as
> very_unlikely()).
>
> Then we have cases where it is default enabled, where we can
> insert the code in the fast path. Or do we even care how the
> compiler places it? Because this leads us to the third use...
These two variants cover all current uses of jump labels.
> The third use is the case were we don't know the branch should
> be taken or not until boot. We don't want the compiler to
> optimize the paths at all. This example is for things like CPU
> features or types (as HPA explained the "if
> (very_unlikely(cpu_vendor_amd))". This is the category that we
> want to have an efficient system for the running hardware. We
> can't bias one way or the other at compile time because
> frankly, we don't know the answer until run time. This could
> also be used by modules that are drivers for several types of
> hardware, and it can dynamically change itself to suit the
> hardware it is driving.
>
> > > I personally find the very_unlikely() confusing, but then
> > > again I like the blue shed over the pink one.
> >
> > Confusing in what way?
>
> Because it really just looks like a stronger "unlikely()" and
> fundamentally it really isn't. [...]
Well, the fact is that right now it *is* a stronger unlikely()
on architectures that have jump-labels and it's mapped to
unlikely() on others.
> [...] For tracing, sure that can be the way we look at it, but
> for the more general case, it is a "We don't know which way
> this branch should go most of the time, lets just pick one
> then optimize later".
That's not how it's used by the scheduler or by perf - and by
the looks of it that's not how the networking code uses it
either.
I tend to concentrate on the common case.
> Again, maybe we need a "very_unlikely()" for the tracing case,
... and that's the usecase for the scheduler, for events and for
networking. I.e. all current usecases in the kernel.
> maybe even a "very_likely()", but then keep a static_branch()
> or whatever for those cases you do not want to optimize at
> compile time.
Once such uses arise maybe we could add such an 'unbiased'
variant.
But the reality is that today jump labels are used for critical
codepaths that don't want to slow themselves down due to some
rare feature that is commonly off: this is how tracing, the
scheduler, perf events and networking uses jump labels.
Thanks,
Ingo
On 02/22/2012 12:18 AM, Ingo Molnar wrote:
>
> For example could you tell *at a glance* what this does:
>
> + if (!static_branch_def_false(&perf_sched_events.key))
>
> ?
Yes. Unlike
> + if (very_unlikely(&perf_sched_events.key))
... which is actively misleading.
-hpa
--
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel. I don't speak on their behalf.
On Wed, 2012-02-22 at 15:56 +0100, Ingo Molnar wrote:
> > Again, maybe we need a "very_unlikely()" for the tracing case,
>
> ... and that's the usecase for the scheduler, for events and for
> networking. I.e. all current usecases in the kernel
The scheduler also really wants the very_likely() variant. We have less
use for the unbiased one though.
On 02/22/2012 06:56 AM, Ingo Molnar wrote:
>
>> maybe even a "very_likely()", but then keep a static_branch()
>> or whatever for those cases you do not want to optimize at
>> compile time.
>
> Once such uses arise maybe we could add such an 'unbiased'
> variant.
>
We already have such use cases, although a lot of them are covered by
static_cpu_has(). However, I fully expect that we'll have cases that
aren't readily covered by CPU feature flags and I'd like to avoid
reinventing new features.
-hpa
--
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel. I don't speak on their behalf.
On Wed, 2012-02-22 at 15:56 +0100, Ingo Molnar wrote:
> > Because it really just looks like a stronger "unlikely()" and
> > fundamentally it really isn't. [...]
>
> Well, the fact is that right now it *is* a stronger unlikely()
> on architectures that have jump-labels and it's mapped to
> unlikely() on others.
>
Has gcc been fix to make it truly an unlikely case and remove the "jmp;
jmp" problem of before? I'm still using gcc 4.6.0 which has the
following code for a tracepoint (example is the
trace_sched_migrate_task().
5b4a: e9 00 00 00 00 jmpq 5b4f <set_task_cpu+0x5e>
The above is the jump label that turns into a nop at boot up.
5b4f: eb 19 jmp 5b6a <set_task_cpu+0x79>
Here we jump over some of the trace code (this is the fast path)
5b51: 49 8b 7d 08 mov 0x8(%r13),%rdi
5b55: 44 89 e2 mov %r12d,%edx
5b58: 48 89 de mov %rbx,%rsi
5b5b: 41 ff 55 00 callq *0x0(%r13)
5b5f: 49 83 c5 10 add $0x10,%r13
5b63: 49 83 7d 00 00 cmpq $0x0,0x0(%r13)
5b68: eb 41 jmp 5bab <set_task_cpu+0xba>
Below is the continuation of the fast path.
5b6a: 48 8b 43 08 mov 0x8(%rbx),%rax
5b6e: 44 39 60 18 cmp %r12d,0x18(%rax)
5b72: 74 0c je 5b80 <set_task_cpu+0x8f>
Again, I'm using gcc 4.6.0 and maybe it has been fixed.
-- Steve
On Wed, 2012-02-22 at 08:54 -0500, Steven Rostedt wrote:
> But we could push a "unlikely(static_branch())" for
> such a case (or keep it as very_unlikely()).
If only that would work.. GCC assumes a bunch of things when you use
'asm goto'. An unbiased 'asm goto' that would take likely() and
unlikely() hints would be ideal, but alas that's not how the thing got
implemented.
Now arguable we could maybe just stick with the static_branch() thing
and decide that GCC is broken for adding bias and not respecting
likely() and unlikely(), but the GCC folks might have an opinion there.
Anyway, if we have to stick with the current implementation then an
unbiased version is impossible to implement.
If we get GCC folks on board to change stuff, who knows.
On 02/22/2012 07:12 AM, Steven Rostedt wrote:
> On Wed, 2012-02-22 at 15:56 +0100, Ingo Molnar wrote:
>
>>> Because it really just looks like a stronger "unlikely()" and
>>> fundamentally it really isn't. [...]
>>
>> Well, the fact is that right now it *is* a stronger unlikely()
>> on architectures that have jump-labels and it's mapped to
>> unlikely() on others.
>>
>
> Has gcc been fix to make it truly an unlikely case and remove the "jmp;
> jmp" problem of before? I'm still using gcc 4.6.0 which has the
> following code for a tracepoint (example is the
> trace_sched_migrate_task().
>
No, the jmp jmp problem still exists... I have discussed it with the gcc
folks and they have an idea for how to fix it, but I haven't even gotten
around to filing a formal RFE.
-hpa
--
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel. I don't speak on their behalf.
On 02/22/2012 07:12 AM, Steven Rostedt wrote:
>
> Again, I'm using gcc 4.6.0 and maybe it has been fixed.
>
I think this really comes down to the branches being as biased as you
might have wanted for this case. gcc will reorder its basic blocks
based on what it thinks is best based on some kinds of voodoo and asm
goto is only one type of control transfer it will take into account.
-hpa
--
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel. I don't speak on their behalf.
On Wed, Feb 22, 2012 at 03:56:16PM +0100, Ingo Molnar wrote:
> * Steven Rostedt <[email protected]> wrote:
>
> > On Wed, 2012-02-22 at 14:34 +0100, Ingo Molnar wrote:
> > > * Steven Rostedt <[email protected]> wrote:
> > >
> > > > I like the blue shed, but anyway...
> > > >
> > > > On Wed, 2012-02-22 at 09:06 +0100, Ingo Molnar wrote:
> > > > > * H. Peter Anvin <[email protected]> wrote:
> > > > >
> > > > > > Not arguing that, but the static aspect is still key... or
> > > > > > people will read it as another version of likely/unlikely.
> > > > >
> > > > > They can *read* it as such, that as is very much intentional!
> > > >
> > > > I mentioned this thread on irc and the first comment I received was:
> > > >
> > > > "is that a new attempt at trying to guide the compiler?"
> > >
> > > It essentially is, implemented partly via compiler help,
> > > combined with runtime code patching, as an extended run-time
> > > arm of the compiler in essence, to make out of line
> > > slowpaths even cheaper to have around - to make the
> > > fast-path even faster.
> >
> > Actually, I was partly under the impression that we didn't
> > care if it was in the fast path. Maybe we need three versions.
> > Let me explain.
> >
> > We have cases where we want it default disabled and the code
> > called when enabled should be as out of line as possible.
> > Tracing definitely falls in this pattern. But we could push a
> > "unlikely(static_branch())" for such a case (or keep it as
> > very_unlikely()).
> >
> > Then we have cases where it is default enabled, where we can
> > insert the code in the fast path. Or do we even care how the
> > compiler places it? Because this leads us to the third use...
>
> These two variants cover all current uses of jump labels.
>
> > The third use is the case were we don't know the branch should
> > be taken or not until boot. We don't want the compiler to
> > optimize the paths at all. This example is for things like CPU
> > features or types (as HPA explained the "if
> > (very_unlikely(cpu_vendor_amd))". This is the category that we
> > want to have an efficient system for the running hardware. We
> > can't bias one way or the other at compile time because
> > frankly, we don't know the answer until run time. This could
> > also be used by modules that are drivers for several types of
> > hardware, and it can dynamically change itself to suit the
> > hardware it is driving.
> >
> > > > I personally find the very_unlikely() confusing, but then
> > > > again I like the blue shed over the pink one.
> > >
> > > Confusing in what way?
> >
> > Because it really just looks like a stronger "unlikely()" and
> > fundamentally it really isn't. [...]
>
> Well, the fact is that right now it *is* a stronger unlikely()
> on architectures that have jump-labels and it's mapped to
> unlikely() on others.
>
> > [...] For tracing, sure that can be the way we look at it, but
> > for the more general case, it is a "We don't know which way
> > this branch should go most of the time, lets just pick one
> > then optimize later".
>
> That's not how it's used by the scheduler or by perf - and by
> the looks of it that's not how the networking code uses it
> either.
>
> I tend to concentrate on the common case.
>
> > Again, maybe we need a "very_unlikely()" for the tracing case,
>
> ... and that's the usecase for the scheduler, for events and for
> networking. I.e. all current usecases in the kernel.
>
> > maybe even a "very_likely()", but then keep a static_branch()
> > or whatever for those cases you do not want to optimize at
> > compile time.
>
> Once such uses arise maybe we could add such an 'unbiased'
> variant.
>
Right, we've discussed an 'unbiased' variant a bit before, but I don't
have a working imlementation for it yet. But I agree all current users are of
the biased variety.
Also, decoupling the higher level interfaces, as Ingo has suggested -
'very_unlikely' 'static_key', from the lower level 'jump label' naming makes
sense. We might eventually add an 'unbias' variant. Another potential variant
is in the 'bias' case is to move the mostly not taken branch even further out
of line. Another possibilty is a 'switch'-like variant or multi-way jump.
Thanks,
-Jason
On Wed, 2012-02-22 at 16:13 +0100, Peter Zijlstra wrote:
> On Wed, 2012-02-22 at 08:54 -0500, Steven Rostedt wrote:
> > But we could push a "unlikely(static_branch())" for
> > such a case (or keep it as very_unlikely()).
>
> If only that would work.. GCC assumes a bunch of things when you use
> 'asm goto'. An unbiased 'asm goto' that would take likely() and
> unlikely() hints would be ideal, but alas that's not how the thing got
> implemented.
>
> Now arguable we could maybe just stick with the static_branch() thing
> and decide that GCC is broken for adding bias and not respecting
> likely() and unlikely(), but the GCC folks might have an opinion there.
>
> Anyway, if we have to stick with the current implementation then an
> unbiased version is impossible to implement.
>
> If we get GCC folks on board to change stuff, who knows.
So I clicked the link Jason provided in his 10/10 Documentation patch
and stumbled upon:
http://gcc.gnu.org/ml/gcc-patches/2009-07/msg01558.html
Where rth suggests that __attribute__((hot,cold)) might work on the
destination labels. Trying this my compiler (4.6.1+crap) pukes all over
me suggesting this isn't (yet) implemented.
Richard, is something like that still on the table?
On 02/22/2012 07:19 AM, Jason Baron wrote:
>
> Right, we've discussed an 'unbiased' variant a bit before, but I don't
> have a working imlementation for it yet. But I agree all current users are of
> the biased variety.
>
> Also, decoupling the higher level interfaces, as Ingo has suggested -
> 'very_unlikely' 'static_key', from the lower level 'jump label' naming makes
> sense. We might eventually add an 'unbias' variant. Another potential variant
> is in the 'bias' case is to move the mostly not taken branch even further out
> of line. Another possibilty is a 'switch'-like variant or multi-way jump.
>
Okay, at least we need something to connect the readout with the key,
otherwise there is nothing that even hints that they are part of the
same fundamental subsystem...
I don't know if key_likely() and key_unlikely() would make sense...
-hpa
--
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel. I don't speak on their behalf.
On Wed, Feb 22, 2012 at 10:12:13AM -0500, Steven Rostedt wrote:
> On Wed, 2012-02-22 at 15:56 +0100, Ingo Molnar wrote:
>
> > > Because it really just looks like a stronger "unlikely()" and
> > > fundamentally it really isn't. [...]
> >
> > Well, the fact is that right now it *is* a stronger unlikely()
> > on architectures that have jump-labels and it's mapped to
> > unlikely() on others.
> >
>
> Has gcc been fix to make it truly an unlikely case and remove the "jmp;
> jmp" problem of before? I'm still using gcc 4.6.0 which has the
> following code for a tracepoint (example is the
> trace_sched_migrate_task().
>
> 5b4a: e9 00 00 00 00 jmpq 5b4f <set_task_cpu+0x5e>
>
> The above is the jump label that turns into a nop at boot up.
>
> 5b4f: eb 19 jmp 5b6a <set_task_cpu+0x79>
>
> Here we jump over some of the trace code (this is the fast path)
>
> 5b51: 49 8b 7d 08 mov 0x8(%r13),%rdi
> 5b55: 44 89 e2 mov %r12d,%edx
> 5b58: 48 89 de mov %rbx,%rsi
> 5b5b: 41 ff 55 00 callq *0x0(%r13)
> 5b5f: 49 83 c5 10 add $0x10,%r13
> 5b63: 49 83 7d 00 00 cmpq $0x0,0x0(%r13)
> 5b68: eb 41 jmp 5bab <set_task_cpu+0xba>
>
> Below is the continuation of the fast path.
>
> 5b6a: 48 8b 43 08 mov 0x8(%rbx),%rax
> 5b6e: 44 39 60 18 cmp %r12d,0x18(%rax)
> 5b72: 74 0c je 5b80 <set_task_cpu+0x8f>
>
>
> Again, I'm using gcc 4.6.0 and maybe it has been fixed.
>
> -- Steve
>
>
Hi Steve,
Using 4.6.2, I don't see the 'jmp;jmp' issue. I have:
ffffffff810610c0 <set_task_cpu>:
ffffffff810610c0: 55 push %rbp
ffffffff810610c1: 48 89 e5 mov %rsp,%rbp
ffffffff810610c4: 48 81 ec e0 00 00 00 sub $0xe0,%rsp
ffffffff810610cb: 48 89 5d d8 mov %rbx,-0x28(%rbp)
ffffffff810610cf: 4c 89 65 e0 mov %r12,-0x20(%rbp)
ffffffff810610d3: 48 89 fb mov %rdi,%rbx
ffffffff810610d6: 4c 89 6d e8 mov %r13,-0x18(%rbp)
ffffffff810610da: 4c 89 75 f0 mov %r14,-0x10(%rbp)
ffffffff810610de: 41 89 f4 mov %esi,%r12d
ffffffff810610e1: 4c 89 7d f8 mov %r15,-0x8(%rbp)
no double jump here.
ffffffff810610e5: e9 00 00 00 00 jmpq ffffffff810610ea <set_task_cpu+0x2a>
ffffffff810610ea: 48 8b 43 08 mov 0x8(%rbx),%rax
ffffffff810610ee: 44 3b 60 18 cmp 0x18(%rax),%r12d
ffffffff810610f2: 74 0d je ffffffff81061101 <set_task_cpu+0x41>
ffffffff810610f4: 48 83 83 a8 00 00 00 addq $0x1,0xa8(%rbx)
ffffffff810610fb: 01
no double jump here.
ffffffff810610fc: e9 00 00 00 00 jmpq ffffffff81061101 <set_task_cpu+0x41>
ffffffff81061101: 48 8b 83 60 06 00 00 mov 0x660(%rbx),%rax
ffffffff81061108: 48 8b 50 40 mov 0x40(%rax),%rdx
ffffffff8106110c: 44 89 e0 mov %r12d,%eax
ffffffff8106110f: 48 8b 4a 28 mov 0x28(%rdx),%rcx
ffffffff81061113: 48 8b 0c c1 mov (%rcx,%rax,8),%rcx
ffffffff81061117: 48 89 8b 90 01 00 00 mov %rcx,0x190(%rbx)
ffffffff8106111e: 48 8b 52 20 mov 0x20(%rdx),%rdx
ffffffff81061122: 48 8b 04 c2 mov (%rdx,%rax,8),%rax
ffffffff81061126: 48 89 83 88 01 00 00 mov %rax,0x188(%rbx)
ffffffff8106112d: 48 8b 43 08 mov 0x8(%rbx),%rax
ffffffff81061131: 44 89 60 18 mov %r12d,0x18(%rax)
ffffffff81061135: 48 8b 5d d8 mov -0x28(%rbp),%rbx
ffffffff81061139: 4c 8b 65 e0 mov -0x20(%rbp),%r12
ffffffff8106113d: 4c 8b 6d e8 mov -0x18(%rbp),%r13
ffffffff81061141: 4c 8b 75 f0 mov -0x10(%rbp),%r14
ffffffff81061145: 4c 8b 7d f8 mov -0x8(%rbp),%r15
ffffffff81061149: c9 leaveq
ffffffff8106114a: c3 retq
....
Do you have 'CONFIG_CC_OPTIMIZE_FOR_SIZE' set?
Thanks,
-Jason
* Peter Zijlstra <[email protected]> wrote:
> On Wed, 2012-02-22 at 15:56 +0100, Ingo Molnar wrote:
> > > Again, maybe we need a "very_unlikely()" for the tracing case,
> >
> > ... and that's the usecase for the scheduler, for events and for
> > networking. I.e. all current usecases in the kernel
>
> The scheduler also really wants the very_likely() variant. We
> have less use for the unbiased one though.
Yeah, it wants a fastpath and a slowpath and whether it's likely
or unlikely is situational. So thus it uses both variants in my
tree:
kernel/sched/core.c: if (very_unlikely((¶virt_steal_rq_enabled))) {
kernel/sched/core.c: if (very_unlikely(¶virt_steal_enabled)) {
kernel/sched/fair.c: return very_unlikely(&__cfs_bandwidth_used);
kernel/sched/sched.h: return very_likely(key); /* Not out of line branch. */
kernel/sched/sched.h: return very_unlikely(key); /* Out of line branch. */
With no need for an 'unbiased' method.
Let me push out what I have in tip:perf/jump-labels (just
finished testing it) so that we are all looking at the same
code.
Thanks,
Ingo
* H. Peter Anvin <[email protected]> wrote:
> On 02/22/2012 12:18 AM, Ingo Molnar wrote:
> >
> > For example could you tell *at a glance* what this does:
> >
> > + if (!static_branch_def_false(&perf_sched_events.key))
> >
> > ?
>
> Yes. [...]
You have specially built eyes I guess - congrats, you are member
of the 1% :-)
> [...] Unlike
>
> > + if (very_unlikely(&perf_sched_events.key))
>
> ... which is actively misleading.
Misleading in what way? It clearly conveys that the slowpath
that follows is a slowpath, that what follows is 'very
unlikely'.
This is how all current kernel code is using these facilities.
If you have a new usecase (CPU feature flags) then you should
outline that, instead of expecting something of current usecases
that they are clearly not ...
Thanks,
Ingo
On Wed, 2012-02-22 at 10:42 -0500, Jason Baron wrote:
> Do you have 'CONFIG_CC_OPTIMIZE_FOR_SIZE' set?
Yes I do :-p
But that's because the distro config I started with had it set too.
-- Steve
On Wed, Feb 22, 2012 at 10:54:03AM -0500, Steven Rostedt wrote:
> On Wed, 2012-02-22 at 10:42 -0500, Jason Baron wrote:
>
> > Do you have 'CONFIG_CC_OPTIMIZE_FOR_SIZE' set?
>
> Yes I do :-p
>
> But that's because the distro config I started with had it set too.
>
> -- Steve
>
>
Right, with CONFIG_CC_OPTIMIZE_FOR_SIZE unset, which is the default, at
least on x86, I have not seen a single case of the 'jmp;jmp'.
Thanks,
-Jason
On Wed, 2012-02-22 at 10:56 -0500, Jason Baron wrote:
> On Wed, Feb 22, 2012 at 10:54:03AM -0500, Steven Rostedt wrote:
> > On Wed, 2012-02-22 at 10:42 -0500, Jason Baron wrote:
> >
> > > Do you have 'CONFIG_CC_OPTIMIZE_FOR_SIZE' set?
> >
> > Yes I do :-p
> >
> > But that's because the distro config I started with had it set too.
> >
> > -- Steve
> >
> >
>
> Right, with CONFIG_CC_OPTIMIZE_FOR_SIZE unset, which is the default, at
> least on x86, I have not seen a single case of the 'jmp;jmp'.
>
What the config default is, is unimportant. It only maters to kernel
hackers that build our own kernels. What's important is what distros do.
On debian testing I have in config-3.0.0-1-amd64:
CONFIG_CC_OPTIMIZE_FOR_SIZE=y
Although it is not set on the latest fedora 16 config.
Someone should tell debian to turn it off.
-- Steve
On 02/22/12 07:32, Peter Zijlstra wrote:
> So I clicked the link Jason provided in his 10/10 Documentation patch
> and stumbled upon:
>
> http://gcc.gnu.org/ml/gcc-patches/2009-07/msg01558.html
>
> Where rth suggests that __attribute__((hot,cold)) might work on the
> destination labels. Trying this my compiler (4.6.1+crap) pukes all over
> me suggesting this isn't (yet) implemented.
>
> Richard, is something like that still on the table?
It's still a possibility. I gave Jason a patch for that quite some time
ago; I don't recall hearing whether it turned out to actually be useful.
r~
On 02/22/2012 09:14 AM, Richard Henderson wrote:
> On 02/22/12 07:32, Peter Zijlstra wrote:
>> So I clicked the link Jason provided in his 10/10 Documentation patch
>> and stumbled upon:
>>
>> http://gcc.gnu.org/ml/gcc-patches/2009-07/msg01558.html
>>
>> Where rth suggests that __attribute__((hot,cold)) might work on the
>> destination labels. Trying this my compiler (4.6.1+crap) pukes all over
>> me suggesting this isn't (yet) implemented.
>>
>> Richard, is something like that still on the table?
>
> It's still a possibility. I gave Jason a patch for that quite some time
> ago; I don't recall hearing whether it turned out to actually be useful.
>
Hi Richard,
One issue we also have is with the jmp;jmp problem... which
fundamentally comes from the following issue:
when asm goto() is used without a fallthrough (a __builtin_unreachable()
immediately after it, only possible in gcc 4.6.1+) then gcc assumes that
it can reorder the successor blocks arbitrarily, since it has to "jump
anyway". This eliminates the very useful optimization of replacing the
jump with a NOP in the common case.
The alternative, having a fallthrough, means that if gcc has to jump
anyway, then you end up with a jump to a jump, even if the first of
those jumps can usually be nullified.
I talked to H.J. about this, and he suggested that we'd do something
like "assume the first label in the asm goto is the preferred
fallthrough." I never got around to writing up an RFE bugzilla on this,
but do you have any feelings about how useful this would be?
-hpa
--
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel. I don't speak on their behalf.
On Wed, Feb 22, 2012 at 09:14:19AM -0800, Richard Henderson wrote:
> On 02/22/12 07:32, Peter Zijlstra wrote:
> > So I clicked the link Jason provided in his 10/10 Documentation patch
> > and stumbled upon:
> >
> > http://gcc.gnu.org/ml/gcc-patches/2009-07/msg01558.html
> >
> > Where rth suggests that __attribute__((hot,cold)) might work on the
> > destination labels. Trying this my compiler (4.6.1+crap) pukes all over
> > me suggesting this isn't (yet) implemented.
> >
> > Richard, is something like that still on the table?
>
> It's still a possibility. I gave Jason a patch for that quite some time
> ago; I don't recall hearing whether it turned out to actually be useful.
>
>
> r~
I don't think I ever quite got it working, and I've unfortunately
misplaced it at this point. I am definitely interested in trying it
again though. If you can re-send it, I'll try it.
Also, I think I seem to recall it wouldn't help the -Os case, b/c we
don't get block re-ordering?
Thanks,
-Jason
On 02/22/2012 10:58 AM, Jason Baron wrote:
>
> I don't think I ever quite got it working, and I've unfortunately
> misplaced it at this point. I am definitely interested in trying it
> again though. If you can re-send it, I'll try it.
>
> Also, I think I seem to recall it wouldn't help the -Os case, b/c we
> don't get block re-ordering?
>
With -Os, your code is going to suck eggs no matter what.
-hpa
--
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel. I don't speak on their behalf.
On Wed, Feb 22, 2012 at 09:18:55AM +0100, Ingo Molnar wrote:
> The problem with static_branch_def_false/def_true was that the
> very intuitively visible bias that we see with
> likely()/unlikely() is confused in jump label constructs through
> two layers of modifiers. And the fix is so easy, a simple rename
> in most cases ;-)
>
> So instead of that, in this series we have:
>
> + if (very_unlikely(&perf_sched_events.key))
>
> which is a heck of an improvement IMO. I'd still up its
> readability a notch, by also signalling the overhead of the
> update path by making it:
>
> + if (very_unlikely(&perf_sched_events.slow_flag))
>
> ... but I don't want to be that much of a readability nazi ;-)
I have to say I don't like the "very_unlikely" name. It's confusing
because the condition being evaluated appears to be the address of
something, i.e. &perf_sched_events.key in your example, and that looks
to me to be very very likely to be true, i.e. non-zero. But the code
is telling me that's very *un*likely, which is confusing.
In other words I can't think about "very_unlikely" as being similar to
"unlikely". Clearly very_unlikely isn't just passing through the
true/falseness of its argument to the surrounding if, the way that
unlikely does; it (very_unlikely) is a real function that is doing
something more complicated with its argument -- dereferencing it,
conceptually at least, for a start.
Just from a readability perspective, I like the static_branch name,
and if we want to add information about the execution bias, I'd
suggest we do:
if (likely(static_branch(&x))) ...
if (unlikely(static_branch(&x))) ...
if that can be made to work, or else have variants like this:
if (likely_static_branch(&x)) ...
if (unlikely_static_branch(&x)) ...
Paul.
* Paul Mackerras <[email protected]> wrote:
> On Wed, Feb 22, 2012 at 09:18:55AM +0100, Ingo Molnar wrote:
>
> > The problem with static_branch_def_false/def_true was that the
> > very intuitively visible bias that we see with
> > likely()/unlikely() is confused in jump label constructs through
> > two layers of modifiers. And the fix is so easy, a simple rename
> > in most cases ;-)
> >
> > So instead of that, in this series we have:
> >
> > + if (very_unlikely(&perf_sched_events.key))
> >
> > which is a heck of an improvement IMO. I'd still up its
> > readability a notch, by also signalling the overhead of the
> > update path by making it:
> >
> > + if (very_unlikely(&perf_sched_events.slow_flag))
> >
> > ... but I don't want to be that much of a readability nazi ;-)
>
> I have to say I don't like the "very_unlikely" name. It's
> confusing because the condition being evaluated appears to be
> the address of something, i.e. &perf_sched_events.key in your
> example, and that looks to me to be very very likely to be
> true, i.e. non-zero. But the code is telling me that's very
> *un*likely, which is confusing.
Having to take the address gives us type safety - i.e. it will
not be possible to accidentally pass in a non-jump-label key and
get it misinterpreted.
If some macro magic could be used to remove the address taking
I'd be in favor of such a simplification, i.e.:
if (very_unlikely(perf_sched_events.key))
which should address your observation.
Thanks,
Ingo
On Thu, Feb 23, 2012 at 11:02:05AM +0100, Ingo Molnar wrote:
> * Paul Mackerras <[email protected]> wrote:
>
> > On Wed, Feb 22, 2012 at 09:18:55AM +0100, Ingo Molnar wrote:
> >
> > > The problem with static_branch_def_false/def_true was that the
> > > very intuitively visible bias that we see with
> > > likely()/unlikely() is confused in jump label constructs through
> > > two layers of modifiers. And the fix is so easy, a simple rename
> > > in most cases ;-)
> > >
> > > So instead of that, in this series we have:
> > >
> > > + if (very_unlikely(&perf_sched_events.key))
> > >
> > > which is a heck of an improvement IMO. I'd still up its
> > > readability a notch, by also signalling the overhead of the
> > > update path by making it:
> > >
> > > + if (very_unlikely(&perf_sched_events.slow_flag))
> > >
> > > ... but I don't want to be that much of a readability nazi ;-)
> >
> > I have to say I don't like the "very_unlikely" name. It's
> > confusing because the condition being evaluated appears to be
> > the address of something, i.e. &perf_sched_events.key in your
> > example, and that looks to me to be very very likely to be
> > true, i.e. non-zero. But the code is telling me that's very
> > *un*likely, which is confusing.
>
> Having to take the address gives us type safety - i.e. it will
> not be possible to accidentally pass in a non-jump-label key and
> get it misinterpreted.
>
> If some macro magic could be used to remove the address taking
> I'd be in favor of such a simplification, i.e.:
>
> if (very_unlikely(perf_sched_events.key))
>
> which should address your observation.
>
> Thanks,
>
> Ingo
So, we could get rid of the '&' with something as simple as:
#define very_unlikely(key) __very_unlikely(&key)
However, it does seem potentially more error prone, b/c if 'key' is
passed to a function, and they we do the very_unlikely() we end up with
the address parameter (due to pass by value). That said, it doesn't look
like anybody is using very_unlikely() in that manner in the tree, and we
could document the usage.
In any case, I did the conversion, to see what it would look like, if
anybody is interested:
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index f0c6fd6..d4cd771 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -438,9 +438,9 @@ void __init kvm_guest_init(void)
static __init int activate_jump_labels(void)
{
if (has_steal_clock) {
- jump_label_inc(¶virt_steal_enabled);
+ jump_label_inc(paravirt_steal_enabled);
if (steal_acc)
- jump_label_inc(¶virt_steal_rq_enabled);
+ jump_label_inc(paravirt_steal_rq_enabled);
}
return 0;
diff --git a/arch/x86/kvm/mmu_audit.c b/arch/x86/kvm/mmu_audit.c
index 7709e02..d4cf406 100644
--- a/arch/x86/kvm/mmu_audit.c
+++ b/arch/x86/kvm/mmu_audit.c
@@ -250,7 +250,7 @@ static void __kvm_mmu_audit(struct kvm_vcpu *vcpu, int point)
static inline void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point)
{
- if (very_unlikely((&mmu_audit_key)))
+ if (very_unlikely((mmu_audit_key)))
__kvm_mmu_audit(vcpu, point);
}
@@ -259,7 +259,7 @@ static void mmu_audit_enable(void)
if (mmu_audit)
return;
- jump_label_inc(&mmu_audit_key);
+ jump_label_inc(mmu_audit_key);
mmu_audit = true;
}
@@ -268,7 +268,7 @@ static void mmu_audit_disable(void)
if (!mmu_audit)
return;
- jump_label_dec(&mmu_audit_key);
+ jump_label_dec(mmu_audit_key);
mmu_audit = false;
}
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index 757d8dc..06e29f2 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -79,6 +79,16 @@ enum jump_label_type {
struct module;
+
+/* this defines the usage interface, maybe separate header ? */
+#define very_likely(key) __very_likely(&key)
+#define very_unlikely(key) __very_unlikely(&key)
+#define jump_label_inc(key) __jump_label_inc(&key)
+#define jump_label_dec(key) __jump_label_dec(&key)
+#define jump_label_dec_deferred(key) __jump_label_dec_deferred(&key)
+#define jump_label_true(key) __jump_label_true(&key)
+#define jump_label_rate_limit(key, rl) __jump_label_rate_limit(&key, rl)
+
#ifdef HAVE_JUMP_LABEL
#define JUMP_LABEL_TRUE_BRANCH 1UL
@@ -97,14 +107,18 @@ static inline bool jump_label_get_branch_default(struct jump_label_key *key)
return false;
}
-static __always_inline bool very_unlikely(struct jump_label_key *key)
+#define very_unlikely(key) __very_unlikely(&key)
+
+static __always_inline bool __very_unlikely(struct jump_label_key *key)
{
return arch_static_branch(key);
}
-static __always_inline bool very_likely(struct jump_label_key *key)
+#define very_likely(key) __very_likely(&key)
+
+static __always_inline bool __very_likely(struct jump_label_key *key)
{
- return !very_unlikely(key);
+ return !__very_unlikely(key);
}
/* Deprecated. Please use 'very_unlikely() instead. */
@@ -124,13 +138,13 @@ extern void arch_jump_label_transform(struct jump_entry *entry,
extern void arch_jump_label_transform_static(struct jump_entry *entry,
enum jump_label_type type);
extern int jump_label_text_reserved(void *start, void *end);
-extern void jump_label_inc(struct jump_label_key *key);
-extern void jump_label_dec(struct jump_label_key *key);
-extern void jump_label_dec_deferred(struct jump_label_key_deferred *key);
-extern bool jump_label_true(struct jump_label_key *key);
+extern void __jump_label_inc(struct jump_label_key *key);
+extern void __jump_label_dec(struct jump_label_key *key);
+extern void __jump_label_dec_deferred(struct jump_label_key_deferred *key);
+extern bool __jump_label_true(struct jump_label_key *key);
extern void jump_label_apply_nops(struct module *mod);
extern void
-jump_label_rate_limit(struct jump_label_key_deferred *key, unsigned long rl);
+__jump_label_rate_limit(struct jump_label_key_deferred *key, unsigned long rl);
#define JUMP_LABEL_INIT_TRUE ((struct jump_label_key) \
{ .enabled = ATOMIC_INIT(1), .entries = (void *)1 })
@@ -153,14 +167,18 @@ struct jump_label_key_deferred {
struct jump_label_key key;
};
-static __always_inline bool very_unlikely(struct jump_label_key *key)
+#define very_unlikely(key) __very_unlikely(&key)
+
+static __always_inline bool __very_unlikely(struct jump_label_key *key)
{
if (unlikely(atomic_read(&key->enabled)) > 0)
return true;
return false;
}
-static __always_inline bool very_likely(struct jump_label_key *key)
+#define very_likely(key) __very_likely(&key)
+
+static __always_inline bool __very_likely(struct jump_label_key *key)
{
if (likely(atomic_read(&key->enabled)) > 0)
return true;
@@ -175,19 +193,19 @@ static __always_inline bool static_branch(struct jump_label_key *key)
return false;
}
-static inline void jump_label_inc(struct jump_label_key *key)
+static inline void __jump_label_inc(struct jump_label_key *key)
{
atomic_inc(&key->enabled);
}
-static inline void jump_label_dec(struct jump_label_key *key)
+static inline void __jump_label_dec(struct jump_label_key *key)
{
atomic_dec(&key->enabled);
}
-static inline void jump_label_dec_deferred(struct jump_label_key_deferred *key)
+static inline void __jump_label_dec_deferred(struct jump_label_key_deferred *key)
{
- jump_label_dec(&key->key);
+ __jump_label_dec(&key->key);
}
static inline int jump_label_text_reserved(void *start, void *end)
@@ -198,7 +216,7 @@ static inline int jump_label_text_reserved(void *start, void *end)
static inline void jump_label_lock(void) {}
static inline void jump_label_unlock(void) {}
-static inline bool jump_label_true(struct jump_label_key *key)
+static inline bool __jump_label_true(struct jump_label_key *key)
{
return (atomic_read(&key->enabled) > 0);
}
@@ -209,7 +227,7 @@ static inline int jump_label_apply_nops(struct module *mod)
}
static inline void
-jump_label_rate_limit(struct jump_label_key_deferred *key,
+__jump_label_rate_limit(struct jump_label_key_deferred *key,
unsigned long rl)
{
}
diff --git a/include/linux/netfilter.h b/include/linux/netfilter.h
index 43d0cc0..6255b3e 100644
--- a/include/linux/netfilter.h
+++ b/include/linux/netfilter.h
@@ -169,7 +169,7 @@ static inline bool nf_hooks_active(u_int8_t pf, unsigned int hook)
{
if (__builtin_constant_p(pf) &&
__builtin_constant_p(hook))
- return very_unlikely(&nf_hooks_needed[pf][hook]);
+ return very_unlikely(nf_hooks_needed[pf][hook]);
return !list_empty(&nf_hooks[pf][hook]);
}
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 041d02b..b96df28 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -1066,7 +1066,7 @@ perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
{
struct pt_regs hot_regs;
- if (very_unlikely(&perf_swevent_enabled[event_id])) {
+ if (very_unlikely(perf_swevent_enabled[event_id])) {
if (!regs) {
perf_fetch_caller_regs(&hot_regs);
regs = &hot_regs;
@@ -1080,7 +1080,7 @@ extern struct jump_label_key_deferred perf_sched_events;
static inline void perf_event_task_sched_in(struct task_struct *prev,
struct task_struct *task)
{
- if (very_unlikely(&perf_sched_events.key))
+ if (very_unlikely(perf_sched_events.key))
__perf_event_task_sched_in(prev, task);
}
@@ -1089,7 +1089,7 @@ static inline void perf_event_task_sched_out(struct task_struct *prev,
{
perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, NULL, 0);
- if (very_unlikely(&perf_sched_events.key))
+ if (very_unlikely(perf_sched_events.key))
__perf_event_task_sched_out(prev, next);
}
diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h
index 282cf59..86717cd 100644
--- a/include/linux/tracepoint.h
+++ b/include/linux/tracepoint.h
@@ -143,7 +143,7 @@ static inline void tracepoint_synchronize_unregister(void)
extern struct tracepoint __tracepoint_##name; \
static inline void trace_##name(proto) \
{ \
- if (very_unlikely(&__tracepoint_##name.key)) \
+ if (very_unlikely(__tracepoint_##name.key)) \
__DO_TRACE(&__tracepoint_##name, \
TP_PROTO(data_proto), \
TP_ARGS(data_args), \
diff --git a/include/net/sock.h b/include/net/sock.h
index 1d16574..af4b58a 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -930,7 +930,7 @@ static inline struct cg_proto *parent_cg_proto(struct proto *proto,
{
return proto->proto_cgroup(parent_mem_cgroup(cg_proto->memcg));
}
-#define mem_cgroup_sockets_enabled very_unlikely(&memcg_socket_limit_enabled)
+#define mem_cgroup_sockets_enabled very_unlikely(memcg_socket_limit_enabled)
#else
#define mem_cgroup_sockets_enabled 0
static inline struct cg_proto *parent_cg_proto(struct proto *proto,
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 7c3b9de..31a80f2 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -2769,7 +2769,7 @@ static void free_event(struct perf_event *event)
if (!event->parent) {
if (event->attach_state & PERF_ATTACH_TASK)
- jump_label_dec_deferred(&perf_sched_events);
+ jump_label_dec_deferred(perf_sched_events);
if (event->attr.mmap || event->attr.mmap_data)
atomic_dec(&nr_mmap_events);
if (event->attr.comm)
@@ -2780,7 +2780,7 @@ static void free_event(struct perf_event *event)
put_callchain_buffers();
if (is_cgroup_event(event)) {
atomic_dec(&per_cpu(perf_cgroup_events, event->cpu));
- jump_label_dec_deferred(&perf_sched_events);
+ jump_label_dec_deferred(perf_sched_events);
}
}
@@ -4990,7 +4990,7 @@ static void sw_perf_event_destroy(struct perf_event *event)
WARN_ON(event->parent);
- jump_label_dec(&perf_swevent_enabled[event_id]);
+ jump_label_dec(perf_swevent_enabled[event_id]);
swevent_hlist_put(event);
}
@@ -5020,7 +5020,7 @@ static int perf_swevent_init(struct perf_event *event)
if (err)
return err;
- jump_label_inc(&perf_swevent_enabled[event_id]);
+ jump_label_inc(perf_swevent_enabled[event_id]);
event->destroy = sw_perf_event_destroy;
}
@@ -5843,7 +5843,7 @@ done:
if (!event->parent) {
if (event->attach_state & PERF_ATTACH_TASK)
- jump_label_inc(&perf_sched_events.key);
+ jump_label_inc(perf_sched_events.key);
if (event->attr.mmap || event->attr.mmap_data)
atomic_inc(&nr_mmap_events);
if (event->attr.comm)
@@ -6081,7 +6081,7 @@ SYSCALL_DEFINE5(perf_event_open,
* - that may need work on context switch
*/
atomic_inc(&per_cpu(perf_cgroup_events, event->cpu));
- jump_label_inc(&perf_sched_events.key);
+ jump_label_inc(perf_sched_events.key);
}
/*
@@ -6929,7 +6929,7 @@ void __init perf_event_init(void)
WARN(ret, "hw_breakpoint initialization failed with: %d", ret);
/* do not patch jump label more than once per second */
- jump_label_rate_limit(&perf_sched_events, HZ);
+ jump_label_rate_limit(perf_sched_events, HZ);
}
static int __init perf_event_sysfs_init(void)
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 2b55284..d45d9e0 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -29,11 +29,11 @@ void jump_label_unlock(void)
mutex_unlock(&jump_label_mutex);
}
-bool jump_label_true(struct jump_label_key *key)
+bool __jump_label_true(struct jump_label_key *key)
{
return (atomic_read(&key->enabled) > 0);
}
-EXPORT_SYMBOL_GPL(jump_label_true);
+EXPORT_SYMBOL_GPL(__jump_label_true);
static int jump_label_cmp(const void *a, const void *b)
{
@@ -61,7 +61,7 @@ jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop)
static void jump_label_update(struct jump_label_key *key, int enable);
-void jump_label_inc(struct jump_label_key *key)
+void __jump_label_inc(struct jump_label_key *key)
{
if (atomic_inc_not_zero(&key->enabled))
return;
@@ -76,9 +76,9 @@ void jump_label_inc(struct jump_label_key *key)
atomic_inc(&key->enabled);
jump_label_unlock();
}
-EXPORT_SYMBOL_GPL(jump_label_inc);
+EXPORT_SYMBOL_GPL(__jump_label_inc);
-static void __jump_label_dec(struct jump_label_key *key,
+static void ____jump_label_dec(struct jump_label_key *key,
unsigned long rate_limit, struct delayed_work *work)
{
if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) {
@@ -103,22 +103,22 @@ static void jump_label_update_timeout(struct work_struct *work)
{
struct jump_label_key_deferred *key =
container_of(work, struct jump_label_key_deferred, work.work);
- __jump_label_dec(&key->key, 0, NULL);
+ ____jump_label_dec(&key->key, 0, NULL);
}
-void jump_label_dec(struct jump_label_key *key)
+void __jump_label_dec(struct jump_label_key *key)
{
- __jump_label_dec(key, 0, NULL);
+ ____jump_label_dec(key, 0, NULL);
}
-EXPORT_SYMBOL_GPL(jump_label_dec);
+EXPORT_SYMBOL_GPL(__jump_label_dec);
-void jump_label_dec_deferred(struct jump_label_key_deferred *key)
+void __jump_label_dec_deferred(struct jump_label_key_deferred *key)
{
- __jump_label_dec(&key->key, key->timeout, &key->work);
+ ____jump_label_dec(&key->key, key->timeout, &key->work);
}
-EXPORT_SYMBOL_GPL(jump_label_dec_deferred);
+EXPORT_SYMBOL_GPL(__jump_label_dec_deferred);
-void jump_label_rate_limit(struct jump_label_key_deferred *key,
+void __jump_label_rate_limit(struct jump_label_key_deferred *key,
unsigned long rl)
{
key->timeout = rl;
@@ -181,7 +181,7 @@ static void __jump_label_update(struct jump_label_key *key,
static enum jump_label_type jump_label_type(struct jump_label_key *key)
{
bool true_branch = jump_label_get_branch_default(key);
- bool state = jump_label_true(key);
+ bool state = __jump_label_true(key);
if ((!true_branch && state) || (true_branch && !state))
return JUMP_LABEL_ENABLE;
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index a357dbf..8288984 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -176,14 +176,14 @@ struct jump_label_key sched_feat_keys[__SCHED_FEAT_NR] = {
static void sched_feat_disable(int i)
{
- if (jump_label_true(&sched_feat_keys[i]))
- jump_label_dec(&sched_feat_keys[i]);
+ if (jump_label_true(sched_feat_keys[i]))
+ jump_label_dec(sched_feat_keys[i]);
}
static void sched_feat_enable(int i)
{
- if (!jump_label_true(&sched_feat_keys[i]))
- jump_label_inc(&sched_feat_keys[i]);
+ if (!jump_label_true(sched_feat_keys[i]))
+ jump_label_inc(sched_feat_keys[i]);
}
#else
static void sched_feat_disable(int i) { };
@@ -894,7 +894,7 @@ static void update_rq_clock_task(struct rq *rq, s64 delta)
delta -= irq_delta;
#endif
#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
- if (very_unlikely((¶virt_steal_rq_enabled))) {
+ if (very_unlikely((paravirt_steal_rq_enabled))) {
u64 st;
steal = paravirt_steal_clock(cpu_of(rq));
@@ -2756,7 +2756,7 @@ void account_idle_time(cputime_t cputime)
static __always_inline bool steal_account_process_tick(void)
{
#ifdef CONFIG_PARAVIRT
- if (very_unlikely(¶virt_steal_enabled)) {
+ if (very_unlikely(paravirt_steal_enabled)) {
u64 steal, st = 0;
steal = paravirt_steal_clock(smp_processor_id());
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 67206ae..075c707 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1403,16 +1403,16 @@ static struct jump_label_key __cfs_bandwidth_used;
static inline bool cfs_bandwidth_used(void)
{
- return very_unlikely(&__cfs_bandwidth_used);
+ return very_unlikely(__cfs_bandwidth_used);
}
void account_cfs_bandwidth_used(int enabled, int was_enabled)
{
/* only need to count groups transitioning between enabled/!enabled */
if (enabled && !was_enabled)
- jump_label_inc(&__cfs_bandwidth_used);
+ jump_label_inc(__cfs_bandwidth_used);
else if (!enabled && was_enabled)
- jump_label_dec(&__cfs_bandwidth_used);
+ jump_label_dec(__cfs_bandwidth_used);
}
#else /* HAVE_JUMP_LABEL */
static bool cfs_bandwidth_used(void)
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index addeb9e..cc00d2a 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -632,12 +632,12 @@ enum {
#if defined(CONFIG_SCHED_DEBUG) && defined(HAVE_JUMP_LABEL)
static __always_inline bool static_branch__true(struct jump_label_key *key)
{
- return very_likely(key); /* Not out of line branch. */
+ return __very_likely(key); /* Not out of line branch. */
}
static __always_inline bool static_branch__false(struct jump_label_key *key)
{
- return very_unlikely(key); /* Out of line branch. */
+ return __very_unlikely(key); /* Out of line branch. */
}
#define SCHED_FEAT(name, enabled) \
diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c
index ad32493..f3e40fc 100644
--- a/kernel/tracepoint.c
+++ b/kernel/tracepoint.c
@@ -256,9 +256,9 @@ static void set_tracepoint(struct tracepoint_entry **entry,
{
WARN_ON(strcmp((*entry)->name, elem->name) != 0);
- if (elem->regfunc && !jump_label_true(&elem->key) && active)
+ if (elem->regfunc && !jump_label_true(elem->key) && active)
elem->regfunc();
- else if (elem->unregfunc && jump_label_true(&elem->key) && !active)
+ else if (elem->unregfunc && jump_label_true(elem->key) && !active)
elem->unregfunc();
/*
@@ -269,10 +269,10 @@ static void set_tracepoint(struct tracepoint_entry **entry,
* is used.
*/
rcu_assign_pointer(elem->funcs, (*entry)->funcs);
- if (active && !jump_label_true(&elem->key))
- jump_label_inc(&elem->key);
- else if (!active && jump_label_true(&elem->key))
- jump_label_dec(&elem->key);
+ if (active && !jump_label_true(elem->key))
+ jump_label_inc(elem->key);
+ else if (!active && jump_label_true(elem->key))
+ jump_label_dec(elem->key);
}
/*
@@ -283,11 +283,11 @@ static void set_tracepoint(struct tracepoint_entry **entry,
*/
static void disable_tracepoint(struct tracepoint *elem)
{
- if (elem->unregfunc && jump_label_true(&elem->key))
+ if (elem->unregfunc && jump_label_true(elem->key))
elem->unregfunc();
- if (jump_label_true(&elem->key))
- jump_label_dec(&elem->key);
+ if (jump_label_true(elem->key))
+ jump_label_dec(elem->key);
rcu_assign_pointer(elem->funcs, NULL);
}
diff --git a/net/core/dev.c b/net/core/dev.c
index 6503c14..5f6ad5f 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -1457,12 +1457,12 @@ void net_enable_timestamp(void)
if (deferred) {
while (--deferred)
- jump_label_dec(&netstamp_needed);
+ jump_label_dec(netstamp_needed);
return;
}
#endif
WARN_ON(in_interrupt());
- jump_label_inc(&netstamp_needed);
+ jump_label_inc(netstamp_needed);
}
EXPORT_SYMBOL(net_enable_timestamp);
@@ -1474,19 +1474,19 @@ void net_disable_timestamp(void)
return;
}
#endif
- jump_label_dec(&netstamp_needed);
+ jump_label_dec(netstamp_needed);
}
EXPORT_SYMBOL(net_disable_timestamp);
static inline void net_timestamp_set(struct sk_buff *skb)
{
skb->tstamp.tv64 = 0;
- if (very_unlikely(&netstamp_needed))
+ if (very_unlikely(netstamp_needed))
__net_timestamp(skb);
}
#define net_timestamp_check(COND, SKB) \
- if (very_unlikely(&netstamp_needed)) { \
+ if (very_unlikely(netstamp_needed)) { \
if ((COND) && !(SKB)->tstamp.tv64) \
__net_timestamp(SKB); \
} \
@@ -2945,7 +2945,7 @@ int netif_rx(struct sk_buff *skb)
trace_netif_rx(skb);
#ifdef CONFIG_RPS
- if (very_unlikely(&rps_needed)) {
+ if (very_unlikely(rps_needed)) {
struct rps_dev_flow voidflow, *rflow = &voidflow;
int cpu;
@@ -3309,7 +3309,7 @@ int netif_receive_skb(struct sk_buff *skb)
return NET_RX_SUCCESS;
#ifdef CONFIG_RPS
- if (very_unlikely(&rps_needed)) {
+ if (very_unlikely(rps_needed)) {
struct rps_dev_flow voidflow, *rflow = &voidflow;
int cpu, ret;
diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c
index a1727cd..a70d56e 100644
--- a/net/core/net-sysfs.c
+++ b/net/core/net-sysfs.c
@@ -608,10 +608,10 @@ static ssize_t store_rps_map(struct netdev_rx_queue *queue,
spin_unlock(&rps_map_lock);
if (map)
- jump_label_inc(&rps_needed);
+ jump_label_inc(rps_needed);
if (old_map) {
kfree_rcu(old_map, rcu);
- jump_label_dec(&rps_needed);
+ jump_label_dec(rps_needed);
}
free_cpumask_var(mask);
return len;
diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c
index d05559d..f3e0d88 100644
--- a/net/core/sysctl_net_core.c
+++ b/net/core/sysctl_net_core.c
@@ -69,9 +69,9 @@ static int rps_sock_flow_sysctl(ctl_table *table, int write,
if (sock_table != orig_sock_table) {
rcu_assign_pointer(rps_sock_flow_table, sock_table);
if (sock_table)
- jump_label_inc(&rps_needed);
+ jump_label_inc(rps_needed);
if (orig_sock_table) {
- jump_label_dec(&rps_needed);
+ jump_label_dec(rps_needed);
synchronize_rcu();
vfree(orig_sock_table);
}
diff --git a/net/ipv4/tcp_memcontrol.c b/net/ipv4/tcp_memcontrol.c
index 4997878..6b7f7eb 100644
--- a/net/ipv4/tcp_memcontrol.c
+++ b/net/ipv4/tcp_memcontrol.c
@@ -111,7 +111,7 @@ void tcp_destroy_cgroup(struct cgroup *cgrp, struct cgroup_subsys *ss)
val = res_counter_read_u64(&tcp->tcp_memory_allocated, RES_LIMIT);
if (val != RESOURCE_MAX)
- jump_label_dec(&memcg_socket_limit_enabled);
+ jump_label_dec(memcg_socket_limit_enabled);
}
EXPORT_SYMBOL(tcp_destroy_cgroup);
@@ -143,9 +143,9 @@ static int tcp_update_limit(struct mem_cgroup *memcg, u64 val)
net->ipv4.sysctl_tcp_mem[i]);
if (val == RESOURCE_MAX && old_lim != RESOURCE_MAX)
- jump_label_dec(&memcg_socket_limit_enabled);
+ jump_label_dec(memcg_socket_limit_enabled);
else if (old_lim == RESOURCE_MAX && val != RESOURCE_MAX)
- jump_label_inc(&memcg_socket_limit_enabled);
+ jump_label_inc(memcg_socket_limit_enabled);
return 0;
}
diff --git a/net/netfilter/core.c b/net/netfilter/core.c
index b4e8ff0..f968784 100644
--- a/net/netfilter/core.c
+++ b/net/netfilter/core.c
@@ -77,7 +77,7 @@ int nf_register_hook(struct nf_hook_ops *reg)
list_add_rcu(®->list, elem->list.prev);
mutex_unlock(&nf_hook_mutex);
#if defined(CONFIG_JUMP_LABEL)
- jump_label_inc(&nf_hooks_needed[reg->pf][reg->hooknum]);
+ jump_label_inc(nf_hooks_needed[reg->pf][reg->hooknum]);
#endif
return 0;
}
@@ -89,7 +89,7 @@ void nf_unregister_hook(struct nf_hook_ops *reg)
list_del_rcu(®->list);
mutex_unlock(&nf_hook_mutex);
#if defined(CONFIG_JUMP_LABEL)
- jump_label_dec(&nf_hooks_needed[reg->pf][reg->hooknum]);
+ jump_label_dec(nf_hooks_needed[reg->pf][reg->hooknum]);
#endif
synchronize_net();
}
On 02/23/2012 08:21 AM, Jason Baron wrote:
>
> So, we could get rid of the '&' with something as simple as:
>
> #define very_unlikely(key) __very_unlikely(&key)
>
> However, it does seem potentially more error prone, b/c if 'key' is
> passed to a function, and they we do the very_unlikely() we end up with
> the address parameter (due to pass by value). That said, it doesn't look
> like anybody is using very_unlikely() in that manner in the tree, and we
> could document the usage.
>
The normal way to do this is to make the type an array of size 1.
-hpa
--
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel. I don't speak on their behalf.
On Thu, Feb 23, 2012 at 2:02 AM, Ingo Molnar <[email protected]> wrote:
>
> Having to take the address gives us type safety - i.e. it will
> not be possible to accidentally pass in a non-jump-label key and
> get it misinterpreted.
Ingo, stop with the stupid denialism.
NOBODY likes that name. NOBODY. It's wrong. It's stupid. It sounds
like a stronger "unlikely()", and it simply IS NOT.
So rename it already.
The "type safety" argument seems bogus too. As far as I can tell, it
fails miserably if you test a void pointer for being NULL.
Sure, you can fix that by doing crazy things to the interface, but in
the end, nothing changes the fact that "very_unlikely()" as a name
sounds like an emphatic version of "unlikely()".
Rename it. Make it clear from the name that it is about these static
variables. Everything else seems to be named for that "static_key"
thing, so make the testing of it be named that way too, instead of
using a bad generic naming scheme that is just confusing.
Linus
* Linus Torvalds <[email protected]> wrote:
> On Thu, Feb 23, 2012 at 2:02 AM, Ingo Molnar <[email protected]> wrote:
> >
> > Having to take the address gives us type safety - i.e. it
> > will not be possible to accidentally pass in a
> > non-jump-label key and get it misinterpreted.
>
> [...]
>
> So rename it already.
No problem - it's all in a separate subject-to-rebasing branch,
with the naming patches at the end (to allow the naming
discussion play out) - which I've just dropped.
> Ingo, stop with the stupid denialism.
>
> NOBODY likes that name. NOBODY. It's wrong. It's stupid. It
> sounds like a stronger "unlikely()", and it simply IS NOT.
At the risk of being flamed some more, where does the confusion
stem from? Is the source of the confusion the fact that we
cannot write arbitrary C conditions inside the unlikely()
portion [like we can do with a regular unlikely()] - but are
forced to use a specific syntax, a 'key'?
Otherwise, call me stupid, but when I look at the disassembly
then the defining characteristic of this construct is that it is
a distinctly stronger unlikely() slowpath branch:
Before:
ffffffff810441f0 <sys_getppid>:
ffffffff810441f0: 8b 05 8a 52 d8 00 mov 0xd8528a(%rip),%eax # ffffffff81dc9480 <key>
ffffffff810441f6: 55 push %rbp
ffffffff810441f7: 48 89 e5 mov %rsp,%rbp
ffffffff810441fa: 85 c0 test %eax,%eax
ffffffff810441fc: 75 27 jne ffffffff81044225 <sys_getppid+0x35>
ffffffff810441fe: 65 48 8b 04 25 c0 b6 mov %gs:0xb6c0,%rax
ffffffff81044205: 00 00
ffffffff81044207: 48 8b 80 80 02 00 00 mov 0x280(%rax),%rax
ffffffff8104420e: 48 8b 80 b0 02 00 00 mov 0x2b0(%rax),%rax
ffffffff81044215: 48 8b b8 e8 02 00 00 mov 0x2e8(%rax),%rdi
ffffffff8104421c: e8 2f da 00 00 callq ffffffff81051c50 <pid_vnr>
ffffffff81044221: 5d pop %rbp
ffffffff81044222: 48 98 cltq
ffffffff81044224: c3 retq
ffffffff81044225: 48 c7 c7 13 53 98 81 mov $0xffffffff81985313,%rdi
ffffffff8104422c: 31 c0 xor %eax,%eax
ffffffff8104422e: e8 60 0f 6d 00 callq ffffffff81715193 <printk>
ffffffff81044233: eb c9 jmp ffffffff810441fe <sys_getppid+0xe>
ffffffff81044235: 66 66 2e 0f 1f 84 00 data32 nopw %cs:0x0(%rax,%rax,1)
ffffffff8104423c: 00 00 00 00
After:
ffffffff81044290 <sys_getppid>:
ffffffff81044290: 55 push %rbp
ffffffff81044291: 48 89 e5 mov %rsp,%rbp
ffffffff81044294: e9 00 00 00 00 jmpq ffffffff81044299 <sys_getppid+0x9>
ffffffff81044299: 65 48 8b 04 25 c0 b6 mov %gs:0xb6c0,%rax
ffffffff810442a0: 00 00
ffffffff810442a2: 48 8b 80 80 02 00 00 mov 0x280(%rax),%rax
ffffffff810442a9: 48 8b 80 b0 02 00 00 mov 0x2b0(%rax),%rax
ffffffff810442b0: 48 8b b8 e8 02 00 00 mov 0x2e8(%rax),%rdi
ffffffff810442b7: e8 f4 d9 00 00 callq ffffffff81051cb0 <pid_vnr>
ffffffff810442bc: 5d pop %rbp
ffffffff810442bd: 48 98 cltq
ffffffff810442bf: c3 retq
ffffffff810442c0: 48 c7 c7 e3 54 98 81 mov $0xffffffff819854e3,%rdi
ffffffff810442c7: 31 c0 xor %eax,%eax
ffffffff810442c9: e8 71 13 6d 00 callq ffffffff8171563f <printk>
ffffffff810442ce: eb c9 jmp ffffffff81044299 <sys_getppid+0x9>
The prior slowpath test:
ffffffff810441fa: 85 c0 test %eax,%eax
ffffffff810441fc: 75 27 jne ffffffff81044225 <sys_getppid+0x35>
became even faster and turned into a single NOP, making the
fastpath even faster.
> The "type safety" argument seems bogus too. As far as I can
> tell, it fails miserably if you test a void pointer for being
> NULL.
>
> Sure, you can fix that by doing crazy things to the interface,
> but in the end, nothing changes the fact that
> "very_unlikely()" as a name sounds like an emphatic version of
> "unlikely()".
>
> Rename it. Make it clear from the name that it is about these
> static variables. Everything else seems to be named for that
> "static_key" thing, so make the testing of it be named that
> way too, instead of using a bad generic naming scheme that is
> just confusing.
Well, the static_key naming is new, it came from me in the last
patch (full patch attached below), so I guess you only disagree
with the very_unlikely()/very_likely() confusion bit, but not
with the static_key renaming?
My main gripe was the jump-label naming, which was a deeply
implementational name that was actively confusing (to me).
So, a modified scheme would be:
#include <linux/static_key.h>
struct static_key key = STATIC_KEY_INIT_TRUE;
if (static_key_false(&key))
do unlikely code
else
do likely code
Or:
if (static_key_true(&key))
do likely code
else
do unlikely code
The static key is modified via:
static_key_slow_inc(&key);
...
static_key_slow_dec(&key);
Is that API fine? I'll rework the series to such an effect if
everyone agrees.
Thanks,
Ingo
>From 77a73e5383a9637dd53e39997bc2bbb8128d7fbb Mon Sep 17 00:00:00 2001
From: Ingo Molnar <[email protected]>
Date: Wed, 22 Feb 2012 09:58:09 +0100
Subject: [PATCH] static keys: Introduce 'struct static_key',
very_[un]likely(), static_key_slow_[inc|dec]()
So here's a boot tested patch on top of Jason's series that does
all the cleanups I talked about and turns jump labels into a
more intuitive to use facility. It should also address the
various misconceptions and confusions that surround jump labels.
Typical usage scenarios:
#include <linux/static_key.h>
struct static_key key = STATIC_KEY_INIT_TRUE;
if (very_unlikely(&key))
do unlikely code
else
do likely code
Or:
if (very_likely(&key))
do likely code
else
do unlikely code
The static key is modified via:
static_key_slow_inc(&key);
...
static_key_slow_dec(&key);
The 'slow' prefix makes it abundantly clear that this is an
expensive operation.
I've updated all in-kernel code to use this everywhere. Note
that I (intentionally) have not pushed through the rename
blindly through to the lowest levels: the actual jump-label
patching arch facility should be named like that, so we want to
decouple jump labels from the static-key facility a bit.
On non-jump-label enabled architectures static keys default to
likely()/unlikely() branches.
Signed-off-by: Ingo Molnar <[email protected]>
Cc: Steven Rostedt <[email protected]>
Cc: Jason Baron <[email protected]>
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: Linus Torvalds <[email protected]>
Link: http://lkml.kernel.org/r/[email protected]
Signed-off-by: Ingo Molnar <[email protected]>
---
Documentation/jump-label.txt | 38 ++++++++--------
arch/ia64/include/asm/paravirt.h | 6 +-
arch/ia64/kernel/paravirt.c | 4 +-
arch/mips/include/asm/jump_label.h | 2 +-
arch/powerpc/include/asm/jump_label.h | 2 +-
arch/s390/include/asm/jump_label.h | 2 +-
arch/sparc/include/asm/jump_label.h | 2 +-
arch/x86/include/asm/jump_label.h | 6 +-
arch/x86/include/asm/paravirt.h | 6 +-
arch/x86/kernel/kvm.c | 4 +-
arch/x86/kernel/paravirt.c | 4 +-
arch/x86/kvm/mmu_audit.c | 6 +-
include/linux/jump_label.h | 76 ++++++++++++++++----------------
include/linux/netdevice.h | 4 +-
include/linux/netfilter.h | 4 +-
include/linux/perf_event.h | 6 +-
include/linux/static_key.h | 1 +
include/linux/tracepoint.h | 6 +-
include/net/sock.h | 4 +-
kernel/events/core.c | 16 +++---
kernel/jump_label.c | 72 +++++++++++++++---------------
kernel/sched/core.c | 14 +++---
kernel/sched/fair.c | 6 +-
kernel/sched/sched.h | 10 ++--
kernel/tracepoint.c | 20 ++++----
net/core/dev.c | 16 +++---
net/core/net-sysfs.c | 4 +-
net/core/sock.c | 4 +-
net/core/sysctl_net_core.c | 4 +-
net/ipv4/tcp_memcontrol.c | 6 +-
net/netfilter/core.c | 6 +-
31 files changed, 181 insertions(+), 180 deletions(-)
diff --git a/Documentation/jump-label.txt b/Documentation/jump-label.txt
index 210b760..ba67ca7 100644
--- a/Documentation/jump-label.txt
+++ b/Documentation/jump-label.txt
@@ -32,7 +32,7 @@ the branch site to change the branch direction.
For example, if we have a simple branch that is disabled by default:
- if (very_unlikely(&jump_key))
+ if (very_unlikely(&key))
printk("I am the true branch\n");
Thus, by default the 'printk' will not be emitted. And the code generated will
@@ -48,58 +48,58 @@ basically 'free'. That is the basic tradeoff of this optimization.
In order to use a jump label you much first define a key:
- struct jump_label_key jump_key;
+ struct static_key key;
Which is initialized as:
- struct jump_label_key jump_key = JUMP_LABEL_INIT_TRUE;
+ struct static_key key = STATIC_KEY_INIT_TRUE;
or:
- struct jump_label_Key jump_key = JUMP_LABEL_INIT_FALSE;
+ struct static_key key = STATIC_KEY_INIT_FALSE;
If the key is not initialized, it is default false. The
-'struct jump_label_key', must be a 'global'. That is, it can't be allocated on
+'struct static_key', must be a 'global'. That is, it can't be allocated on
the stack or dynamically allocated at run-time.
The key is then used in code as:
- if (very_unlikely(&jump_key))
+ if (very_unlikely(&key))
do unlikely code
else
do likely code
Or:
- if (very_likely(&jump_key))
+ if (very_likely(&key))
do likely code
else
do unlikely code
-A key that is initialized via 'JUMP_LABEL_INIT_FALSE', must be used in a
+A key that is initialized via 'STATIC_KEY_INIT_FALSE', must be used in a
'very_unlikely()' construct. Likewise, a key initialized via
-'JUMP_LABEL_INIT_TRUE' must be used in a 'very_likely()' construct.
+'STATIC_KEY_INIT_TRUE' must be used in a 'very_likely()' construct.
A single key can be used in many branches, but all the branches must match
the way that the key has been initialized.
The branch(es) can then be switched via:
- jump_label_inc(&jump_key);
- jump_label_dec(&jump_key);
+ static_key_slow_inc(&key);
+ static_key_slow_dec(&key);
-Thus, 'jump_label_inc()' means 'make the branch true', and
-'jump_label_dec()' means 'make the the branch false' with appropriate reference
-counting. For example, if the key is initialized true, a jump_label_dec(), will
-switch the branch to false. And a subsequent jump_label_inc(), will change
+Thus, 'static_key_slow_inc()' means 'make the branch true', and
+'static_key_slow_dec()' means 'make the the branch false' with appropriate reference
+counting. For example, if the key is initialized true, a static_key_slow_dec(), will
+switch the branch to false. And a subsequent static_key_slow_inc(), will change
the branch back to true. Likewise, if the key is initialized false, a
-'jump_label_inc()', will change the branch to true. And then a
-'jump_label_dec()', will again make the branch false.
+'static_key_slow_inc()', will change the branch to true. And then a
+'static_key_slow_dec()', will again make the branch false.
An example usage in the kernel is the implementation of tracepoints:
static inline void trace_##name(proto) \
{ \
- if (very_unlikely(&__tracepoint_##name.key)) \
+ if (very_unlikely(&__tracepoint_##name.key)) \
__DO_TRACE(&__tracepoint_##name, \
TP_PROTO(data_proto), \
TP_ARGS(data_args), \
@@ -122,7 +122,7 @@ simply fall back to a traditional, load, test, and jump sequence.
* #define JUMP_LABEL_NOP_SIZE, see: arch/x86/include/asm/jump_label.h
-* __always_inline bool arch_static_branch(struct jump_label_key *key), see:
+* __always_inline bool arch_static_branch(struct static_key *key), see:
arch/x86/include/asm/jump_label.h
* void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type),
diff --git a/arch/ia64/include/asm/paravirt.h b/arch/ia64/include/asm/paravirt.h
index 32551d3..b149b88 100644
--- a/arch/ia64/include/asm/paravirt.h
+++ b/arch/ia64/include/asm/paravirt.h
@@ -281,9 +281,9 @@ paravirt_init_missing_ticks_accounting(int cpu)
pv_time_ops.init_missing_ticks_accounting(cpu);
}
-struct jump_label_key;
-extern struct jump_label_key paravirt_steal_enabled;
-extern struct jump_label_key paravirt_steal_rq_enabled;
+struct static_key;
+extern struct static_key paravirt_steal_enabled;
+extern struct static_key paravirt_steal_rq_enabled;
static inline int
paravirt_do_steal_accounting(unsigned long *new_itm)
diff --git a/arch/ia64/kernel/paravirt.c b/arch/ia64/kernel/paravirt.c
index 1008682..1b22f6d 100644
--- a/arch/ia64/kernel/paravirt.c
+++ b/arch/ia64/kernel/paravirt.c
@@ -634,8 +634,8 @@ struct pv_irq_ops pv_irq_ops = {
* pv_time_ops
* time operations
*/
-struct jump_label_key paravirt_steal_enabled;
-struct jump_label_key paravirt_steal_rq_enabled;
+struct static_key paravirt_steal_enabled;
+struct static_key paravirt_steal_rq_enabled;
static int
ia64_native_do_steal_accounting(unsigned long *new_itm)
diff --git a/arch/mips/include/asm/jump_label.h b/arch/mips/include/asm/jump_label.h
index 1881b31..4d6d77e 100644
--- a/arch/mips/include/asm/jump_label.h
+++ b/arch/mips/include/asm/jump_label.h
@@ -20,7 +20,7 @@
#define WORD_INSN ".word"
#endif
-static __always_inline bool arch_static_branch(struct jump_label_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key)
{
asm goto("1:\tnop\n\t"
"nop\n\t"
diff --git a/arch/powerpc/include/asm/jump_label.h b/arch/powerpc/include/asm/jump_label.h
index 938986e..ae098c4 100644
--- a/arch/powerpc/include/asm/jump_label.h
+++ b/arch/powerpc/include/asm/jump_label.h
@@ -17,7 +17,7 @@
#define JUMP_ENTRY_TYPE stringify_in_c(FTR_ENTRY_LONG)
#define JUMP_LABEL_NOP_SIZE 4
-static __always_inline bool arch_static_branch(struct jump_label_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key)
{
asm goto("1:\n\t"
"nop\n\t"
diff --git a/arch/s390/include/asm/jump_label.h b/arch/s390/include/asm/jump_label.h
index 95a6cf2..6c32190 100644
--- a/arch/s390/include/asm/jump_label.h
+++ b/arch/s390/include/asm/jump_label.h
@@ -13,7 +13,7 @@
#define ASM_ALIGN ".balign 4"
#endif
-static __always_inline bool arch_static_branch(struct jump_label_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key)
{
asm goto("0: brcl 0,0\n"
".pushsection __jump_table, \"aw\"\n"
diff --git a/arch/sparc/include/asm/jump_label.h b/arch/sparc/include/asm/jump_label.h
index fc73a82..5080d16 100644
--- a/arch/sparc/include/asm/jump_label.h
+++ b/arch/sparc/include/asm/jump_label.h
@@ -7,7 +7,7 @@
#define JUMP_LABEL_NOP_SIZE 4
-static __always_inline bool arch_static_branch(struct jump_label_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key)
{
asm goto("1:\n\t"
"nop\n\t"
diff --git a/arch/x86/include/asm/jump_label.h b/arch/x86/include/asm/jump_label.h
index a32b18c..3a16c14 100644
--- a/arch/x86/include/asm/jump_label.h
+++ b/arch/x86/include/asm/jump_label.h
@@ -9,12 +9,12 @@
#define JUMP_LABEL_NOP_SIZE 5
-#define JUMP_LABEL_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t"
+#define STATIC_KEY_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t"
-static __always_inline bool arch_static_branch(struct jump_label_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key)
{
asm goto("1:"
- JUMP_LABEL_INITIAL_NOP
+ STATIC_KEY_INITIAL_NOP
".pushsection __jump_table, \"aw\" \n\t"
_ASM_ALIGN "\n\t"
_ASM_PTR "1b, %l[l_yes], %c0 \n\t"
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index a7d2db9..c0180fd 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -230,9 +230,9 @@ static inline unsigned long long paravirt_sched_clock(void)
return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock);
}
-struct jump_label_key;
-extern struct jump_label_key paravirt_steal_enabled;
-extern struct jump_label_key paravirt_steal_rq_enabled;
+struct static_key;
+extern struct static_key paravirt_steal_enabled;
+extern struct static_key paravirt_steal_rq_enabled;
static inline u64 paravirt_steal_clock(int cpu)
{
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index f0c6fd6..694d801 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -438,9 +438,9 @@ void __init kvm_guest_init(void)
static __init int activate_jump_labels(void)
{
if (has_steal_clock) {
- jump_label_inc(¶virt_steal_enabled);
+ static_key_slow_inc(¶virt_steal_enabled);
if (steal_acc)
- jump_label_inc(¶virt_steal_rq_enabled);
+ static_key_slow_inc(¶virt_steal_rq_enabled);
}
return 0;
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index d90272e..ada2f99 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -202,8 +202,8 @@ static void native_flush_tlb_single(unsigned long addr)
__native_flush_tlb_single(addr);
}
-struct jump_label_key paravirt_steal_enabled;
-struct jump_label_key paravirt_steal_rq_enabled;
+struct static_key paravirt_steal_enabled;
+struct static_key paravirt_steal_rq_enabled;
static u64 native_steal_clock(int cpu)
{
diff --git a/arch/x86/kvm/mmu_audit.c b/arch/x86/kvm/mmu_audit.c
index 7709e02..b21c123 100644
--- a/arch/x86/kvm/mmu_audit.c
+++ b/arch/x86/kvm/mmu_audit.c
@@ -234,7 +234,7 @@ static void audit_vcpu_spte(struct kvm_vcpu *vcpu)
}
static bool mmu_audit;
-static struct jump_label_key mmu_audit_key;
+static struct static_key mmu_audit_key;
static void __kvm_mmu_audit(struct kvm_vcpu *vcpu, int point)
{
@@ -259,7 +259,7 @@ static void mmu_audit_enable(void)
if (mmu_audit)
return;
- jump_label_inc(&mmu_audit_key);
+ static_key_slow_inc(&mmu_audit_key);
mmu_audit = true;
}
@@ -268,7 +268,7 @@ static void mmu_audit_disable(void)
if (!mmu_audit)
return;
- jump_label_dec(&mmu_audit_key);
+ static_key_slow_dec(&mmu_audit_key);
mmu_audit = false;
}
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index 757d8dc..25c589b 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -13,11 +13,11 @@
* defaults to false - and the true block is placed out of line).
*
* However at runtime we can change the branch target using
- * jump_label_{inc,dec}(). These function as a 'reference' count on the key
+ * static_key_slow_{inc,dec}(). These function as a 'reference' count on the key
* object and for as long as there are references all branches referring to
* that particular key will point to the (out of line) true block.
*
- * Since this relies on modifying code the jump_label_{inc,dec}() functions
+ * Since this relies on modifying code the static_key_slow_{inc,dec}() functions
* must be considered absolute slow paths (machine wide synchronization etc.).
* OTOH, since the affected branches are unconditional their runtime overhead
* will be absolutely minimal, esp. in the default (off) case where the total
@@ -26,13 +26,13 @@
*
* When the control is directly exposed to userspace it is prudent to delay the
* decrement to avoid high frequency code modifications which can (and do)
- * cause significant performance degradation. Struct jump_label_key_deferred and
- * jump_label_dec_deferred() provide for this.
+ * cause significant performance degradation. Struct static_key_deferred and
+ * static_key_slow_dec_deferred() provide for this.
*
* Lacking toolchain and or architecture support, it falls back to a simple
* conditional branch.
*
- * struct jump_label_key my_key = JUMP_LABEL_INIT_TRUE;
+ * struct static_key my_key = STATIC_KEY_INIT_TRUE;
*
* if (very_likely(&my_key)) {
* }
@@ -42,7 +42,7 @@
* allowed.
*
* Not initializing the key (static data is initialized to 0s anyway) is the
- * same as using JUMP_LABEL_INIT_FALSE and very_unlikely() is
+ * same as using STATIC_KEY_INIT_FALSE and very_unlikely() is
* equivalent with static_branch().
*
*/
@@ -53,17 +53,17 @@
#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
-struct jump_label_key {
+struct static_key {
atomic_t enabled;
/* Set lsb bit to 1 if branch is default true, 0 ot */
struct jump_entry *entries;
#ifdef CONFIG_MODULES
- struct jump_label_mod *next;
+ struct static_key_mod *next;
#endif
};
-struct jump_label_key_deferred {
- struct jump_label_key key;
+struct static_key_deferred {
+ struct static_key key;
unsigned long timeout;
struct delayed_work work;
};
@@ -84,31 +84,31 @@ struct module;
#define JUMP_LABEL_TRUE_BRANCH 1UL
static
-inline struct jump_entry *jump_label_get_entries(struct jump_label_key *key)
+inline struct jump_entry *jump_label_get_entries(struct static_key *key)
{
return (struct jump_entry *)((unsigned long)key->entries
& ~JUMP_LABEL_TRUE_BRANCH);
}
-static inline bool jump_label_get_branch_default(struct jump_label_key *key)
+static inline bool jump_label_get_branch_default(struct static_key *key)
{
if ((unsigned long)key->entries & JUMP_LABEL_TRUE_BRANCH)
return true;
return false;
}
-static __always_inline bool very_unlikely(struct jump_label_key *key)
+static __always_inline bool very_unlikely(struct static_key *key)
{
return arch_static_branch(key);
}
-static __always_inline bool very_likely(struct jump_label_key *key)
+static __always_inline bool very_likely(struct static_key *key)
{
return !very_unlikely(key);
}
/* Deprecated. Please use 'very_unlikely() instead. */
-static __always_inline bool static_branch(struct jump_label_key *key)
+static __always_inline bool static_branch(struct static_key *key)
{
return arch_static_branch(key);
}
@@ -124,24 +124,24 @@ extern void arch_jump_label_transform(struct jump_entry *entry,
extern void arch_jump_label_transform_static(struct jump_entry *entry,
enum jump_label_type type);
extern int jump_label_text_reserved(void *start, void *end);
-extern void jump_label_inc(struct jump_label_key *key);
-extern void jump_label_dec(struct jump_label_key *key);
-extern void jump_label_dec_deferred(struct jump_label_key_deferred *key);
-extern bool jump_label_true(struct jump_label_key *key);
+extern void static_key_slow_inc(struct static_key *key);
+extern void static_key_slow_dec(struct static_key *key);
+extern void static_key_slow_dec_deferred(struct static_key_deferred *key);
+extern bool static_key_true(struct static_key *key);
extern void jump_label_apply_nops(struct module *mod);
extern void
-jump_label_rate_limit(struct jump_label_key_deferred *key, unsigned long rl);
+jump_label_rate_limit(struct static_key_deferred *key, unsigned long rl);
-#define JUMP_LABEL_INIT_TRUE ((struct jump_label_key) \
+#define STATIC_KEY_INIT_TRUE ((struct static_key) \
{ .enabled = ATOMIC_INIT(1), .entries = (void *)1 })
-#define JUMP_LABEL_INIT_FALSE ((struct jump_label_key) \
+#define STATIC_KEY_INIT_FALSE ((struct static_key) \
{ .enabled = ATOMIC_INIT(0), .entries = (void *)0 })
#else /* !HAVE_JUMP_LABEL */
#include <linux/atomic.h>
-struct jump_label_key {
+struct static_key {
atomic_t enabled;
};
@@ -149,18 +149,18 @@ static __always_inline void jump_label_init(void)
{
}
-struct jump_label_key_deferred {
- struct jump_label_key key;
+struct static_key_deferred {
+ struct static_key key;
};
-static __always_inline bool very_unlikely(struct jump_label_key *key)
+static __always_inline bool very_unlikely(struct static_key *key)
{
if (unlikely(atomic_read(&key->enabled)) > 0)
return true;
return false;
}
-static __always_inline bool very_likely(struct jump_label_key *key)
+static __always_inline bool very_likely(struct static_key *key)
{
if (likely(atomic_read(&key->enabled)) > 0)
return true;
@@ -168,26 +168,26 @@ static __always_inline bool very_likely(struct jump_label_key *key)
}
/* Deprecated. Please use 'very_unlikely() instead. */
-static __always_inline bool static_branch(struct jump_label_key *key)
+static __always_inline bool static_branch(struct static_key *key)
{
if (unlikely(atomic_read(&key->enabled)) > 0)
return true;
return false;
}
-static inline void jump_label_inc(struct jump_label_key *key)
+static inline void static_key_slow_inc(struct static_key *key)
{
atomic_inc(&key->enabled);
}
-static inline void jump_label_dec(struct jump_label_key *key)
+static inline void static_key_slow_dec(struct static_key *key)
{
atomic_dec(&key->enabled);
}
-static inline void jump_label_dec_deferred(struct jump_label_key_deferred *key)
+static inline void static_key_slow_dec_deferred(struct static_key_deferred *key)
{
- jump_label_dec(&key->key);
+ static_key_slow_dec(&key->key);
}
static inline int jump_label_text_reserved(void *start, void *end)
@@ -198,7 +198,7 @@ static inline int jump_label_text_reserved(void *start, void *end)
static inline void jump_label_lock(void) {}
static inline void jump_label_unlock(void) {}
-static inline bool jump_label_true(struct jump_label_key *key)
+static inline bool static_key_true(struct static_key *key)
{
return (atomic_read(&key->enabled) > 0);
}
@@ -209,19 +209,19 @@ static inline int jump_label_apply_nops(struct module *mod)
}
static inline void
-jump_label_rate_limit(struct jump_label_key_deferred *key,
+jump_label_rate_limit(struct static_key_deferred *key,
unsigned long rl)
{
}
-#define JUMP_LABEL_INIT_TRUE ((struct jump_label_key) \
+#define STATIC_KEY_INIT_TRUE ((struct static_key) \
{ .enabled = ATOMIC_INIT(1) })
-#define JUMP_LABEL_INIT_FALSE ((struct jump_label_key) \
+#define STATIC_KEY_INIT_FALSE ((struct static_key) \
{ .enabled = ATOMIC_INIT(0) })
#endif /* HAVE_JUMP_LABEL */
-#define JUMP_LABEL_INIT JUMP_LABEL_INIT_FALSE
-#define jump_label_enabled jump_label_true
+#define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
+#define jump_label_enabled static_key_true
#endif /* _LINUX_JUMP_LABEL_H */
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 0eac07c..7dfaae7 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -214,8 +214,8 @@ enum {
#include <linux/skbuff.h>
#ifdef CONFIG_RPS
-#include <linux/jump_label.h>
-extern struct jump_label_key rps_needed;
+#include <linux/static_key.h>
+extern struct static_key rps_needed;
#endif
struct neighbour;
diff --git a/include/linux/netfilter.h b/include/linux/netfilter.h
index 43d0cc0..9f10318 100644
--- a/include/linux/netfilter.h
+++ b/include/linux/netfilter.h
@@ -163,8 +163,8 @@ extern struct ctl_path nf_net_ipv4_netfilter_sysctl_path[];
extern struct list_head nf_hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
#if defined(CONFIG_JUMP_LABEL)
-#include <linux/jump_label.h>
-extern struct jump_label_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
+#include <linux/static_key.h>
+extern struct static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
static inline bool nf_hooks_active(u_int8_t pf, unsigned int hook)
{
if (__builtin_constant_p(pf) &&
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 041d02b..434b51b 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -514,7 +514,7 @@ struct perf_guest_info_callbacks {
#include <linux/ftrace.h>
#include <linux/cpu.h>
#include <linux/irq_work.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
#include <linux/atomic.h>
#include <asm/local.h>
@@ -1038,7 +1038,7 @@ static inline int is_software_event(struct perf_event *event)
return event->pmu->task_ctx_nr == perf_sw_context;
}
-extern struct jump_label_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
+extern struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
extern void __perf_sw_event(u32, u64, struct pt_regs *, u64);
@@ -1075,7 +1075,7 @@ perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
}
}
-extern struct jump_label_key_deferred perf_sched_events;
+extern struct static_key_deferred perf_sched_events;
static inline void perf_event_task_sched_in(struct task_struct *prev,
struct task_struct *task)
diff --git a/include/linux/static_key.h b/include/linux/static_key.h
new file mode 100644
index 0000000..27bd3f8
--- /dev/null
+++ b/include/linux/static_key.h
@@ -0,0 +1 @@
+#include <linux/jump_label.h>
diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h
index 44ac001..ec28a94 100644
--- a/include/linux/tracepoint.h
+++ b/include/linux/tracepoint.h
@@ -17,7 +17,7 @@
#include <linux/errno.h>
#include <linux/types.h>
#include <linux/rcupdate.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
struct module;
struct tracepoint;
@@ -29,7 +29,7 @@ struct tracepoint_func {
struct tracepoint {
const char *name; /* Tracepoint name */
- struct jump_label_key key;
+ struct static_key key;
void (*regfunc)(void);
void (*unregfunc)(void);
struct tracepoint_func __rcu *funcs;
@@ -188,7 +188,7 @@ static inline void tracepoint_synchronize_unregister(void)
__attribute__((section("__tracepoints_strings"))) = #name; \
struct tracepoint __tracepoint_##name \
__attribute__((section("__tracepoints"))) = \
- { __tpstrtab_##name, JUMP_LABEL_INIT_FALSE, reg, unreg, NULL };\
+ { __tpstrtab_##name, STATIC_KEY_INIT_FALSE, reg, unreg, NULL };\
static struct tracepoint * const __tracepoint_ptr_##name __used \
__attribute__((section("__tracepoints_ptrs"))) = \
&__tracepoint_##name;
diff --git a/include/net/sock.h b/include/net/sock.h
index 1d16574..907bbe0 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -55,7 +55,7 @@
#include <linux/uaccess.h>
#include <linux/memcontrol.h>
#include <linux/res_counter.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
#include <linux/filter.h>
#include <linux/rculist_nulls.h>
@@ -924,7 +924,7 @@ inline void sk_refcnt_debug_release(const struct sock *sk)
#endif /* SOCK_REFCNT_DEBUG */
#if defined(CONFIG_CGROUP_MEM_RES_CTLR_KMEM) && defined(CONFIG_NET)
-extern struct jump_label_key memcg_socket_limit_enabled;
+extern struct static_key memcg_socket_limit_enabled;
static inline struct cg_proto *parent_cg_proto(struct proto *proto,
struct cg_proto *cg_proto)
{
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 7c3b9de..5e0f8bb 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -128,7 +128,7 @@ enum event_type_t {
* perf_sched_events : >0 events exist
* perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
*/
-struct jump_label_key_deferred perf_sched_events __read_mostly;
+struct static_key_deferred perf_sched_events __read_mostly;
static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
static atomic_t nr_mmap_events __read_mostly;
@@ -2769,7 +2769,7 @@ static void free_event(struct perf_event *event)
if (!event->parent) {
if (event->attach_state & PERF_ATTACH_TASK)
- jump_label_dec_deferred(&perf_sched_events);
+ static_key_slow_dec_deferred(&perf_sched_events);
if (event->attr.mmap || event->attr.mmap_data)
atomic_dec(&nr_mmap_events);
if (event->attr.comm)
@@ -2780,7 +2780,7 @@ static void free_event(struct perf_event *event)
put_callchain_buffers();
if (is_cgroup_event(event)) {
atomic_dec(&per_cpu(perf_cgroup_events, event->cpu));
- jump_label_dec_deferred(&perf_sched_events);
+ static_key_slow_dec_deferred(&perf_sched_events);
}
}
@@ -4982,7 +4982,7 @@ fail:
return err;
}
-struct jump_label_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
+struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
static void sw_perf_event_destroy(struct perf_event *event)
{
@@ -4990,7 +4990,7 @@ static void sw_perf_event_destroy(struct perf_event *event)
WARN_ON(event->parent);
- jump_label_dec(&perf_swevent_enabled[event_id]);
+ static_key_slow_dec(&perf_swevent_enabled[event_id]);
swevent_hlist_put(event);
}
@@ -5020,7 +5020,7 @@ static int perf_swevent_init(struct perf_event *event)
if (err)
return err;
- jump_label_inc(&perf_swevent_enabled[event_id]);
+ static_key_slow_inc(&perf_swevent_enabled[event_id]);
event->destroy = sw_perf_event_destroy;
}
@@ -5843,7 +5843,7 @@ done:
if (!event->parent) {
if (event->attach_state & PERF_ATTACH_TASK)
- jump_label_inc(&perf_sched_events.key);
+ static_key_slow_inc(&perf_sched_events.key);
if (event->attr.mmap || event->attr.mmap_data)
atomic_inc(&nr_mmap_events);
if (event->attr.comm)
@@ -6081,7 +6081,7 @@ SYSCALL_DEFINE5(perf_event_open,
* - that may need work on context switch
*/
atomic_inc(&per_cpu(perf_cgroup_events, event->cpu));
- jump_label_inc(&perf_sched_events.key);
+ static_key_slow_inc(&perf_sched_events.key);
}
/*
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 2b55284..1cc32e3 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -12,7 +12,7 @@
#include <linux/slab.h>
#include <linux/sort.h>
#include <linux/err.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
#ifdef HAVE_JUMP_LABEL
@@ -29,11 +29,11 @@ void jump_label_unlock(void)
mutex_unlock(&jump_label_mutex);
}
-bool jump_label_true(struct jump_label_key *key)
+bool static_key_true(struct static_key *key)
{
return (atomic_read(&key->enabled) > 0);
}
-EXPORT_SYMBOL_GPL(jump_label_true);
+EXPORT_SYMBOL_GPL(static_key_true);
static int jump_label_cmp(const void *a, const void *b)
{
@@ -59,9 +59,9 @@ jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop)
sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL);
}
-static void jump_label_update(struct jump_label_key *key, int enable);
+static void jump_label_update(struct static_key *key, int enable);
-void jump_label_inc(struct jump_label_key *key)
+void static_key_slow_inc(struct static_key *key)
{
if (atomic_inc_not_zero(&key->enabled))
return;
@@ -76,9 +76,9 @@ void jump_label_inc(struct jump_label_key *key)
atomic_inc(&key->enabled);
jump_label_unlock();
}
-EXPORT_SYMBOL_GPL(jump_label_inc);
+EXPORT_SYMBOL_GPL(static_key_slow_inc);
-static void __jump_label_dec(struct jump_label_key *key,
+static void __static_key_slow_dec(struct static_key *key,
unsigned long rate_limit, struct delayed_work *work)
{
if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) {
@@ -101,24 +101,24 @@ static void __jump_label_dec(struct jump_label_key *key,
static void jump_label_update_timeout(struct work_struct *work)
{
- struct jump_label_key_deferred *key =
- container_of(work, struct jump_label_key_deferred, work.work);
- __jump_label_dec(&key->key, 0, NULL);
+ struct static_key_deferred *key =
+ container_of(work, struct static_key_deferred, work.work);
+ __static_key_slow_dec(&key->key, 0, NULL);
}
-void jump_label_dec(struct jump_label_key *key)
+void static_key_slow_dec(struct static_key *key)
{
- __jump_label_dec(key, 0, NULL);
+ __static_key_slow_dec(key, 0, NULL);
}
-EXPORT_SYMBOL_GPL(jump_label_dec);
+EXPORT_SYMBOL_GPL(static_key_slow_dec);
-void jump_label_dec_deferred(struct jump_label_key_deferred *key)
+void static_key_slow_dec_deferred(struct static_key_deferred *key)
{
- __jump_label_dec(&key->key, key->timeout, &key->work);
+ __static_key_slow_dec(&key->key, key->timeout, &key->work);
}
-EXPORT_SYMBOL_GPL(jump_label_dec_deferred);
+EXPORT_SYMBOL_GPL(static_key_slow_dec_deferred);
-void jump_label_rate_limit(struct jump_label_key_deferred *key,
+void jump_label_rate_limit(struct static_key_deferred *key,
unsigned long rl)
{
key->timeout = rl;
@@ -161,7 +161,7 @@ void __weak __init_or_module arch_jump_label_transform_static(struct jump_entry
arch_jump_label_transform(entry, type);
}
-static void __jump_label_update(struct jump_label_key *key,
+static void __jump_label_update(struct static_key *key,
struct jump_entry *entry,
struct jump_entry *stop, int enable)
{
@@ -178,10 +178,10 @@ static void __jump_label_update(struct jump_label_key *key,
}
}
-static enum jump_label_type jump_label_type(struct jump_label_key *key)
+static enum jump_label_type jump_label_type(struct static_key *key)
{
bool true_branch = jump_label_get_branch_default(key);
- bool state = jump_label_true(key);
+ bool state = static_key_true(key);
if ((!true_branch && state) || (true_branch && !state))
return JUMP_LABEL_ENABLE;
@@ -193,16 +193,16 @@ void __init jump_label_init(void)
{
struct jump_entry *iter_start = __start___jump_table;
struct jump_entry *iter_stop = __stop___jump_table;
- struct jump_label_key *key = NULL;
+ struct static_key *key = NULL;
struct jump_entry *iter;
jump_label_lock();
jump_label_sort_entries(iter_start, iter_stop);
for (iter = iter_start; iter < iter_stop; iter++) {
- struct jump_label_key *iterk;
+ struct static_key *iterk;
- iterk = (struct jump_label_key *)(unsigned long)iter->key;
+ iterk = (struct static_key *)(unsigned long)iter->key;
arch_jump_label_transform_static(iter, jump_label_type(iterk));
if (iterk == key)
continue;
@@ -221,8 +221,8 @@ void __init jump_label_init(void)
#ifdef CONFIG_MODULES
-struct jump_label_mod {
- struct jump_label_mod *next;
+struct static_key_mod {
+ struct static_key_mod *next;
struct jump_entry *entries;
struct module *mod;
};
@@ -242,9 +242,9 @@ static int __jump_label_mod_text_reserved(void *start, void *end)
start, end);
}
-static void __jump_label_mod_update(struct jump_label_key *key, int enable)
+static void __jump_label_mod_update(struct static_key *key, int enable)
{
- struct jump_label_mod *mod = key->next;
+ struct static_key_mod *mod = key->next;
while (mod) {
struct module *m = mod->mod;
@@ -284,8 +284,8 @@ static int jump_label_add_module(struct module *mod)
struct jump_entry *iter_start = mod->jump_entries;
struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
struct jump_entry *iter;
- struct jump_label_key *key = NULL;
- struct jump_label_mod *jlm;
+ struct static_key *key = NULL;
+ struct static_key_mod *jlm;
/* if the module doesn't have jump label entries, just return */
if (iter_start == iter_stop)
@@ -294,9 +294,9 @@ static int jump_label_add_module(struct module *mod)
jump_label_sort_entries(iter_start, iter_stop);
for (iter = iter_start; iter < iter_stop; iter++) {
- struct jump_label_key *iterk;
+ struct static_key *iterk;
- iterk = (struct jump_label_key *)(unsigned long)iter->key;
+ iterk = (struct static_key *)(unsigned long)iter->key;
if (iterk == key)
continue;
@@ -309,7 +309,7 @@ static int jump_label_add_module(struct module *mod)
key->next = NULL;
continue;
}
- jlm = kzalloc(sizeof(struct jump_label_mod), GFP_KERNEL);
+ jlm = kzalloc(sizeof(struct static_key_mod), GFP_KERNEL);
if (!jlm)
return -ENOMEM;
jlm->mod = mod;
@@ -329,14 +329,14 @@ static void jump_label_del_module(struct module *mod)
struct jump_entry *iter_start = mod->jump_entries;
struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
struct jump_entry *iter;
- struct jump_label_key *key = NULL;
- struct jump_label_mod *jlm, **prev;
+ struct static_key *key = NULL;
+ struct static_key_mod *jlm, **prev;
for (iter = iter_start; iter < iter_stop; iter++) {
if (iter->key == (jump_label_t)(unsigned long)key)
continue;
- key = (struct jump_label_key *)(unsigned long)iter->key;
+ key = (struct static_key *)(unsigned long)iter->key;
if (__module_address(iter->key) == mod)
continue;
@@ -438,7 +438,7 @@ int jump_label_text_reserved(void *start, void *end)
return ret;
}
-static void jump_label_update(struct jump_label_key *key, int enable)
+static void jump_label_update(struct static_key *key, int enable)
{
struct jump_entry *stop = __stop___jump_table;
struct jump_entry *entry = jump_label_get_entries(key);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index a357dbf..ef5e2c4 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -162,13 +162,13 @@ static int sched_feat_show(struct seq_file *m, void *v)
#ifdef HAVE_JUMP_LABEL
-#define jump_label_key__true JUMP_LABEL_INIT_TRUE
-#define jump_label_key__false JUMP_LABEL_INIT_FALSE
+#define jump_label_key__true STATIC_KEY_INIT_TRUE
+#define jump_label_key__false STATIC_KEY_INIT_FALSE
#define SCHED_FEAT(name, enabled) \
jump_label_key__##enabled ,
-struct jump_label_key sched_feat_keys[__SCHED_FEAT_NR] = {
+struct static_key sched_feat_keys[__SCHED_FEAT_NR] = {
#include "features.h"
};
@@ -176,14 +176,14 @@ struct jump_label_key sched_feat_keys[__SCHED_FEAT_NR] = {
static void sched_feat_disable(int i)
{
- if (jump_label_true(&sched_feat_keys[i]))
- jump_label_dec(&sched_feat_keys[i]);
+ if (static_key_true(&sched_feat_keys[i]))
+ static_key_slow_dec(&sched_feat_keys[i]);
}
static void sched_feat_enable(int i)
{
- if (!jump_label_true(&sched_feat_keys[i]))
- jump_label_inc(&sched_feat_keys[i]);
+ if (!static_key_true(&sched_feat_keys[i]))
+ static_key_slow_inc(&sched_feat_keys[i]);
}
#else
static void sched_feat_disable(int i) { };
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 67206ae..ca3af63 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1399,7 +1399,7 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued)
#ifdef CONFIG_CFS_BANDWIDTH
#ifdef HAVE_JUMP_LABEL
-static struct jump_label_key __cfs_bandwidth_used;
+static struct static_key __cfs_bandwidth_used;
static inline bool cfs_bandwidth_used(void)
{
@@ -1410,9 +1410,9 @@ void account_cfs_bandwidth_used(int enabled, int was_enabled)
{
/* only need to count groups transitioning between enabled/!enabled */
if (enabled && !was_enabled)
- jump_label_inc(&__cfs_bandwidth_used);
+ static_key_slow_inc(&__cfs_bandwidth_used);
else if (!enabled && was_enabled)
- jump_label_dec(&__cfs_bandwidth_used);
+ static_key_slow_dec(&__cfs_bandwidth_used);
}
#else /* HAVE_JUMP_LABEL */
static bool cfs_bandwidth_used(void)
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index addeb9e..cdfa8fc 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -611,7 +611,7 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
* Tunables that become constants when CONFIG_SCHED_DEBUG is off:
*/
#ifdef CONFIG_SCHED_DEBUG
-# include <linux/jump_label.h>
+# include <linux/static_key.h>
# define const_debug __read_mostly
#else
# define const_debug const
@@ -630,18 +630,18 @@ enum {
#undef SCHED_FEAT
#if defined(CONFIG_SCHED_DEBUG) && defined(HAVE_JUMP_LABEL)
-static __always_inline bool static_branch__true(struct jump_label_key *key)
+static __always_inline bool static_branch__true(struct static_key *key)
{
return very_likely(key); /* Not out of line branch. */
}
-static __always_inline bool static_branch__false(struct jump_label_key *key)
+static __always_inline bool static_branch__false(struct static_key *key)
{
return very_unlikely(key); /* Out of line branch. */
}
#define SCHED_FEAT(name, enabled) \
-static __always_inline bool static_branch_##name(struct jump_label_key *key) \
+static __always_inline bool static_branch_##name(struct static_key *key) \
{ \
return static_branch__##enabled(key); \
}
@@ -650,7 +650,7 @@ static __always_inline bool static_branch_##name(struct jump_label_key *key) \
#undef SCHED_FEAT
-extern struct jump_label_key sched_feat_keys[__SCHED_FEAT_NR];
+extern struct static_key sched_feat_keys[__SCHED_FEAT_NR];
#define sched_feat(x) (static_branch_##x(&sched_feat_keys[__SCHED_FEAT_##x]))
#else /* !(SCHED_DEBUG && HAVE_JUMP_LABEL) */
#define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x))
diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c
index ad32493..5e37c26 100644
--- a/kernel/tracepoint.c
+++ b/kernel/tracepoint.c
@@ -25,7 +25,7 @@
#include <linux/err.h>
#include <linux/slab.h>
#include <linux/sched.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
extern struct tracepoint * const __start___tracepoints_ptrs[];
extern struct tracepoint * const __stop___tracepoints_ptrs[];
@@ -256,9 +256,9 @@ static void set_tracepoint(struct tracepoint_entry **entry,
{
WARN_ON(strcmp((*entry)->name, elem->name) != 0);
- if (elem->regfunc && !jump_label_true(&elem->key) && active)
+ if (elem->regfunc && !static_key_true(&elem->key) && active)
elem->regfunc();
- else if (elem->unregfunc && jump_label_true(&elem->key) && !active)
+ else if (elem->unregfunc && static_key_true(&elem->key) && !active)
elem->unregfunc();
/*
@@ -269,10 +269,10 @@ static void set_tracepoint(struct tracepoint_entry **entry,
* is used.
*/
rcu_assign_pointer(elem->funcs, (*entry)->funcs);
- if (active && !jump_label_true(&elem->key))
- jump_label_inc(&elem->key);
- else if (!active && jump_label_true(&elem->key))
- jump_label_dec(&elem->key);
+ if (active && !static_key_true(&elem->key))
+ static_key_slow_inc(&elem->key);
+ else if (!active && static_key_true(&elem->key))
+ static_key_slow_dec(&elem->key);
}
/*
@@ -283,11 +283,11 @@ static void set_tracepoint(struct tracepoint_entry **entry,
*/
static void disable_tracepoint(struct tracepoint *elem)
{
- if (elem->unregfunc && jump_label_true(&elem->key))
+ if (elem->unregfunc && static_key_true(&elem->key))
elem->unregfunc();
- if (jump_label_true(&elem->key))
- jump_label_dec(&elem->key);
+ if (static_key_true(&elem->key))
+ static_key_slow_dec(&elem->key);
rcu_assign_pointer(elem->funcs, NULL);
}
diff --git a/net/core/dev.c b/net/core/dev.c
index 6503c14..9fa6d8a 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -134,7 +134,7 @@
#include <linux/inetdevice.h>
#include <linux/cpu_rmap.h>
#include <linux/net_tstamp.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
#include <net/flow_keys.h>
#include "net-sysfs.h"
@@ -1441,11 +1441,11 @@ int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
}
EXPORT_SYMBOL(call_netdevice_notifiers);
-static struct jump_label_key netstamp_needed __read_mostly;
+static struct static_key netstamp_needed __read_mostly;
#ifdef HAVE_JUMP_LABEL
-/* We are not allowed to call jump_label_dec() from irq context
+/* We are not allowed to call static_key_slow_dec() from irq context
* If net_disable_timestamp() is called from irq context, defer the
- * jump_label_dec() calls.
+ * static_key_slow_dec() calls.
*/
static atomic_t netstamp_needed_deferred;
#endif
@@ -1457,12 +1457,12 @@ void net_enable_timestamp(void)
if (deferred) {
while (--deferred)
- jump_label_dec(&netstamp_needed);
+ static_key_slow_dec(&netstamp_needed);
return;
}
#endif
WARN_ON(in_interrupt());
- jump_label_inc(&netstamp_needed);
+ static_key_slow_inc(&netstamp_needed);
}
EXPORT_SYMBOL(net_enable_timestamp);
@@ -1474,7 +1474,7 @@ void net_disable_timestamp(void)
return;
}
#endif
- jump_label_dec(&netstamp_needed);
+ static_key_slow_dec(&netstamp_needed);
}
EXPORT_SYMBOL(net_disable_timestamp);
@@ -2660,7 +2660,7 @@ EXPORT_SYMBOL(__skb_get_rxhash);
struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
EXPORT_SYMBOL(rps_sock_flow_table);
-struct jump_label_key rps_needed __read_mostly;
+struct static_key rps_needed __read_mostly;
static struct rps_dev_flow *
set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c
index a1727cd..4955862 100644
--- a/net/core/net-sysfs.c
+++ b/net/core/net-sysfs.c
@@ -608,10 +608,10 @@ static ssize_t store_rps_map(struct netdev_rx_queue *queue,
spin_unlock(&rps_map_lock);
if (map)
- jump_label_inc(&rps_needed);
+ static_key_slow_inc(&rps_needed);
if (old_map) {
kfree_rcu(old_map, rcu);
- jump_label_dec(&rps_needed);
+ static_key_slow_dec(&rps_needed);
}
free_cpumask_var(mask);
return len;
diff --git a/net/core/sock.c b/net/core/sock.c
index 3e81fd2..3a4e581 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -111,7 +111,7 @@
#include <linux/init.h>
#include <linux/highmem.h>
#include <linux/user_namespace.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
#include <linux/memcontrol.h>
#include <asm/uaccess.h>
@@ -184,7 +184,7 @@ void mem_cgroup_sockets_destroy(struct cgroup *cgrp, struct cgroup_subsys *ss)
static struct lock_class_key af_family_keys[AF_MAX];
static struct lock_class_key af_family_slock_keys[AF_MAX];
-struct jump_label_key memcg_socket_limit_enabled;
+struct static_key memcg_socket_limit_enabled;
EXPORT_SYMBOL(memcg_socket_limit_enabled);
/*
diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c
index d05559d..0c28508 100644
--- a/net/core/sysctl_net_core.c
+++ b/net/core/sysctl_net_core.c
@@ -69,9 +69,9 @@ static int rps_sock_flow_sysctl(ctl_table *table, int write,
if (sock_table != orig_sock_table) {
rcu_assign_pointer(rps_sock_flow_table, sock_table);
if (sock_table)
- jump_label_inc(&rps_needed);
+ static_key_slow_inc(&rps_needed);
if (orig_sock_table) {
- jump_label_dec(&rps_needed);
+ static_key_slow_dec(&rps_needed);
synchronize_rcu();
vfree(orig_sock_table);
}
diff --git a/net/ipv4/tcp_memcontrol.c b/net/ipv4/tcp_memcontrol.c
index 4997878..602fb30 100644
--- a/net/ipv4/tcp_memcontrol.c
+++ b/net/ipv4/tcp_memcontrol.c
@@ -111,7 +111,7 @@ void tcp_destroy_cgroup(struct cgroup *cgrp, struct cgroup_subsys *ss)
val = res_counter_read_u64(&tcp->tcp_memory_allocated, RES_LIMIT);
if (val != RESOURCE_MAX)
- jump_label_dec(&memcg_socket_limit_enabled);
+ static_key_slow_dec(&memcg_socket_limit_enabled);
}
EXPORT_SYMBOL(tcp_destroy_cgroup);
@@ -143,9 +143,9 @@ static int tcp_update_limit(struct mem_cgroup *memcg, u64 val)
net->ipv4.sysctl_tcp_mem[i]);
if (val == RESOURCE_MAX && old_lim != RESOURCE_MAX)
- jump_label_dec(&memcg_socket_limit_enabled);
+ static_key_slow_dec(&memcg_socket_limit_enabled);
else if (old_lim == RESOURCE_MAX && val != RESOURCE_MAX)
- jump_label_inc(&memcg_socket_limit_enabled);
+ static_key_slow_inc(&memcg_socket_limit_enabled);
return 0;
}
diff --git a/net/netfilter/core.c b/net/netfilter/core.c
index b4e8ff0..e1b7e05 100644
--- a/net/netfilter/core.c
+++ b/net/netfilter/core.c
@@ -56,7 +56,7 @@ struct list_head nf_hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS] __read_mostly;
EXPORT_SYMBOL(nf_hooks);
#if defined(CONFIG_JUMP_LABEL)
-struct jump_label_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
+struct static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
EXPORT_SYMBOL(nf_hooks_needed);
#endif
@@ -77,7 +77,7 @@ int nf_register_hook(struct nf_hook_ops *reg)
list_add_rcu(®->list, elem->list.prev);
mutex_unlock(&nf_hook_mutex);
#if defined(CONFIG_JUMP_LABEL)
- jump_label_inc(&nf_hooks_needed[reg->pf][reg->hooknum]);
+ static_key_slow_inc(&nf_hooks_needed[reg->pf][reg->hooknum]);
#endif
return 0;
}
@@ -89,7 +89,7 @@ void nf_unregister_hook(struct nf_hook_ops *reg)
list_del_rcu(®->list);
mutex_unlock(&nf_hook_mutex);
#if defined(CONFIG_JUMP_LABEL)
- jump_label_dec(&nf_hooks_needed[reg->pf][reg->hooknum]);
+ static_key_slow_dec(&nf_hooks_needed[reg->pf][reg->hooknum]);
#endif
synchronize_net();
}
* Ingo Molnar <[email protected]> wrote:
> So, a modified scheme would be:
>
> #include <linux/static_key.h>
>
> struct static_key key = STATIC_KEY_INIT_TRUE;
>
> if (static_key_false(&key))
> do unlikely code
> else
> do likely code
>
> Or:
>
> if (static_key_true(&key))
> do likely code
> else
> do unlikely code
>
> The static key is modified via:
>
> static_key_slow_inc(&key);
> ...
> static_key_slow_dec(&key);
>
> Is that API fine? I'll rework the series to such an effect if
> everyone agrees.
I.e. something like the patch below on top of
tip:perf/jump-labels.
Untested - will test it and will refactor the series if
everyone's happy.
Thanks,
Ingo
diff --git a/Documentation/jump-label.txt b/Documentation/jump-label.txt
index ba67ca7..9570e22 100644
--- a/Documentation/jump-label.txt
+++ b/Documentation/jump-label.txt
@@ -32,7 +32,7 @@ the branch site to change the branch direction.
For example, if we have a simple branch that is disabled by default:
- if (very_unlikely(&key))
+ if (static_key_false(&key))
printk("I am the true branch\n");
Thus, by default the 'printk' will not be emitted. And the code generated will
@@ -64,21 +64,21 @@ the stack or dynamically allocated at run-time.
The key is then used in code as:
- if (very_unlikely(&key))
+ if (static_key_false(&key))
do unlikely code
else
do likely code
Or:
- if (very_likely(&key))
+ if (static_key_true(&key))
do likely code
else
do unlikely code
A key that is initialized via 'STATIC_KEY_INIT_FALSE', must be used in a
-'very_unlikely()' construct. Likewise, a key initialized via
-'STATIC_KEY_INIT_TRUE' must be used in a 'very_likely()' construct.
+'static_key_false()' construct. Likewise, a key initialized via
+'STATIC_KEY_INIT_TRUE' must be used in a 'static_key_true()' construct.
A single key can be used in many branches, but all the branches must match
the way that the key has been initialized.
@@ -99,7 +99,7 @@ An example usage in the kernel is the implementation of tracepoints:
static inline void trace_##name(proto) \
{ \
- if (very_unlikely(&__tracepoint_##name.key)) \
+ if (static_key_false(&__tracepoint_##name.key)) \
__DO_TRACE(&__tracepoint_##name, \
TP_PROTO(data_proto), \
TP_ARGS(data_args), \
@@ -145,7 +145,7 @@ SYSCALL_DEFINE0(getppid)
{
int pid;
-+ if (very_unlikely(&key))
++ if (static_key_false(&key))
+ printk("I am the true branch\n");
rcu_read_lock();
diff --git a/arch/x86/kvm/mmu_audit.c b/arch/x86/kvm/mmu_audit.c
index b21c123..ea7b4fd 100644
--- a/arch/x86/kvm/mmu_audit.c
+++ b/arch/x86/kvm/mmu_audit.c
@@ -250,7 +250,7 @@ static void __kvm_mmu_audit(struct kvm_vcpu *vcpu, int point)
static inline void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point)
{
- if (very_unlikely((&mmu_audit_key)))
+ if (static_key_false((&mmu_audit_key)))
__kvm_mmu_audit(vcpu, point);
}
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index 25c589b..30d0023 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -9,7 +9,7 @@
*
* Jump labels provide an interface to generate dynamic branches using
* self-modifying code. Assuming toolchain and architecture support the result
- * of a "if (very_unlikely(&key))" statement is a unconditional branch (which
+ * of a "if (static_key_false(&key))" statement is a unconditional branch (which
* defaults to false - and the true block is placed out of line).
*
* However at runtime we can change the branch target using
@@ -34,15 +34,15 @@
*
* struct static_key my_key = STATIC_KEY_INIT_TRUE;
*
- * if (very_likely(&my_key)) {
+ * if (static_key_true(&my_key)) {
* }
*
* will result in the true case being in-line and starts the key with a single
- * reference. Mixing very_likely() and very_unlikely() on the same key is not
+ * reference. Mixing static_key_true() and static_key_false() on the same key is not
* allowed.
*
* Not initializing the key (static data is initialized to 0s anyway) is the
- * same as using STATIC_KEY_INIT_FALSE and very_unlikely() is
+ * same as using STATIC_KEY_INIT_FALSE and static_key_false() is
* equivalent with static_branch().
*
*/
@@ -97,17 +97,17 @@ static inline bool jump_label_get_branch_default(struct static_key *key)
return false;
}
-static __always_inline bool very_unlikely(struct static_key *key)
+static __always_inline bool static_key_false(struct static_key *key)
{
return arch_static_branch(key);
}
-static __always_inline bool very_likely(struct static_key *key)
+static __always_inline bool static_key_true(struct static_key *key)
{
- return !very_unlikely(key);
+ return !static_key_false(key);
}
-/* Deprecated. Please use 'very_unlikely() instead. */
+/* Deprecated. Please use 'static_key_false() instead. */
static __always_inline bool static_branch(struct static_key *key)
{
return arch_static_branch(key);
@@ -153,21 +153,21 @@ struct static_key_deferred {
struct static_key key;
};
-static __always_inline bool very_unlikely(struct static_key *key)
+static __always_inline bool static_key_false(struct static_key *key)
{
if (unlikely(atomic_read(&key->enabled)) > 0)
return true;
return false;
}
-static __always_inline bool very_likely(struct static_key *key)
+static __always_inline bool static_key_true(struct static_key *key)
{
if (likely(atomic_read(&key->enabled)) > 0)
return true;
return false;
}
-/* Deprecated. Please use 'very_unlikely() instead. */
+/* Deprecated. Please use 'static_key_false() instead. */
static __always_inline bool static_branch(struct static_key *key)
{
if (unlikely(atomic_read(&key->enabled)) > 0)
diff --git a/include/linux/netfilter.h b/include/linux/netfilter.h
index 9f10318..29734be 100644
--- a/include/linux/netfilter.h
+++ b/include/linux/netfilter.h
@@ -169,7 +169,7 @@ static inline bool nf_hooks_active(u_int8_t pf, unsigned int hook)
{
if (__builtin_constant_p(pf) &&
__builtin_constant_p(hook))
- return very_unlikely(&nf_hooks_needed[pf][hook]);
+ return static_key_false(&nf_hooks_needed[pf][hook]);
return !list_empty(&nf_hooks[pf][hook]);
}
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 434b51b..0d21e6f 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -1066,7 +1066,7 @@ perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
{
struct pt_regs hot_regs;
- if (very_unlikely(&perf_swevent_enabled[event_id])) {
+ if (static_key_false(&perf_swevent_enabled[event_id])) {
if (!regs) {
perf_fetch_caller_regs(&hot_regs);
regs = &hot_regs;
@@ -1080,7 +1080,7 @@ extern struct static_key_deferred perf_sched_events;
static inline void perf_event_task_sched_in(struct task_struct *prev,
struct task_struct *task)
{
- if (very_unlikely(&perf_sched_events.key))
+ if (static_key_false(&perf_sched_events.key))
__perf_event_task_sched_in(prev, task);
}
@@ -1089,7 +1089,7 @@ static inline void perf_event_task_sched_out(struct task_struct *prev,
{
perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, NULL, 0);
- if (very_unlikely(&perf_sched_events.key))
+ if (static_key_false(&perf_sched_events.key))
__perf_event_task_sched_out(prev, next);
}
diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h
index ec28a94..bd96ecd 100644
--- a/include/linux/tracepoint.h
+++ b/include/linux/tracepoint.h
@@ -145,7 +145,7 @@ static inline void tracepoint_synchronize_unregister(void)
extern struct tracepoint __tracepoint_##name; \
static inline void trace_##name(proto) \
{ \
- if (very_unlikely(&__tracepoint_##name.key)) \
+ if (static_key_false(&__tracepoint_##name.key)) \
__DO_TRACE(&__tracepoint_##name, \
TP_PROTO(data_proto), \
TP_ARGS(data_args), \
diff --git a/include/net/sock.h b/include/net/sock.h
index 907bbe0..dcde2d9 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -930,7 +930,7 @@ static inline struct cg_proto *parent_cg_proto(struct proto *proto,
{
return proto->proto_cgroup(parent_mem_cgroup(cg_proto->memcg));
}
-#define mem_cgroup_sockets_enabled very_unlikely(&memcg_socket_limit_enabled)
+#define mem_cgroup_sockets_enabled static_key_false(&memcg_socket_limit_enabled)
#else
#define mem_cgroup_sockets_enabled 0
static inline struct cg_proto *parent_cg_proto(struct proto *proto,
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 1169246..26b02da 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -894,7 +894,7 @@ static void update_rq_clock_task(struct rq *rq, s64 delta)
delta -= irq_delta;
#endif
#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
- if (very_unlikely((¶virt_steal_rq_enabled))) {
+ if (static_key_false((¶virt_steal_rq_enabled))) {
u64 st;
steal = paravirt_steal_clock(cpu_of(rq));
@@ -2755,7 +2755,7 @@ void account_idle_time(cputime_t cputime)
static __always_inline bool steal_account_process_tick(void)
{
#ifdef CONFIG_PARAVIRT
- if (very_unlikely(¶virt_steal_enabled)) {
+ if (static_key_false(¶virt_steal_enabled)) {
u64 steal, st = 0;
steal = paravirt_steal_clock(smp_processor_id());
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 3c41bf0..34748d6 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1405,7 +1405,7 @@ static struct static_key __cfs_bandwidth_used;
static inline bool cfs_bandwidth_used(void)
{
- return very_unlikely(&__cfs_bandwidth_used);
+ return static_key_false(&__cfs_bandwidth_used);
}
void account_cfs_bandwidth_used(int enabled, int was_enabled)
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 02a166f..7a1dc91 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -627,12 +627,12 @@ enum {
#if defined(CONFIG_SCHED_DEBUG) && defined(HAVE_JUMP_LABEL)
static __always_inline bool static_branch__true(struct static_key *key)
{
- return very_likely(key); /* Not out of line branch. */
+ return static_key_true(key); /* Not out of line branch. */
}
static __always_inline bool static_branch__false(struct static_key *key)
{
- return very_unlikely(key); /* Out of line branch. */
+ return static_key_false(key); /* Out of line branch. */
}
#define SCHED_FEAT(name, enabled) \
diff --git a/net/core/dev.c b/net/core/dev.c
index dd7377f..6982bfd 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -1481,12 +1481,12 @@ EXPORT_SYMBOL(net_disable_timestamp);
static inline void net_timestamp_set(struct sk_buff *skb)
{
skb->tstamp.tv64 = 0;
- if (very_unlikely(&netstamp_needed))
+ if (static_key_false(&netstamp_needed))
__net_timestamp(skb);
}
#define net_timestamp_check(COND, SKB) \
- if (very_unlikely(&netstamp_needed)) { \
+ if (static_key_false(&netstamp_needed)) { \
if ((COND) && !(SKB)->tstamp.tv64) \
__net_timestamp(SKB); \
} \
@@ -2945,7 +2945,7 @@ int netif_rx(struct sk_buff *skb)
trace_netif_rx(skb);
#ifdef CONFIG_RPS
- if (very_unlikely(&rps_needed)) {
+ if (static_key_false(&rps_needed)) {
struct rps_dev_flow voidflow, *rflow = &voidflow;
int cpu;
@@ -3309,7 +3309,7 @@ int netif_receive_skb(struct sk_buff *skb)
return NET_RX_SUCCESS;
#ifdef CONFIG_RPS
- if (very_unlikely(&rps_needed)) {
+ if (static_key_false(&rps_needed)) {
struct rps_dev_flow voidflow, *rflow = &voidflow;
int cpu, ret;
On Thu, 2012-02-23 at 23:39 +0100, Ingo Molnar wrote:
> Untested - will test it and will refactor the series if
> everyone's happy.
>
Acked-by: Steven Rostedt <[email protected]>
;-)
-- Steve
On Thu, Feb 23, 2012 at 2:33 PM, Ingo Molnar <[email protected]> wrote:
>
> At the risk of being flamed some more, where does the confusion
> stem from?
>From the fact that "very" is an English word that means "very", and
"unlikely()" and "likely()" are already used in their obvious meaning.
So quite frankly, "very_unlikely()" quite naturally means something
else than you are trying to make it mean.
The fact that EVERY SINGLE OTHER OPERATION that worked on that data
structure used to be named "jump_label_xyz()" and is now named
"static_key_xyz()" is also a big clue, I think. Naming it anything
else was always a mistake.
Seriously, I don't understand why you don't just use the obvious name.
The data structure is named "static_key". The things that change it
are named "static_key_inc()" or something. So a name like
"static_key_true()" is simply *better*, isn't it?
It's not just about less confusion, it's actually about just having
consistent naming.
Linus
* Ingo Molnar ([email protected]) wrote:
>
> * Ingo Molnar <[email protected]> wrote:
>
> > So, a modified scheme would be:
> >
> > #include <linux/static_key.h>
> >
> > struct static_key key = STATIC_KEY_INIT_TRUE;
> >
> > if (static_key_false(&key))
> > do unlikely code
> > else
> > do likely code
> >
> > Or:
> >
> > if (static_key_true(&key))
> > do likely code
> > else
> > do unlikely code
> >
> > The static key is modified via:
> >
> > static_key_slow_inc(&key);
> > ...
> > static_key_slow_dec(&key);
> >
> > Is that API fine? I'll rework the series to such an effect if
> > everyone agrees.
>
> I.e. something like the patch below on top of
> tip:perf/jump-labels.
>
> Untested - will test it and will refactor the series if
> everyone's happy.
Hi Ingo,
Reading your documentation updates makes me realise that adding the
"inline" keyword in there would make the whole thing even clearer:
struct static_key key = STATIC_KEY_INLINE_TRUE_INIT;
struct static_key key = STATIC_KEY_INLINE_FALSE_INIT;
static_key_inline_true() / static_key_inline_false()
to show that the "true/false" in there does not mean that the key will
always be true or false (the key value can indeed by changed by calling
static_key_slow_inc/dec), but that the inlined path is either the true
of false branch.
The rest looks fine.
Best regards,
Mathieu
--
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com
On Thu, Feb 23, 2012 at 06:18:42PM -0500, Mathieu Desnoyers wrote:
> * Ingo Molnar ([email protected]) wrote:
> >
> > * Ingo Molnar <[email protected]> wrote:
> >
> > > So, a modified scheme would be:
> > >
> > > #include <linux/static_key.h>
> > >
> > > struct static_key key = STATIC_KEY_INIT_TRUE;
> > >
> > > if (static_key_false(&key))
> > > do unlikely code
> > > else
> > > do likely code
> > >
> > > Or:
> > >
> > > if (static_key_true(&key))
> > > do likely code
> > > else
> > > do unlikely code
> > >
> > > The static key is modified via:
> > >
> > > static_key_slow_inc(&key);
> > > ...
> > > static_key_slow_dec(&key);
> > >
> > > Is that API fine? I'll rework the series to such an effect if
> > > everyone agrees.
> >
> > I.e. something like the patch below on top of
> > tip:perf/jump-labels.
> >
> > Untested - will test it and will refactor the series if
> > everyone's happy.
>
> Hi Ingo,
>
> Reading your documentation updates makes me realise that adding the
> "inline" keyword in there would make the whole thing even clearer:
>
> struct static_key key = STATIC_KEY_INLINE_TRUE_INIT;
> struct static_key key = STATIC_KEY_INLINE_FALSE_INIT;
>
> static_key_inline_true() / static_key_inline_false()
>
> to show that the "true/false" in there does not mean that the key will
> always be true or false (the key value can indeed by changed by calling
> static_key_slow_inc/dec), but that the inlined path is either the true
> of false branch.
>
Also, as part of the API, there is a test to check the branch direction
- which was 'jump_label_true(key)', but is now also 'static_key_true(key)', so
we are going to have to change either the branch site or the test for
true/false name. The above 'static_key_inline_true/false' solves that.
Also, I do like having a consistent 'static_key_*' prefix for the entire API -
definitely an improvement for me.
Thanks,
-Jason
On Thu, Feb 23, 2012 at 11:33:48PM +0100, Ingo Molnar wrote:
>
> Before:
>
> ffffffff810441f0 <sys_getppid>:
> ffffffff810441f0: 8b 05 8a 52 d8 00 mov 0xd8528a(%rip),%eax # ffffffff81dc9480 <key>
> ffffffff810441f6: 55 push %rbp
> ffffffff810441f7: 48 89 e5 mov %rsp,%rbp
> ffffffff810441fa: 85 c0 test %eax,%eax
> ffffffff810441fc: 75 27 jne ffffffff81044225 <sys_getppid+0x35>
> ffffffff810441fe: 65 48 8b 04 25 c0 b6 mov %gs:0xb6c0,%rax
> ffffffff81044205: 00 00
> ffffffff81044207: 48 8b 80 80 02 00 00 mov 0x280(%rax),%rax
> ffffffff8104420e: 48 8b 80 b0 02 00 00 mov 0x2b0(%rax),%rax
> ffffffff81044215: 48 8b b8 e8 02 00 00 mov 0x2e8(%rax),%rdi
> ffffffff8104421c: e8 2f da 00 00 callq ffffffff81051c50 <pid_vnr>
> ffffffff81044221: 5d pop %rbp
> ffffffff81044222: 48 98 cltq
> ffffffff81044224: c3 retq
> ffffffff81044225: 48 c7 c7 13 53 98 81 mov $0xffffffff81985313,%rdi
> ffffffff8104422c: 31 c0 xor %eax,%eax
> ffffffff8104422e: e8 60 0f 6d 00 callq ffffffff81715193 <printk>
> ffffffff81044233: eb c9 jmp ffffffff810441fe <sys_getppid+0xe>
> ffffffff81044235: 66 66 2e 0f 1f 84 00 data32 nopw %cs:0x0(%rax,%rax,1)
> ffffffff8104423c: 00 00 00 00
>
> After:
>
> ffffffff81044290 <sys_getppid>:
> ffffffff81044290: 55 push %rbp
> ffffffff81044291: 48 89 e5 mov %rsp,%rbp
> ffffffff81044294: e9 00 00 00 00 jmpq ffffffff81044299 <sys_getppid+0x9>
> ffffffff81044299: 65 48 8b 04 25 c0 b6 mov %gs:0xb6c0,%rax
> ffffffff810442a0: 00 00
> ffffffff810442a2: 48 8b 80 80 02 00 00 mov 0x280(%rax),%rax
> ffffffff810442a9: 48 8b 80 b0 02 00 00 mov 0x2b0(%rax),%rax
> ffffffff810442b0: 48 8b b8 e8 02 00 00 mov 0x2e8(%rax),%rdi
> ffffffff810442b7: e8 f4 d9 00 00 callq ffffffff81051cb0 <pid_vnr>
> ffffffff810442bc: 5d pop %rbp
> ffffffff810442bd: 48 98 cltq
> ffffffff810442bf: c3 retq
> ffffffff810442c0: 48 c7 c7 e3 54 98 81 mov $0xffffffff819854e3,%rdi
> ffffffff810442c7: 31 c0 xor %eax,%eax
> ffffffff810442c9: e8 71 13 6d 00 callq ffffffff8171563f <printk>
> ffffffff810442ce: eb c9 jmp ffffffff81044299 <sys_getppid+0x9>
>
> The prior slowpath test:
>
> ffffffff810441fa: 85 c0 test %eax,%eax
> ffffffff810441fc: 75 27 jne ffffffff81044225 <sys_getppid+0x35>
>
> became even faster and turned into a single NOP, making the
> fastpath even faster.
>
and the: mov 0xd8528a(%rip),%eax # ffffffff81dc9480 <key>
is no longer present.
Thanks,
-Jason
* Ingo Molnar <[email protected]> wrote:
> * Ingo Molnar <[email protected]> wrote:
>
> > So, a modified scheme would be:
> >
> > #include <linux/static_key.h>
> >
> > struct static_key key = STATIC_KEY_INIT_TRUE;
> >
> > if (static_key_false(&key))
> > do unlikely code
> > else
> > do likely code
> >
> > Or:
> >
> > if (static_key_true(&key))
> > do likely code
> > else
> > do unlikely code
> >
> > The static key is modified via:
> >
> > static_key_slow_inc(&key);
> > ...
> > static_key_slow_dec(&key);
> >
> > Is that API fine? I'll rework the series to such an effect if
> > everyone agrees.
>
> I.e. something like the patch below on top of
> tip:perf/jump-labels.
>
> Untested - will test it and will refactor the series if
> everyone's happy.
So, below is the second version of my 'struct static_key' patch,
with the confusing very_likely()/very_unlikely() method renamed
to static_key_true()/static_key_false().
Thanks,
Ingo
-------------------->
>From 25764dffac2414774b65a831733b232d33006884 Mon Sep 17 00:00:00 2001
From: Ingo Molnar <[email protected]>
Date: Fri, 24 Feb 2012 08:31:31 +0100
Subject: [PATCH] static keys: Introduce 'struct static_key', static_key_true()/false() and static_key_slow_[inc|dec]()
So here's a boot tested patch on top of Jason's series that does
all the cleanups I talked about and turns jump labels into a
more intuitive to use facility. It should also address the
various misconceptions and confusions that surround jump labels.
Typical usage scenarios:
#include <linux/static_key.h>
struct static_key key = STATIC_KEY_INIT_TRUE;
if (static_key_false(&key))
do unlikely code
else
do likely code
Or:
if (static_key_true(&key))
do likely code
else
do unlikely code
The static key is modified via:
static_key_slow_inc(&key);
...
static_key_slow_dec(&key);
The 'slow' prefix makes it abundantly clear that this is an
expensive operation.
I've updated all in-kernel code to use this everywhere. Note
that I (intentionally) have not pushed through the rename
blindly through to the lowest levels: the actual jump-label
patching arch facility should be named like that, so we want to
decouple jump labels from the static-key facility a bit.
On non-jump-label enabled architectures static keys default to
likely()/unlikely() branches.
Signed-off-by: Ingo Molnar <[email protected]>
Cc: Steven Rostedt <[email protected]>
Cc: Jason Baron <[email protected]>
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: Linus Torvalds <[email protected]>
Link: http://lkml.kernel.org/r/[email protected]
Signed-off-by: Ingo Molnar <[email protected]>
---
arch/Kconfig | 29 +++++--
arch/ia64/include/asm/paravirt.h | 6 +-
arch/ia64/kernel/paravirt.c | 4 +-
arch/mips/include/asm/jump_label.h | 2 +-
arch/powerpc/include/asm/jump_label.h | 2 +-
arch/s390/include/asm/jump_label.h | 2 +-
arch/sparc/include/asm/jump_label.h | 2 +-
arch/x86/include/asm/jump_label.h | 6 +-
arch/x86/include/asm/paravirt.h | 6 +-
arch/x86/kernel/kvm.c | 4 +-
arch/x86/kernel/paravirt.c | 4 +-
arch/x86/kvm/mmu_audit.c | 8 +-
include/linux/jump_label.h | 139 +++++++++++++++++++++++---------
include/linux/netdevice.h | 4 +-
include/linux/netfilter.h | 6 +-
include/linux/perf_event.h | 12 ++--
include/linux/static_key.h | 1 +
include/linux/tracepoint.h | 8 +-
include/net/sock.h | 6 +-
kernel/events/core.c | 16 ++--
kernel/jump_label.c | 128 +++++++++++++++++-------------
kernel/sched/core.c | 18 ++--
kernel/sched/fair.c | 8 +-
kernel/sched/sched.h | 14 ++--
kernel/tracepoint.c | 20 +++---
net/core/dev.c | 24 +++---
net/core/net-sysfs.c | 4 +-
net/core/sock.c | 4 +-
net/core/sysctl_net_core.c | 4 +-
net/ipv4/tcp_memcontrol.c | 6 +-
net/netfilter/core.c | 6 +-
31 files changed, 298 insertions(+), 205 deletions(-)
diff --git a/arch/Kconfig b/arch/Kconfig
index 4f55c73..5b448a7 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -47,18 +47,29 @@ config KPROBES
If in doubt, say "N".
config JUMP_LABEL
- bool "Optimize trace point call sites"
+ bool "Optimize very unlikely/likely branches"
depends on HAVE_ARCH_JUMP_LABEL
help
+ This option enables a transparent branch optimization that
+ makes certain almost-always-true or almost-always-false branch
+ conditions even cheaper to execute within the kernel.
+
+ Certain performance-sensitive kernel code, such as trace points,
+ scheduler functionality, networking code and KVM have such
+ branches and include support for this optimization technique.
+
If it is detected that the compiler has support for "asm goto",
- the kernel will compile trace point locations with just a
- nop instruction. When trace points are enabled, the nop will
- be converted to a jump to the trace function. This technique
- lowers overhead and stress on the branch prediction of the
- processor.
-
- On i386, options added to the compiler flags may increase
- the size of the kernel slightly.
+ the kernel will compile such branches with just a nop
+ instruction. When the condition flag is toggled to true, the
+ nop will be converted to a jump instruction to execute the
+ conditional block of instructions.
+
+ This technique lowers overhead and stress on the branch prediction
+ of the processor and generally makes the kernel faster. The update
+ of the condition is slower, but those are always very rare.
+
+ ( On 32-bit x86, the necessary options added to the compiler
+ flags may increase the size of the kernel slightly. )
config OPTPROBES
def_bool y
diff --git a/arch/ia64/include/asm/paravirt.h b/arch/ia64/include/asm/paravirt.h
index 32551d3..b149b88 100644
--- a/arch/ia64/include/asm/paravirt.h
+++ b/arch/ia64/include/asm/paravirt.h
@@ -281,9 +281,9 @@ paravirt_init_missing_ticks_accounting(int cpu)
pv_time_ops.init_missing_ticks_accounting(cpu);
}
-struct jump_label_key;
-extern struct jump_label_key paravirt_steal_enabled;
-extern struct jump_label_key paravirt_steal_rq_enabled;
+struct static_key;
+extern struct static_key paravirt_steal_enabled;
+extern struct static_key paravirt_steal_rq_enabled;
static inline int
paravirt_do_steal_accounting(unsigned long *new_itm)
diff --git a/arch/ia64/kernel/paravirt.c b/arch/ia64/kernel/paravirt.c
index 1008682..1b22f6d 100644
--- a/arch/ia64/kernel/paravirt.c
+++ b/arch/ia64/kernel/paravirt.c
@@ -634,8 +634,8 @@ struct pv_irq_ops pv_irq_ops = {
* pv_time_ops
* time operations
*/
-struct jump_label_key paravirt_steal_enabled;
-struct jump_label_key paravirt_steal_rq_enabled;
+struct static_key paravirt_steal_enabled;
+struct static_key paravirt_steal_rq_enabled;
static int
ia64_native_do_steal_accounting(unsigned long *new_itm)
diff --git a/arch/mips/include/asm/jump_label.h b/arch/mips/include/asm/jump_label.h
index 1881b31..4d6d77e 100644
--- a/arch/mips/include/asm/jump_label.h
+++ b/arch/mips/include/asm/jump_label.h
@@ -20,7 +20,7 @@
#define WORD_INSN ".word"
#endif
-static __always_inline bool arch_static_branch(struct jump_label_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key)
{
asm goto("1:\tnop\n\t"
"nop\n\t"
diff --git a/arch/powerpc/include/asm/jump_label.h b/arch/powerpc/include/asm/jump_label.h
index 938986e..ae098c4 100644
--- a/arch/powerpc/include/asm/jump_label.h
+++ b/arch/powerpc/include/asm/jump_label.h
@@ -17,7 +17,7 @@
#define JUMP_ENTRY_TYPE stringify_in_c(FTR_ENTRY_LONG)
#define JUMP_LABEL_NOP_SIZE 4
-static __always_inline bool arch_static_branch(struct jump_label_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key)
{
asm goto("1:\n\t"
"nop\n\t"
diff --git a/arch/s390/include/asm/jump_label.h b/arch/s390/include/asm/jump_label.h
index 95a6cf2..6c32190 100644
--- a/arch/s390/include/asm/jump_label.h
+++ b/arch/s390/include/asm/jump_label.h
@@ -13,7 +13,7 @@
#define ASM_ALIGN ".balign 4"
#endif
-static __always_inline bool arch_static_branch(struct jump_label_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key)
{
asm goto("0: brcl 0,0\n"
".pushsection __jump_table, \"aw\"\n"
diff --git a/arch/sparc/include/asm/jump_label.h b/arch/sparc/include/asm/jump_label.h
index fc73a82..5080d16 100644
--- a/arch/sparc/include/asm/jump_label.h
+++ b/arch/sparc/include/asm/jump_label.h
@@ -7,7 +7,7 @@
#define JUMP_LABEL_NOP_SIZE 4
-static __always_inline bool arch_static_branch(struct jump_label_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key)
{
asm goto("1:\n\t"
"nop\n\t"
diff --git a/arch/x86/include/asm/jump_label.h b/arch/x86/include/asm/jump_label.h
index a32b18c..3a16c14 100644
--- a/arch/x86/include/asm/jump_label.h
+++ b/arch/x86/include/asm/jump_label.h
@@ -9,12 +9,12 @@
#define JUMP_LABEL_NOP_SIZE 5
-#define JUMP_LABEL_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t"
+#define STATIC_KEY_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t"
-static __always_inline bool arch_static_branch(struct jump_label_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key)
{
asm goto("1:"
- JUMP_LABEL_INITIAL_NOP
+ STATIC_KEY_INITIAL_NOP
".pushsection __jump_table, \"aw\" \n\t"
_ASM_ALIGN "\n\t"
_ASM_PTR "1b, %l[l_yes], %c0 \n\t"
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index a7d2db9..c0180fd 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -230,9 +230,9 @@ static inline unsigned long long paravirt_sched_clock(void)
return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock);
}
-struct jump_label_key;
-extern struct jump_label_key paravirt_steal_enabled;
-extern struct jump_label_key paravirt_steal_rq_enabled;
+struct static_key;
+extern struct static_key paravirt_steal_enabled;
+extern struct static_key paravirt_steal_rq_enabled;
static inline u64 paravirt_steal_clock(int cpu)
{
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index f0c6fd6..694d801 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -438,9 +438,9 @@ void __init kvm_guest_init(void)
static __init int activate_jump_labels(void)
{
if (has_steal_clock) {
- jump_label_inc(¶virt_steal_enabled);
+ static_key_slow_inc(¶virt_steal_enabled);
if (steal_acc)
- jump_label_inc(¶virt_steal_rq_enabled);
+ static_key_slow_inc(¶virt_steal_rq_enabled);
}
return 0;
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index d90272e..ada2f99 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -202,8 +202,8 @@ static void native_flush_tlb_single(unsigned long addr)
__native_flush_tlb_single(addr);
}
-struct jump_label_key paravirt_steal_enabled;
-struct jump_label_key paravirt_steal_rq_enabled;
+struct static_key paravirt_steal_enabled;
+struct static_key paravirt_steal_rq_enabled;
static u64 native_steal_clock(int cpu)
{
diff --git a/arch/x86/kvm/mmu_audit.c b/arch/x86/kvm/mmu_audit.c
index fe15dcc..ea7b4fd 100644
--- a/arch/x86/kvm/mmu_audit.c
+++ b/arch/x86/kvm/mmu_audit.c
@@ -234,7 +234,7 @@ static void audit_vcpu_spte(struct kvm_vcpu *vcpu)
}
static bool mmu_audit;
-static struct jump_label_key mmu_audit_key;
+static struct static_key mmu_audit_key;
static void __kvm_mmu_audit(struct kvm_vcpu *vcpu, int point)
{
@@ -250,7 +250,7 @@ static void __kvm_mmu_audit(struct kvm_vcpu *vcpu, int point)
static inline void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point)
{
- if (static_branch((&mmu_audit_key)))
+ if (static_key_false((&mmu_audit_key)))
__kvm_mmu_audit(vcpu, point);
}
@@ -259,7 +259,7 @@ static void mmu_audit_enable(void)
if (mmu_audit)
return;
- jump_label_inc(&mmu_audit_key);
+ static_key_slow_inc(&mmu_audit_key);
mmu_audit = true;
}
@@ -268,7 +268,7 @@ static void mmu_audit_disable(void)
if (!mmu_audit)
return;
- jump_label_dec(&mmu_audit_key);
+ static_key_slow_dec(&mmu_audit_key);
mmu_audit = false;
}
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index f7c6958..30d0023 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -9,15 +9,15 @@
*
* Jump labels provide an interface to generate dynamic branches using
* self-modifying code. Assuming toolchain and architecture support the result
- * of a "if (static_branch(&key))" statement is a unconditional branch (which
+ * of a "if (static_key_false(&key))" statement is a unconditional branch (which
* defaults to false - and the true block is placed out of line).
*
- * However at runtime we can change the 'static' branch target using
- * jump_label_{inc,dec}(). These function as a 'reference' count on the key
+ * However at runtime we can change the branch target using
+ * static_key_slow_{inc,dec}(). These function as a 'reference' count on the key
* object and for as long as there are references all branches referring to
* that particular key will point to the (out of line) true block.
*
- * Since this relies on modifying code the jump_label_{inc,dec}() functions
+ * Since this relies on modifying code the static_key_slow_{inc,dec}() functions
* must be considered absolute slow paths (machine wide synchronization etc.).
* OTOH, since the affected branches are unconditional their runtime overhead
* will be absolutely minimal, esp. in the default (off) case where the total
@@ -26,12 +26,26 @@
*
* When the control is directly exposed to userspace it is prudent to delay the
* decrement to avoid high frequency code modifications which can (and do)
- * cause significant performance degradation. Struct jump_label_key_deferred and
- * jump_label_dec_deferred() provide for this.
+ * cause significant performance degradation. Struct static_key_deferred and
+ * static_key_slow_dec_deferred() provide for this.
*
* Lacking toolchain and or architecture support, it falls back to a simple
* conditional branch.
- */
+ *
+ * struct static_key my_key = STATIC_KEY_INIT_TRUE;
+ *
+ * if (static_key_true(&my_key)) {
+ * }
+ *
+ * will result in the true case being in-line and starts the key with a single
+ * reference. Mixing static_key_true() and static_key_false() on the same key is not
+ * allowed.
+ *
+ * Not initializing the key (static data is initialized to 0s anyway) is the
+ * same as using STATIC_KEY_INIT_FALSE and static_key_false() is
+ * equivalent with static_branch().
+ *
+*/
#include <linux/types.h>
#include <linux/compiler.h>
@@ -39,16 +53,17 @@
#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
-struct jump_label_key {
+struct static_key {
atomic_t enabled;
+/* Set lsb bit to 1 if branch is default true, 0 ot */
struct jump_entry *entries;
#ifdef CONFIG_MODULES
- struct jump_label_mod *next;
+ struct static_key_mod *next;
#endif
};
-struct jump_label_key_deferred {
- struct jump_label_key key;
+struct static_key_deferred {
+ struct static_key key;
unsigned long timeout;
struct delayed_work work;
};
@@ -66,13 +81,34 @@ struct module;
#ifdef HAVE_JUMP_LABEL
-#ifdef CONFIG_MODULES
-#define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL, NULL}
-#else
-#define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL}
-#endif
+#define JUMP_LABEL_TRUE_BRANCH 1UL
+
+static
+inline struct jump_entry *jump_label_get_entries(struct static_key *key)
+{
+ return (struct jump_entry *)((unsigned long)key->entries
+ & ~JUMP_LABEL_TRUE_BRANCH);
+}
+
+static inline bool jump_label_get_branch_default(struct static_key *key)
+{
+ if ((unsigned long)key->entries & JUMP_LABEL_TRUE_BRANCH)
+ return true;
+ return false;
+}
+
+static __always_inline bool static_key_false(struct static_key *key)
+{
+ return arch_static_branch(key);
+}
-static __always_inline bool static_branch(struct jump_label_key *key)
+static __always_inline bool static_key_true(struct static_key *key)
+{
+ return !static_key_false(key);
+}
+
+/* Deprecated. Please use 'static_key_false() instead. */
+static __always_inline bool static_branch(struct static_key *key)
{
return arch_static_branch(key);
}
@@ -88,21 +124,24 @@ extern void arch_jump_label_transform(struct jump_entry *entry,
extern void arch_jump_label_transform_static(struct jump_entry *entry,
enum jump_label_type type);
extern int jump_label_text_reserved(void *start, void *end);
-extern void jump_label_inc(struct jump_label_key *key);
-extern void jump_label_dec(struct jump_label_key *key);
-extern void jump_label_dec_deferred(struct jump_label_key_deferred *key);
-extern bool jump_label_enabled(struct jump_label_key *key);
+extern void static_key_slow_inc(struct static_key *key);
+extern void static_key_slow_dec(struct static_key *key);
+extern void static_key_slow_dec_deferred(struct static_key_deferred *key);
+extern bool static_key_true(struct static_key *key);
extern void jump_label_apply_nops(struct module *mod);
-extern void jump_label_rate_limit(struct jump_label_key_deferred *key,
- unsigned long rl);
+extern void
+jump_label_rate_limit(struct static_key_deferred *key, unsigned long rl);
+
+#define STATIC_KEY_INIT_TRUE ((struct static_key) \
+ { .enabled = ATOMIC_INIT(1), .entries = (void *)1 })
+#define STATIC_KEY_INIT_FALSE ((struct static_key) \
+ { .enabled = ATOMIC_INIT(0), .entries = (void *)0 })
#else /* !HAVE_JUMP_LABEL */
#include <linux/atomic.h>
-#define JUMP_LABEL_INIT {ATOMIC_INIT(0)}
-
-struct jump_label_key {
+struct static_key {
atomic_t enabled;
};
@@ -110,30 +149,45 @@ static __always_inline void jump_label_init(void)
{
}
-struct jump_label_key_deferred {
- struct jump_label_key key;
+struct static_key_deferred {
+ struct static_key key;
};
-static __always_inline bool static_branch(struct jump_label_key *key)
+static __always_inline bool static_key_false(struct static_key *key)
+{
+ if (unlikely(atomic_read(&key->enabled)) > 0)
+ return true;
+ return false;
+}
+
+static __always_inline bool static_key_true(struct static_key *key)
{
- if (unlikely(atomic_read(&key->enabled)))
+ if (likely(atomic_read(&key->enabled)) > 0)
return true;
return false;
}
-static inline void jump_label_inc(struct jump_label_key *key)
+/* Deprecated. Please use 'static_key_false() instead. */
+static __always_inline bool static_branch(struct static_key *key)
+{
+ if (unlikely(atomic_read(&key->enabled)) > 0)
+ return true;
+ return false;
+}
+
+static inline void static_key_slow_inc(struct static_key *key)
{
atomic_inc(&key->enabled);
}
-static inline void jump_label_dec(struct jump_label_key *key)
+static inline void static_key_slow_dec(struct static_key *key)
{
atomic_dec(&key->enabled);
}
-static inline void jump_label_dec_deferred(struct jump_label_key_deferred *key)
+static inline void static_key_slow_dec_deferred(struct static_key_deferred *key)
{
- jump_label_dec(&key->key);
+ static_key_slow_dec(&key->key);
}
static inline int jump_label_text_reserved(void *start, void *end)
@@ -144,9 +198,9 @@ static inline int jump_label_text_reserved(void *start, void *end)
static inline void jump_label_lock(void) {}
static inline void jump_label_unlock(void) {}
-static inline bool jump_label_enabled(struct jump_label_key *key)
+static inline bool static_key_true(struct static_key *key)
{
- return !!atomic_read(&key->enabled);
+ return (atomic_read(&key->enabled) > 0);
}
static inline int jump_label_apply_nops(struct module *mod)
@@ -154,13 +208,20 @@ static inline int jump_label_apply_nops(struct module *mod)
return 0;
}
-static inline void jump_label_rate_limit(struct jump_label_key_deferred *key,
+static inline void
+jump_label_rate_limit(struct static_key_deferred *key,
unsigned long rl)
{
}
+
+#define STATIC_KEY_INIT_TRUE ((struct static_key) \
+ { .enabled = ATOMIC_INIT(1) })
+#define STATIC_KEY_INIT_FALSE ((struct static_key) \
+ { .enabled = ATOMIC_INIT(0) })
+
#endif /* HAVE_JUMP_LABEL */
-#define jump_label_key_enabled ((struct jump_label_key){ .enabled = ATOMIC_INIT(1), })
-#define jump_label_key_disabled ((struct jump_label_key){ .enabled = ATOMIC_INIT(0), })
+#define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
+#define jump_label_enabled static_key_true
#endif /* _LINUX_JUMP_LABEL_H */
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 0eac07c..7dfaae7 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -214,8 +214,8 @@ enum {
#include <linux/skbuff.h>
#ifdef CONFIG_RPS
-#include <linux/jump_label.h>
-extern struct jump_label_key rps_needed;
+#include <linux/static_key.h>
+extern struct static_key rps_needed;
#endif
struct neighbour;
diff --git a/include/linux/netfilter.h b/include/linux/netfilter.h
index b809265..29734be 100644
--- a/include/linux/netfilter.h
+++ b/include/linux/netfilter.h
@@ -163,13 +163,13 @@ extern struct ctl_path nf_net_ipv4_netfilter_sysctl_path[];
extern struct list_head nf_hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
#if defined(CONFIG_JUMP_LABEL)
-#include <linux/jump_label.h>
-extern struct jump_label_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
+#include <linux/static_key.h>
+extern struct static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
static inline bool nf_hooks_active(u_int8_t pf, unsigned int hook)
{
if (__builtin_constant_p(pf) &&
__builtin_constant_p(hook))
- return static_branch(&nf_hooks_needed[pf][hook]);
+ return static_key_false(&nf_hooks_needed[pf][hook]);
return !list_empty(&nf_hooks[pf][hook]);
}
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 412b790..0d21e6f 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -514,7 +514,7 @@ struct perf_guest_info_callbacks {
#include <linux/ftrace.h>
#include <linux/cpu.h>
#include <linux/irq_work.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
#include <linux/atomic.h>
#include <asm/local.h>
@@ -1038,7 +1038,7 @@ static inline int is_software_event(struct perf_event *event)
return event->pmu->task_ctx_nr == perf_sw_context;
}
-extern struct jump_label_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
+extern struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
extern void __perf_sw_event(u32, u64, struct pt_regs *, u64);
@@ -1066,7 +1066,7 @@ perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
{
struct pt_regs hot_regs;
- if (static_branch(&perf_swevent_enabled[event_id])) {
+ if (static_key_false(&perf_swevent_enabled[event_id])) {
if (!regs) {
perf_fetch_caller_regs(&hot_regs);
regs = &hot_regs;
@@ -1075,12 +1075,12 @@ perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
}
}
-extern struct jump_label_key_deferred perf_sched_events;
+extern struct static_key_deferred perf_sched_events;
static inline void perf_event_task_sched_in(struct task_struct *prev,
struct task_struct *task)
{
- if (static_branch(&perf_sched_events.key))
+ if (static_key_false(&perf_sched_events.key))
__perf_event_task_sched_in(prev, task);
}
@@ -1089,7 +1089,7 @@ static inline void perf_event_task_sched_out(struct task_struct *prev,
{
perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, NULL, 0);
- if (static_branch(&perf_sched_events.key))
+ if (static_key_false(&perf_sched_events.key))
__perf_event_task_sched_out(prev, next);
}
diff --git a/include/linux/static_key.h b/include/linux/static_key.h
new file mode 100644
index 0000000..27bd3f8
--- /dev/null
+++ b/include/linux/static_key.h
@@ -0,0 +1 @@
+#include <linux/jump_label.h>
diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h
index fc36da9..bd96ecd 100644
--- a/include/linux/tracepoint.h
+++ b/include/linux/tracepoint.h
@@ -17,7 +17,7 @@
#include <linux/errno.h>
#include <linux/types.h>
#include <linux/rcupdate.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
struct module;
struct tracepoint;
@@ -29,7 +29,7 @@ struct tracepoint_func {
struct tracepoint {
const char *name; /* Tracepoint name */
- struct jump_label_key key;
+ struct static_key key;
void (*regfunc)(void);
void (*unregfunc)(void);
struct tracepoint_func __rcu *funcs;
@@ -145,7 +145,7 @@ static inline void tracepoint_synchronize_unregister(void)
extern struct tracepoint __tracepoint_##name; \
static inline void trace_##name(proto) \
{ \
- if (static_branch(&__tracepoint_##name.key)) \
+ if (static_key_false(&__tracepoint_##name.key)) \
__DO_TRACE(&__tracepoint_##name, \
TP_PROTO(data_proto), \
TP_ARGS(data_args), \
@@ -188,7 +188,7 @@ static inline void tracepoint_synchronize_unregister(void)
__attribute__((section("__tracepoints_strings"))) = #name; \
struct tracepoint __tracepoint_##name \
__attribute__((section("__tracepoints"))) = \
- { __tpstrtab_##name, JUMP_LABEL_INIT, reg, unreg, NULL };\
+ { __tpstrtab_##name, STATIC_KEY_INIT_FALSE, reg, unreg, NULL };\
static struct tracepoint * const __tracepoint_ptr_##name __used \
__attribute__((section("__tracepoints_ptrs"))) = \
&__tracepoint_##name;
diff --git a/include/net/sock.h b/include/net/sock.h
index 91c1c8b..dcde2d9 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -55,7 +55,7 @@
#include <linux/uaccess.h>
#include <linux/memcontrol.h>
#include <linux/res_counter.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
#include <linux/filter.h>
#include <linux/rculist_nulls.h>
@@ -924,13 +924,13 @@ inline void sk_refcnt_debug_release(const struct sock *sk)
#endif /* SOCK_REFCNT_DEBUG */
#if defined(CONFIG_CGROUP_MEM_RES_CTLR_KMEM) && defined(CONFIG_NET)
-extern struct jump_label_key memcg_socket_limit_enabled;
+extern struct static_key memcg_socket_limit_enabled;
static inline struct cg_proto *parent_cg_proto(struct proto *proto,
struct cg_proto *cg_proto)
{
return proto->proto_cgroup(parent_mem_cgroup(cg_proto->memcg));
}
-#define mem_cgroup_sockets_enabled static_branch(&memcg_socket_limit_enabled)
+#define mem_cgroup_sockets_enabled static_key_false(&memcg_socket_limit_enabled)
#else
#define mem_cgroup_sockets_enabled 0
static inline struct cg_proto *parent_cg_proto(struct proto *proto,
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 7c3b9de..5e0f8bb 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -128,7 +128,7 @@ enum event_type_t {
* perf_sched_events : >0 events exist
* perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
*/
-struct jump_label_key_deferred perf_sched_events __read_mostly;
+struct static_key_deferred perf_sched_events __read_mostly;
static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
static atomic_t nr_mmap_events __read_mostly;
@@ -2769,7 +2769,7 @@ static void free_event(struct perf_event *event)
if (!event->parent) {
if (event->attach_state & PERF_ATTACH_TASK)
- jump_label_dec_deferred(&perf_sched_events);
+ static_key_slow_dec_deferred(&perf_sched_events);
if (event->attr.mmap || event->attr.mmap_data)
atomic_dec(&nr_mmap_events);
if (event->attr.comm)
@@ -2780,7 +2780,7 @@ static void free_event(struct perf_event *event)
put_callchain_buffers();
if (is_cgroup_event(event)) {
atomic_dec(&per_cpu(perf_cgroup_events, event->cpu));
- jump_label_dec_deferred(&perf_sched_events);
+ static_key_slow_dec_deferred(&perf_sched_events);
}
}
@@ -4982,7 +4982,7 @@ fail:
return err;
}
-struct jump_label_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
+struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
static void sw_perf_event_destroy(struct perf_event *event)
{
@@ -4990,7 +4990,7 @@ static void sw_perf_event_destroy(struct perf_event *event)
WARN_ON(event->parent);
- jump_label_dec(&perf_swevent_enabled[event_id]);
+ static_key_slow_dec(&perf_swevent_enabled[event_id]);
swevent_hlist_put(event);
}
@@ -5020,7 +5020,7 @@ static int perf_swevent_init(struct perf_event *event)
if (err)
return err;
- jump_label_inc(&perf_swevent_enabled[event_id]);
+ static_key_slow_inc(&perf_swevent_enabled[event_id]);
event->destroy = sw_perf_event_destroy;
}
@@ -5843,7 +5843,7 @@ done:
if (!event->parent) {
if (event->attach_state & PERF_ATTACH_TASK)
- jump_label_inc(&perf_sched_events.key);
+ static_key_slow_inc(&perf_sched_events.key);
if (event->attr.mmap || event->attr.mmap_data)
atomic_inc(&nr_mmap_events);
if (event->attr.comm)
@@ -6081,7 +6081,7 @@ SYSCALL_DEFINE5(perf_event_open,
* - that may need work on context switch
*/
atomic_inc(&per_cpu(perf_cgroup_events, event->cpu));
- jump_label_inc(&perf_sched_events.key);
+ static_key_slow_inc(&perf_sched_events.key);
}
/*
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 543782e..1cc32e3 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -12,7 +12,7 @@
#include <linux/slab.h>
#include <linux/sort.h>
#include <linux/err.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
#ifdef HAVE_JUMP_LABEL
@@ -29,10 +29,11 @@ void jump_label_unlock(void)
mutex_unlock(&jump_label_mutex);
}
-bool jump_label_enabled(struct jump_label_key *key)
+bool static_key_true(struct static_key *key)
{
- return !!atomic_read(&key->enabled);
+ return (atomic_read(&key->enabled) > 0);
}
+EXPORT_SYMBOL_GPL(static_key_true);
static int jump_label_cmp(const void *a, const void *b)
{
@@ -58,22 +59,26 @@ jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop)
sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL);
}
-static void jump_label_update(struct jump_label_key *key, int enable);
+static void jump_label_update(struct static_key *key, int enable);
-void jump_label_inc(struct jump_label_key *key)
+void static_key_slow_inc(struct static_key *key)
{
if (atomic_inc_not_zero(&key->enabled))
return;
jump_label_lock();
- if (atomic_read(&key->enabled) == 0)
- jump_label_update(key, JUMP_LABEL_ENABLE);
+ if (atomic_read(&key->enabled) == 0) {
+ if (!jump_label_get_branch_default(key))
+ jump_label_update(key, JUMP_LABEL_ENABLE);
+ else
+ jump_label_update(key, JUMP_LABEL_DISABLE);
+ }
atomic_inc(&key->enabled);
jump_label_unlock();
}
-EXPORT_SYMBOL_GPL(jump_label_inc);
+EXPORT_SYMBOL_GPL(static_key_slow_inc);
-static void __jump_label_dec(struct jump_label_key *key,
+static void __static_key_slow_dec(struct static_key *key,
unsigned long rate_limit, struct delayed_work *work)
{
if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) {
@@ -85,32 +90,35 @@ static void __jump_label_dec(struct jump_label_key *key,
if (rate_limit) {
atomic_inc(&key->enabled);
schedule_delayed_work(work, rate_limit);
- } else
- jump_label_update(key, JUMP_LABEL_DISABLE);
-
+ } else {
+ if (!jump_label_get_branch_default(key))
+ jump_label_update(key, JUMP_LABEL_DISABLE);
+ else
+ jump_label_update(key, JUMP_LABEL_ENABLE);
+ }
jump_label_unlock();
}
-EXPORT_SYMBOL_GPL(jump_label_dec);
static void jump_label_update_timeout(struct work_struct *work)
{
- struct jump_label_key_deferred *key =
- container_of(work, struct jump_label_key_deferred, work.work);
- __jump_label_dec(&key->key, 0, NULL);
+ struct static_key_deferred *key =
+ container_of(work, struct static_key_deferred, work.work);
+ __static_key_slow_dec(&key->key, 0, NULL);
}
-void jump_label_dec(struct jump_label_key *key)
+void static_key_slow_dec(struct static_key *key)
{
- __jump_label_dec(key, 0, NULL);
+ __static_key_slow_dec(key, 0, NULL);
}
+EXPORT_SYMBOL_GPL(static_key_slow_dec);
-void jump_label_dec_deferred(struct jump_label_key_deferred *key)
+void static_key_slow_dec_deferred(struct static_key_deferred *key)
{
- __jump_label_dec(&key->key, key->timeout, &key->work);
+ __static_key_slow_dec(&key->key, key->timeout, &key->work);
}
+EXPORT_SYMBOL_GPL(static_key_slow_dec_deferred);
-
-void jump_label_rate_limit(struct jump_label_key_deferred *key,
+void jump_label_rate_limit(struct static_key_deferred *key,
unsigned long rl)
{
key->timeout = rl;
@@ -153,7 +161,7 @@ void __weak __init_or_module arch_jump_label_transform_static(struct jump_entry
arch_jump_label_transform(entry, type);
}
-static void __jump_label_update(struct jump_label_key *key,
+static void __jump_label_update(struct static_key *key,
struct jump_entry *entry,
struct jump_entry *stop, int enable)
{
@@ -170,27 +178,40 @@ static void __jump_label_update(struct jump_label_key *key,
}
}
+static enum jump_label_type jump_label_type(struct static_key *key)
+{
+ bool true_branch = jump_label_get_branch_default(key);
+ bool state = static_key_true(key);
+
+ if ((!true_branch && state) || (true_branch && !state))
+ return JUMP_LABEL_ENABLE;
+
+ return JUMP_LABEL_DISABLE;
+}
+
void __init jump_label_init(void)
{
struct jump_entry *iter_start = __start___jump_table;
struct jump_entry *iter_stop = __stop___jump_table;
- struct jump_label_key *key = NULL;
+ struct static_key *key = NULL;
struct jump_entry *iter;
jump_label_lock();
jump_label_sort_entries(iter_start, iter_stop);
for (iter = iter_start; iter < iter_stop; iter++) {
- struct jump_label_key *iterk;
+ struct static_key *iterk;
- iterk = (struct jump_label_key *)(unsigned long)iter->key;
- arch_jump_label_transform_static(iter, jump_label_enabled(iterk) ?
- JUMP_LABEL_ENABLE : JUMP_LABEL_DISABLE);
+ iterk = (struct static_key *)(unsigned long)iter->key;
+ arch_jump_label_transform_static(iter, jump_label_type(iterk));
if (iterk == key)
continue;
key = iterk;
- key->entries = iter;
+ /*
+ * Set key->entries to iter, but preserve JUMP_LABEL_TRUE_BRANCH.
+ */
+ *((unsigned long *)&key->entries) += (unsigned long)iter;
#ifdef CONFIG_MODULES
key->next = NULL;
#endif
@@ -200,8 +221,8 @@ void __init jump_label_init(void)
#ifdef CONFIG_MODULES
-struct jump_label_mod {
- struct jump_label_mod *next;
+struct static_key_mod {
+ struct static_key_mod *next;
struct jump_entry *entries;
struct module *mod;
};
@@ -221,9 +242,9 @@ static int __jump_label_mod_text_reserved(void *start, void *end)
start, end);
}
-static void __jump_label_mod_update(struct jump_label_key *key, int enable)
+static void __jump_label_mod_update(struct static_key *key, int enable)
{
- struct jump_label_mod *mod = key->next;
+ struct static_key_mod *mod = key->next;
while (mod) {
struct module *m = mod->mod;
@@ -254,11 +275,7 @@ void jump_label_apply_nops(struct module *mod)
return;
for (iter = iter_start; iter < iter_stop; iter++) {
- struct jump_label_key *iterk;
-
- iterk = (struct jump_label_key *)(unsigned long)iter->key;
- arch_jump_label_transform_static(iter, jump_label_enabled(iterk) ?
- JUMP_LABEL_ENABLE : JUMP_LABEL_DISABLE);
+ arch_jump_label_transform_static(iter, JUMP_LABEL_DISABLE);
}
}
@@ -267,8 +284,8 @@ static int jump_label_add_module(struct module *mod)
struct jump_entry *iter_start = mod->jump_entries;
struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
struct jump_entry *iter;
- struct jump_label_key *key = NULL;
- struct jump_label_mod *jlm;
+ struct static_key *key = NULL;
+ struct static_key_mod *jlm;
/* if the module doesn't have jump label entries, just return */
if (iter_start == iter_stop)
@@ -277,28 +294,30 @@ static int jump_label_add_module(struct module *mod)
jump_label_sort_entries(iter_start, iter_stop);
for (iter = iter_start; iter < iter_stop; iter++) {
- if (iter->key == (jump_label_t)(unsigned long)key)
- continue;
+ struct static_key *iterk;
- key = (struct jump_label_key *)(unsigned long)iter->key;
+ iterk = (struct static_key *)(unsigned long)iter->key;
+ if (iterk == key)
+ continue;
+ key = iterk;
if (__module_address(iter->key) == mod) {
- atomic_set(&key->enabled, 0);
- key->entries = iter;
+ /*
+ * Set key->entries to iter, but preserve JUMP_LABEL_TRUE_BRANCH.
+ */
+ *((unsigned long *)&key->entries) += (unsigned long)iter;
key->next = NULL;
continue;
}
-
- jlm = kzalloc(sizeof(struct jump_label_mod), GFP_KERNEL);
+ jlm = kzalloc(sizeof(struct static_key_mod), GFP_KERNEL);
if (!jlm)
return -ENOMEM;
-
jlm->mod = mod;
jlm->entries = iter;
jlm->next = key->next;
key->next = jlm;
- if (jump_label_enabled(key))
+ if (jump_label_type(key) == JUMP_LABEL_ENABLE)
__jump_label_update(key, iter, iter_stop, JUMP_LABEL_ENABLE);
}
@@ -310,14 +329,14 @@ static void jump_label_del_module(struct module *mod)
struct jump_entry *iter_start = mod->jump_entries;
struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
struct jump_entry *iter;
- struct jump_label_key *key = NULL;
- struct jump_label_mod *jlm, **prev;
+ struct static_key *key = NULL;
+ struct static_key_mod *jlm, **prev;
for (iter = iter_start; iter < iter_stop; iter++) {
if (iter->key == (jump_label_t)(unsigned long)key)
continue;
- key = (struct jump_label_key *)(unsigned long)iter->key;
+ key = (struct static_key *)(unsigned long)iter->key;
if (__module_address(iter->key) == mod)
continue;
@@ -419,9 +438,10 @@ int jump_label_text_reserved(void *start, void *end)
return ret;
}
-static void jump_label_update(struct jump_label_key *key, int enable)
+static void jump_label_update(struct static_key *key, int enable)
{
- struct jump_entry *entry = key->entries, *stop = __stop___jump_table;
+ struct jump_entry *stop = __stop___jump_table;
+ struct jump_entry *entry = jump_label_get_entries(key);
#ifdef CONFIG_MODULES
struct module *mod = __module_address((unsigned long)key);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 5255c9d..05fb5df 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -162,13 +162,13 @@ static int sched_feat_show(struct seq_file *m, void *v)
#ifdef HAVE_JUMP_LABEL
-#define jump_label_key__true jump_label_key_enabled
-#define jump_label_key__false jump_label_key_disabled
+#define jump_label_key__true STATIC_KEY_INIT_TRUE
+#define jump_label_key__false STATIC_KEY_INIT_FALSE
#define SCHED_FEAT(name, enabled) \
jump_label_key__##enabled ,
-struct jump_label_key sched_feat_keys[__SCHED_FEAT_NR] = {
+struct static_key sched_feat_keys[__SCHED_FEAT_NR] = {
#include "features.h"
};
@@ -176,14 +176,14 @@ struct jump_label_key sched_feat_keys[__SCHED_FEAT_NR] = {
static void sched_feat_disable(int i)
{
- if (jump_label_enabled(&sched_feat_keys[i]))
- jump_label_dec(&sched_feat_keys[i]);
+ if (static_key_true(&sched_feat_keys[i]))
+ static_key_slow_dec(&sched_feat_keys[i]);
}
static void sched_feat_enable(int i)
{
- if (!jump_label_enabled(&sched_feat_keys[i]))
- jump_label_inc(&sched_feat_keys[i]);
+ if (!static_key_true(&sched_feat_keys[i]))
+ static_key_slow_inc(&sched_feat_keys[i]);
}
#else
static void sched_feat_disable(int i) { };
@@ -894,7 +894,7 @@ static void update_rq_clock_task(struct rq *rq, s64 delta)
delta -= irq_delta;
#endif
#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
- if (static_branch((¶virt_steal_rq_enabled))) {
+ if (static_key_false((¶virt_steal_rq_enabled))) {
u64 st;
steal = paravirt_steal_clock(cpu_of(rq));
@@ -2756,7 +2756,7 @@ void account_idle_time(cputime_t cputime)
static __always_inline bool steal_account_process_tick(void)
{
#ifdef CONFIG_PARAVIRT
- if (static_branch(¶virt_steal_enabled)) {
+ if (static_key_false(¶virt_steal_enabled)) {
u64 steal, st = 0;
steal = paravirt_steal_clock(smp_processor_id());
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 7c6414f..423547a 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1399,20 +1399,20 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued)
#ifdef CONFIG_CFS_BANDWIDTH
#ifdef HAVE_JUMP_LABEL
-static struct jump_label_key __cfs_bandwidth_used;
+static struct static_key __cfs_bandwidth_used;
static inline bool cfs_bandwidth_used(void)
{
- return static_branch(&__cfs_bandwidth_used);
+ return static_key_false(&__cfs_bandwidth_used);
}
void account_cfs_bandwidth_used(int enabled, int was_enabled)
{
/* only need to count groups transitioning between enabled/!enabled */
if (enabled && !was_enabled)
- jump_label_inc(&__cfs_bandwidth_used);
+ static_key_slow_inc(&__cfs_bandwidth_used);
else if (!enabled && was_enabled)
- jump_label_dec(&__cfs_bandwidth_used);
+ static_key_slow_dec(&__cfs_bandwidth_used);
}
#else /* HAVE_JUMP_LABEL */
static bool cfs_bandwidth_used(void)
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 98c0c26..b4cd6d8 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -611,7 +611,7 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
* Tunables that become constants when CONFIG_SCHED_DEBUG is off:
*/
#ifdef CONFIG_SCHED_DEBUG
-# include <linux/jump_label.h>
+# include <linux/static_key.h>
# define const_debug __read_mostly
#else
# define const_debug const
@@ -630,18 +630,18 @@ enum {
#undef SCHED_FEAT
#if defined(CONFIG_SCHED_DEBUG) && defined(HAVE_JUMP_LABEL)
-static __always_inline bool static_branch__true(struct jump_label_key *key)
+static __always_inline bool static_branch__true(struct static_key *key)
{
- return likely(static_branch(key)); /* Not out of line branch. */
+ return static_key_true(key); /* Not out of line branch. */
}
-static __always_inline bool static_branch__false(struct jump_label_key *key)
+static __always_inline bool static_branch__false(struct static_key *key)
{
- return unlikely(static_branch(key)); /* Out of line branch. */
+ return static_key_false(key); /* Out of line branch. */
}
#define SCHED_FEAT(name, enabled) \
-static __always_inline bool static_branch_##name(struct jump_label_key *key) \
+static __always_inline bool static_branch_##name(struct static_key *key) \
{ \
return static_branch__##enabled(key); \
}
@@ -650,7 +650,7 @@ static __always_inline bool static_branch_##name(struct jump_label_key *key) \
#undef SCHED_FEAT
-extern struct jump_label_key sched_feat_keys[__SCHED_FEAT_NR];
+extern struct static_key sched_feat_keys[__SCHED_FEAT_NR];
#define sched_feat(x) (static_branch_##x(&sched_feat_keys[__SCHED_FEAT_##x]))
#else /* !(SCHED_DEBUG && HAVE_JUMP_LABEL) */
#define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x))
diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c
index f1539de..5e37c26 100644
--- a/kernel/tracepoint.c
+++ b/kernel/tracepoint.c
@@ -25,7 +25,7 @@
#include <linux/err.h>
#include <linux/slab.h>
#include <linux/sched.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
extern struct tracepoint * const __start___tracepoints_ptrs[];
extern struct tracepoint * const __stop___tracepoints_ptrs[];
@@ -256,9 +256,9 @@ static void set_tracepoint(struct tracepoint_entry **entry,
{
WARN_ON(strcmp((*entry)->name, elem->name) != 0);
- if (elem->regfunc && !jump_label_enabled(&elem->key) && active)
+ if (elem->regfunc && !static_key_true(&elem->key) && active)
elem->regfunc();
- else if (elem->unregfunc && jump_label_enabled(&elem->key) && !active)
+ else if (elem->unregfunc && static_key_true(&elem->key) && !active)
elem->unregfunc();
/*
@@ -269,10 +269,10 @@ static void set_tracepoint(struct tracepoint_entry **entry,
* is used.
*/
rcu_assign_pointer(elem->funcs, (*entry)->funcs);
- if (active && !jump_label_enabled(&elem->key))
- jump_label_inc(&elem->key);
- else if (!active && jump_label_enabled(&elem->key))
- jump_label_dec(&elem->key);
+ if (active && !static_key_true(&elem->key))
+ static_key_slow_inc(&elem->key);
+ else if (!active && static_key_true(&elem->key))
+ static_key_slow_dec(&elem->key);
}
/*
@@ -283,11 +283,11 @@ static void set_tracepoint(struct tracepoint_entry **entry,
*/
static void disable_tracepoint(struct tracepoint *elem)
{
- if (elem->unregfunc && jump_label_enabled(&elem->key))
+ if (elem->unregfunc && static_key_true(&elem->key))
elem->unregfunc();
- if (jump_label_enabled(&elem->key))
- jump_label_dec(&elem->key);
+ if (static_key_true(&elem->key))
+ static_key_slow_dec(&elem->key);
rcu_assign_pointer(elem->funcs, NULL);
}
diff --git a/net/core/dev.c b/net/core/dev.c
index 115dee1..da7ce7f 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -134,7 +134,7 @@
#include <linux/inetdevice.h>
#include <linux/cpu_rmap.h>
#include <linux/net_tstamp.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
#include <net/flow_keys.h>
#include "net-sysfs.h"
@@ -1441,11 +1441,11 @@ int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
}
EXPORT_SYMBOL(call_netdevice_notifiers);
-static struct jump_label_key netstamp_needed __read_mostly;
+static struct static_key netstamp_needed __read_mostly;
#ifdef HAVE_JUMP_LABEL
-/* We are not allowed to call jump_label_dec() from irq context
+/* We are not allowed to call static_key_slow_dec() from irq context
* If net_disable_timestamp() is called from irq context, defer the
- * jump_label_dec() calls.
+ * static_key_slow_dec() calls.
*/
static atomic_t netstamp_needed_deferred;
#endif
@@ -1457,12 +1457,12 @@ void net_enable_timestamp(void)
if (deferred) {
while (--deferred)
- jump_label_dec(&netstamp_needed);
+ static_key_slow_dec(&netstamp_needed);
return;
}
#endif
WARN_ON(in_interrupt());
- jump_label_inc(&netstamp_needed);
+ static_key_slow_inc(&netstamp_needed);
}
EXPORT_SYMBOL(net_enable_timestamp);
@@ -1474,19 +1474,19 @@ void net_disable_timestamp(void)
return;
}
#endif
- jump_label_dec(&netstamp_needed);
+ static_key_slow_dec(&netstamp_needed);
}
EXPORT_SYMBOL(net_disable_timestamp);
static inline void net_timestamp_set(struct sk_buff *skb)
{
skb->tstamp.tv64 = 0;
- if (static_branch(&netstamp_needed))
+ if (static_key_false(&netstamp_needed))
__net_timestamp(skb);
}
#define net_timestamp_check(COND, SKB) \
- if (static_branch(&netstamp_needed)) { \
+ if (static_key_false(&netstamp_needed)) { \
if ((COND) && !(SKB)->tstamp.tv64) \
__net_timestamp(SKB); \
} \
@@ -2660,7 +2660,7 @@ EXPORT_SYMBOL(__skb_get_rxhash);
struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
EXPORT_SYMBOL(rps_sock_flow_table);
-struct jump_label_key rps_needed __read_mostly;
+struct static_key rps_needed __read_mostly;
static struct rps_dev_flow *
set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
@@ -2945,7 +2945,7 @@ int netif_rx(struct sk_buff *skb)
trace_netif_rx(skb);
#ifdef CONFIG_RPS
- if (static_branch(&rps_needed)) {
+ if (static_key_false(&rps_needed)) {
struct rps_dev_flow voidflow, *rflow = &voidflow;
int cpu;
@@ -3309,7 +3309,7 @@ int netif_receive_skb(struct sk_buff *skb)
return NET_RX_SUCCESS;
#ifdef CONFIG_RPS
- if (static_branch(&rps_needed)) {
+ if (static_key_false(&rps_needed)) {
struct rps_dev_flow voidflow, *rflow = &voidflow;
int cpu, ret;
diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c
index a1727cd..4955862 100644
--- a/net/core/net-sysfs.c
+++ b/net/core/net-sysfs.c
@@ -608,10 +608,10 @@ static ssize_t store_rps_map(struct netdev_rx_queue *queue,
spin_unlock(&rps_map_lock);
if (map)
- jump_label_inc(&rps_needed);
+ static_key_slow_inc(&rps_needed);
if (old_map) {
kfree_rcu(old_map, rcu);
- jump_label_dec(&rps_needed);
+ static_key_slow_dec(&rps_needed);
}
free_cpumask_var(mask);
return len;
diff --git a/net/core/sock.c b/net/core/sock.c
index 3e81fd2..3a4e581 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -111,7 +111,7 @@
#include <linux/init.h>
#include <linux/highmem.h>
#include <linux/user_namespace.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
#include <linux/memcontrol.h>
#include <asm/uaccess.h>
@@ -184,7 +184,7 @@ void mem_cgroup_sockets_destroy(struct cgroup *cgrp, struct cgroup_subsys *ss)
static struct lock_class_key af_family_keys[AF_MAX];
static struct lock_class_key af_family_slock_keys[AF_MAX];
-struct jump_label_key memcg_socket_limit_enabled;
+struct static_key memcg_socket_limit_enabled;
EXPORT_SYMBOL(memcg_socket_limit_enabled);
/*
diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c
index d05559d..0c28508 100644
--- a/net/core/sysctl_net_core.c
+++ b/net/core/sysctl_net_core.c
@@ -69,9 +69,9 @@ static int rps_sock_flow_sysctl(ctl_table *table, int write,
if (sock_table != orig_sock_table) {
rcu_assign_pointer(rps_sock_flow_table, sock_table);
if (sock_table)
- jump_label_inc(&rps_needed);
+ static_key_slow_inc(&rps_needed);
if (orig_sock_table) {
- jump_label_dec(&rps_needed);
+ static_key_slow_dec(&rps_needed);
synchronize_rcu();
vfree(orig_sock_table);
}
diff --git a/net/ipv4/tcp_memcontrol.c b/net/ipv4/tcp_memcontrol.c
index 4997878..602fb30 100644
--- a/net/ipv4/tcp_memcontrol.c
+++ b/net/ipv4/tcp_memcontrol.c
@@ -111,7 +111,7 @@ void tcp_destroy_cgroup(struct cgroup *cgrp, struct cgroup_subsys *ss)
val = res_counter_read_u64(&tcp->tcp_memory_allocated, RES_LIMIT);
if (val != RESOURCE_MAX)
- jump_label_dec(&memcg_socket_limit_enabled);
+ static_key_slow_dec(&memcg_socket_limit_enabled);
}
EXPORT_SYMBOL(tcp_destroy_cgroup);
@@ -143,9 +143,9 @@ static int tcp_update_limit(struct mem_cgroup *memcg, u64 val)
net->ipv4.sysctl_tcp_mem[i]);
if (val == RESOURCE_MAX && old_lim != RESOURCE_MAX)
- jump_label_dec(&memcg_socket_limit_enabled);
+ static_key_slow_dec(&memcg_socket_limit_enabled);
else if (old_lim == RESOURCE_MAX && val != RESOURCE_MAX)
- jump_label_inc(&memcg_socket_limit_enabled);
+ static_key_slow_inc(&memcg_socket_limit_enabled);
return 0;
}
diff --git a/net/netfilter/core.c b/net/netfilter/core.c
index b4e8ff0..e1b7e05 100644
--- a/net/netfilter/core.c
+++ b/net/netfilter/core.c
@@ -56,7 +56,7 @@ struct list_head nf_hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS] __read_mostly;
EXPORT_SYMBOL(nf_hooks);
#if defined(CONFIG_JUMP_LABEL)
-struct jump_label_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
+struct static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
EXPORT_SYMBOL(nf_hooks_needed);
#endif
@@ -77,7 +77,7 @@ int nf_register_hook(struct nf_hook_ops *reg)
list_add_rcu(®->list, elem->list.prev);
mutex_unlock(&nf_hook_mutex);
#if defined(CONFIG_JUMP_LABEL)
- jump_label_inc(&nf_hooks_needed[reg->pf][reg->hooknum]);
+ static_key_slow_inc(&nf_hooks_needed[reg->pf][reg->hooknum]);
#endif
return 0;
}
@@ -89,7 +89,7 @@ void nf_unregister_hook(struct nf_hook_ops *reg)
list_del_rcu(®->list);
mutex_unlock(&nf_hook_mutex);
#if defined(CONFIG_JUMP_LABEL)
- jump_label_dec(&nf_hooks_needed[reg->pf][reg->hooknum]);
+ static_key_slow_dec(&nf_hooks_needed[reg->pf][reg->hooknum]);
#endif
synchronize_net();
}
Here's the reworked documentation patch from Jason. I extended
it with a 'Summary' section and propagated the static key
concept into it where appropriate and fixed a few typos.
Thanks,
Ingo
----------->
>From 4f99cc24ced69ee3b126c5d1abd85d74a6036251 Mon Sep 17 00:00:00 2001
From: Jason Baron <[email protected]>
Date: Tue, 21 Feb 2012 15:03:30 -0500
Subject: [PATCH] static keys: Add docs better explaining the whole 'struct
static_key' mechanism
Add better documentation for static keys.
Signed-off-by: Jason Baron <[email protected]>
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Link: http://lkml.kernel.org/r/52570e566e5f1914f27b67e4eafb5781b8f9f9db.1329851692.git.jbaron@redhat.com
[ Added a 'Summary' section and rewrote it to explain static keys ]
Signed-off-by: Ingo Molnar <[email protected]>
---
Documentation/static-keys.txt | 286 +++++++++++++++++++++++++++++++++++++++++
1 files changed, 286 insertions(+), 0 deletions(-)
diff --git a/Documentation/static-keys.txt b/Documentation/static-keys.txt
new file mode 100644
index 0000000..d93f3c0
--- /dev/null
+++ b/Documentation/static-keys.txt
@@ -0,0 +1,286 @@
+ Static Keys
+ -----------
+
+By: Jason Baron <[email protected]>
+
+0) Abstract
+
+Static keys allows the inclusion of seldom used features in
+performance-sensitive fast-path kernel code, via a GCC feature and a code
+patching technique. A quick example:
+
+ struct static_key key = STATIC_KEY_INIT_FALSE;
+
+ ...
+
+ if (static_key_false(&key))
+ do unlikely code
+ else
+ do likely code
+
+ ...
+ static_key_slow_inc();
+ ...
+ static_key_slow_inc();
+ ...
+
+The static_key_false() branch will be generated into the code with as little
+impact to the likely code path as possible.
+
+
+1) Motivation
+
+
+Currently, tracepoints are implemented using a conditional branch. The
+conditional check requires checking a global variable for each tracepoint.
+Although the overhead of this check is small, it increases when the memory
+cache comes under pressure (memory cache lines for these global variables may
+be shared with other memory accesses). As we increase the number of tracepoints
+in the kernel this overhead may become more of an issue. In addition,
+tracepoints are often dormant (disabled) and provide no direct kernel
+functionality. Thus, it is highly desirable to reduce their impact as much as
+possible. Although tracepoints are the original motivation for this work, other
+kernel code paths should be able to make use of the static keys facility.
+
+
+2) Solution
+
+
+gcc (v4.5) adds a new 'asm goto' statement that allows branching to a label:
+
+http://gcc.gnu.org/ml/gcc-patches/2009-07/msg01556.html
+
+Using the 'asm goto', we can create branches that are either taken or not taken
+by default, without the need to check memory. Then, at run-time, we can patch
+the branch site to change the branch direction.
+
+For example, if we have a simple branch that is disabled by default:
+
+ if (static_key_false(&key))
+ printk("I am the true branch\n");
+
+Thus, by default the 'printk' will not be emitted. And the code generated will
+consist of a single atomic 'no-op' instruction (5 bytes on x86), in the
+straight-line code path. When the branch is 'flipped', we will patch the
+'no-op' in the straight-line codepath with a 'jump' instruction to the
+out-of-line true branch. Thus, changing branch direction is expensive but
+branch selection is basically 'free'. That is the basic tradeoff of this
+optimization.
+
+This lowlevel patching mechanism is called 'jump label patching', and it gives
+the basis for the static keys facility.
+
+3) Static key label API, usage and examples:
+
+
+In order to make use of this optimization you must first define a key:
+
+ struct static_key key;
+
+Which is initialized as:
+
+ struct static_key key = STATIC_KEY_INIT_TRUE;
+
+or:
+
+ struct static_key key = STATIC_KEY_INIT_FALSE;
+
+If the key is not initialized, it is default false. The 'struct static_key',
+must be a 'global'. That is, it can't be allocated on the stack or dynamically
+allocated at run-time.
+
+The key is then used in code as:
+
+ if (static_key_false(&key))
+ do unlikely code
+ else
+ do likely code
+
+Or:
+
+ if (static_key_true(&key))
+ do likely code
+ else
+ do unlikely code
+
+A key that is initialized via 'STATIC_KEY_INIT_FALSE', must be used in a
+'static_key_false()' construct. Likewise, a key initialized via
+'STATIC_KEY_INIT_TRUE' must be used in a 'static_key_true()' construct. A
+single key can be used in many branches, but all the branches must match the
+way that the key has been initialized.
+
+The branch(es) can then be switched via:
+
+ static_key_slow_inc(&key);
+ ...
+ static_key_slow_dec(&key);
+
+Thus, 'static_key_slow_inc()' means 'make the branch true', and
+'static_key_slow_dec()' means 'make the the branch false' with appropriate
+reference counting. For example, if the key is initialized true, a
+static_key_slow_dec(), will switch the branch to false. And a subsequent
+static_key_slow_inc(), will change the branch back to true. Likewise, if the
+key is initialized false, a 'static_key_slow_inc()', will change the branch to
+true. And then a 'static_key_slow_dec()', will again make the branch false.
+
+An example usage in the kernel is the implementation of tracepoints:
+
+ static inline void trace_##name(proto) \
+ { \
+ if (static_key_false(&__tracepoint_##name.key)) \
+ __DO_TRACE(&__tracepoint_##name, \
+ TP_PROTO(data_proto), \
+ TP_ARGS(data_args), \
+ TP_CONDITION(cond)); \
+ }
+
+Tracepoints are disabled by default, and can be placed in performance critical
+pieces of the kernel. Thus, by using a static key, the tracepoints can have
+absolutely minimal impact when not in use.
+
+
+4) Architecture level code patching interface, 'jump labels'
+
+
+There are a few functions and macros that architectures must implement in order
+to take advantage of this optimization. If there is no architecture support, we
+simply fall back to a traditional, load, test, and jump sequence.
+
+* select HAVE_ARCH_JUMP_LABEL, see: arch/x86/Kconfig
+
+* #define JUMP_LABEL_NOP_SIZE, see: arch/x86/include/asm/jump_label.h
+
+* __always_inline bool arch_static_branch(struct static_key *key), see:
+ arch/x86/include/asm/jump_label.h
+
+* void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type),
+ see: arch/x86/kernel/jump_label.c
+
+* __init_or_module void arch_jump_label_transform_static(struct jump_entry *entry, enum jump_label_type type),
+ see: arch/x86/kernel/jump_label.c
+
+
+* struct jump_entry, see: arch/x86/include/asm/jump_label.h
+
+
+5) Static keys / jump label analysis, results (x86_64):
+
+
+As an example, let's add the following branch to 'getppid()', such that the
+system call now looks like:
+
+SYSCALL_DEFINE0(getppid)
+{
+ int pid;
+
++ if (static_key_false(&key))
++ printk("I am the true branch\n");
+
+ rcu_read_lock();
+ pid = task_tgid_vnr(rcu_dereference(current->real_parent));
+ rcu_read_unlock();
+
+ return pid;
+}
+
+The resulting instructions with jump labels generated by GCC is:
+
+ffffffff81044290 <sys_getppid>:
+ffffffff81044290: 55 push %rbp
+ffffffff81044291: 48 89 e5 mov %rsp,%rbp
+ffffffff81044294: e9 00 00 00 00 jmpq ffffffff81044299 <sys_getppid+0x9>
+ffffffff81044299: 65 48 8b 04 25 c0 b6 mov %gs:0xb6c0,%rax
+ffffffff810442a0: 00 00
+ffffffff810442a2: 48 8b 80 80 02 00 00 mov 0x280(%rax),%rax
+ffffffff810442a9: 48 8b 80 b0 02 00 00 mov 0x2b0(%rax),%rax
+ffffffff810442b0: 48 8b b8 e8 02 00 00 mov 0x2e8(%rax),%rdi
+ffffffff810442b7: e8 f4 d9 00 00 callq ffffffff81051cb0 <pid_vnr>
+ffffffff810442bc: 5d pop %rbp
+ffffffff810442bd: 48 98 cltq
+ffffffff810442bf: c3 retq
+ffffffff810442c0: 48 c7 c7 e3 54 98 81 mov $0xffffffff819854e3,%rdi
+ffffffff810442c7: 31 c0 xor %eax,%eax
+ffffffff810442c9: e8 71 13 6d 00 callq ffffffff8171563f <printk>
+ffffffff810442ce: eb c9 jmp ffffffff81044299 <sys_getppid+0x9>
+
+Without the jump label optimization it looks like:
+
+ffffffff810441f0 <sys_getppid>:
+ffffffff810441f0: 8b 05 8a 52 d8 00 mov 0xd8528a(%rip),%eax # ffffffff81dc9480 <key>
+ffffffff810441f6: 55 push %rbp
+ffffffff810441f7: 48 89 e5 mov %rsp,%rbp
+ffffffff810441fa: 85 c0 test %eax,%eax
+ffffffff810441fc: 75 27 jne ffffffff81044225 <sys_getppid+0x35>
+ffffffff810441fe: 65 48 8b 04 25 c0 b6 mov %gs:0xb6c0,%rax
+ffffffff81044205: 00 00
+ffffffff81044207: 48 8b 80 80 02 00 00 mov 0x280(%rax),%rax
+ffffffff8104420e: 48 8b 80 b0 02 00 00 mov 0x2b0(%rax),%rax
+ffffffff81044215: 48 8b b8 e8 02 00 00 mov 0x2e8(%rax),%rdi
+ffffffff8104421c: e8 2f da 00 00 callq ffffffff81051c50 <pid_vnr>
+ffffffff81044221: 5d pop %rbp
+ffffffff81044222: 48 98 cltq
+ffffffff81044224: c3 retq
+ffffffff81044225: 48 c7 c7 13 53 98 81 mov $0xffffffff81985313,%rdi
+ffffffff8104422c: 31 c0 xor %eax,%eax
+ffffffff8104422e: e8 60 0f 6d 00 callq ffffffff81715193 <printk>
+ffffffff81044233: eb c9 jmp ffffffff810441fe <sys_getppid+0xe>
+ffffffff81044235: 66 66 2e 0f 1f 84 00 data32 nopw %cs:0x0(%rax,%rax,1)
+ffffffff8104423c: 00 00 00 00
+
+Thus, the disable jump label case adds a 'mov', 'test' and 'jne' instruction
+vs. the jump label case just has a 'no-op' or 'jmp 0'. (The jmp 0, is patched
+to a 5 byte atomic no-op instruction at boot-time.) Thus, the disabled jump
+label case adds:
+
+6 (mov) + 2 (test) + 2 (jne) = 10 - 5 (5 byte jump 0) = 5 addition bytes.
+
+If we then include the padding bytes, the jump label code saves, 16 total bytes
+of instruction memory for this small fucntion. In this case the non-jump label
+function is 80 bytes long. Thus, we have have saved 20% of the instruction
+footprint. We can in fact improve this even further, since the 5-byte no-op
+really can be a 2-byte no-op since we can reach the branch with a 2-byte jmp.
+However, we have not yet implemented optimal no-op sizes (they are currently
+hard-coded).
+
+Since there are a number of static key API uses in the scheduler paths,
+'pipe-test' (also known as 'perf bench sched pipe') can be used to show the
+performance improvement. Testing done on 3.3.0-rc2:
+
+jump label disabled:
+
+ Performance counter stats for 'bash -c /tmp/pipe-test' (50 runs):
+
+ 855.700314 task-clock # 0.534 CPUs utilized ( +- 0.11% )
+ 200,003 context-switches # 0.234 M/sec ( +- 0.00% )
+ 0 CPU-migrations # 0.000 M/sec ( +- 39.58% )
+ 487 page-faults # 0.001 M/sec ( +- 0.02% )
+ 1,474,374,262 cycles # 1.723 GHz ( +- 0.17% )
+ <not supported> stalled-cycles-frontend
+ <not supported> stalled-cycles-backend
+ 1,178,049,567 instructions # 0.80 insns per cycle ( +- 0.06% )
+ 208,368,926 branches # 243.507 M/sec ( +- 0.06% )
+ 5,569,188 branch-misses # 2.67% of all branches ( +- 0.54% )
+
+ 1.601607384 seconds time elapsed ( +- 0.07% )
+
+jump label enabled:
+
+ Performance counter stats for 'bash -c /tmp/pipe-test' (50 runs):
+
+ 841.043185 task-clock # 0.533 CPUs utilized ( +- 0.12% )
+ 200,004 context-switches # 0.238 M/sec ( +- 0.00% )
+ 0 CPU-migrations # 0.000 M/sec ( +- 40.87% )
+ 487 page-faults # 0.001 M/sec ( +- 0.05% )
+ 1,432,559,428 cycles # 1.703 GHz ( +- 0.18% )
+ <not supported> stalled-cycles-frontend
+ <not supported> stalled-cycles-backend
+ 1,175,363,994 instructions # 0.82 insns per cycle ( +- 0.04% )
+ 206,859,359 branches # 245.956 M/sec ( +- 0.04% )
+ 4,884,119 branch-misses # 2.36% of all branches ( +- 0.85% )
+
+ 1.579384366 seconds time elapsed
+
+The percentage of saved branches is .7%, and we've saved 12% on
+'branch-misses'. This is where we would expect to get the most savings, since
+this optimization is about reducing the number of branches. In addition, we've
+saved .2% on instructions, and 2.8% on cycles and 1.4% on elapsed time.
[ Resent with a [PATCH] prefix added to the subject line.
Sorry about the duplicate. ]
* Ingo Molnar <[email protected]> wrote:
> * Ingo Molnar <[email protected]> wrote:
>
> > So, a modified scheme would be:
> >
> > #include <linux/static_key.h>
> >
> > struct static_key key = STATIC_KEY_INIT_TRUE;
> >
> > if (static_key_false(&key))
> > do unlikely code
> > else
> > do likely code
> >
> > Or:
> >
> > if (static_key_true(&key))
> > do likely code
> > else
> > do unlikely code
> >
> > The static key is modified via:
> >
> > static_key_slow_inc(&key);
> > ...
> > static_key_slow_dec(&key);
> >
> > Is that API fine? I'll rework the series to such an effect if
> > everyone agrees.
>
> I.e. something like the patch below on top of
> tip:perf/jump-labels.
>
> Untested - will test it and will refactor the series if
> everyone's happy.
So, below is the second version of my 'struct static_key' patch,
with the confusing very_likely()/very_unlikely() method renamed
to static_key_true()/static_key_false().
Thanks,
Ingo
-------------------->
>From 25764dffac2414774b65a831733b232d33006884 Mon Sep 17 00:00:00 2001
From: Ingo Molnar <[email protected]>
Date: Fri, 24 Feb 2012 08:31:31 +0100
Subject: [PATCH] static keys: Introduce 'struct static_key', static_key_true()/false() and static_key_slow_[inc|dec]()
So here's a boot tested patch on top of Jason's series that does
all the cleanups I talked about and turns jump labels into a
more intuitive to use facility. It should also address the
various misconceptions and confusions that surround jump labels.
Typical usage scenarios:
#include <linux/static_key.h>
struct static_key key = STATIC_KEY_INIT_TRUE;
if (static_key_false(&key))
do unlikely code
else
do likely code
Or:
if (static_key_true(&key))
do likely code
else
do unlikely code
The static key is modified via:
static_key_slow_inc(&key);
...
static_key_slow_dec(&key);
The 'slow' prefix makes it abundantly clear that this is an
expensive operation.
I've updated all in-kernel code to use this everywhere. Note
that I (intentionally) have not pushed through the rename
blindly through to the lowest levels: the actual jump-label
patching arch facility should be named like that, so we want to
decouple jump labels from the static-key facility a bit.
On non-jump-label enabled architectures static keys default to
likely()/unlikely() branches.
Signed-off-by: Ingo Molnar <[email protected]>
Cc: Steven Rostedt <[email protected]>
Cc: Jason Baron <[email protected]>
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: Linus Torvalds <[email protected]>
Link: http://lkml.kernel.org/r/[email protected]
Signed-off-by: Ingo Molnar <[email protected]>
---
arch/Kconfig | 29 +++++--
arch/ia64/include/asm/paravirt.h | 6 +-
arch/ia64/kernel/paravirt.c | 4 +-
arch/mips/include/asm/jump_label.h | 2 +-
arch/powerpc/include/asm/jump_label.h | 2 +-
arch/s390/include/asm/jump_label.h | 2 +-
arch/sparc/include/asm/jump_label.h | 2 +-
arch/x86/include/asm/jump_label.h | 6 +-
arch/x86/include/asm/paravirt.h | 6 +-
arch/x86/kernel/kvm.c | 4 +-
arch/x86/kernel/paravirt.c | 4 +-
arch/x86/kvm/mmu_audit.c | 8 +-
include/linux/jump_label.h | 139 +++++++++++++++++++++++---------
include/linux/netdevice.h | 4 +-
include/linux/netfilter.h | 6 +-
include/linux/perf_event.h | 12 ++--
include/linux/static_key.h | 1 +
include/linux/tracepoint.h | 8 +-
include/net/sock.h | 6 +-
kernel/events/core.c | 16 ++--
kernel/jump_label.c | 128 +++++++++++++++++-------------
kernel/sched/core.c | 18 ++--
kernel/sched/fair.c | 8 +-
kernel/sched/sched.h | 14 ++--
kernel/tracepoint.c | 20 +++---
net/core/dev.c | 24 +++---
net/core/net-sysfs.c | 4 +-
net/core/sock.c | 4 +-
net/core/sysctl_net_core.c | 4 +-
net/ipv4/tcp_memcontrol.c | 6 +-
net/netfilter/core.c | 6 +-
31 files changed, 298 insertions(+), 205 deletions(-)
diff --git a/arch/Kconfig b/arch/Kconfig
index 4f55c73..5b448a7 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -47,18 +47,29 @@ config KPROBES
If in doubt, say "N".
config JUMP_LABEL
- bool "Optimize trace point call sites"
+ bool "Optimize very unlikely/likely branches"
depends on HAVE_ARCH_JUMP_LABEL
help
+ This option enables a transparent branch optimization that
+ makes certain almost-always-true or almost-always-false branch
+ conditions even cheaper to execute within the kernel.
+
+ Certain performance-sensitive kernel code, such as trace points,
+ scheduler functionality, networking code and KVM have such
+ branches and include support for this optimization technique.
+
If it is detected that the compiler has support for "asm goto",
- the kernel will compile trace point locations with just a
- nop instruction. When trace points are enabled, the nop will
- be converted to a jump to the trace function. This technique
- lowers overhead and stress on the branch prediction of the
- processor.
-
- On i386, options added to the compiler flags may increase
- the size of the kernel slightly.
+ the kernel will compile such branches with just a nop
+ instruction. When the condition flag is toggled to true, the
+ nop will be converted to a jump instruction to execute the
+ conditional block of instructions.
+
+ This technique lowers overhead and stress on the branch prediction
+ of the processor and generally makes the kernel faster. The update
+ of the condition is slower, but those are always very rare.
+
+ ( On 32-bit x86, the necessary options added to the compiler
+ flags may increase the size of the kernel slightly. )
config OPTPROBES
def_bool y
diff --git a/arch/ia64/include/asm/paravirt.h b/arch/ia64/include/asm/paravirt.h
index 32551d3..b149b88 100644
--- a/arch/ia64/include/asm/paravirt.h
+++ b/arch/ia64/include/asm/paravirt.h
@@ -281,9 +281,9 @@ paravirt_init_missing_ticks_accounting(int cpu)
pv_time_ops.init_missing_ticks_accounting(cpu);
}
-struct jump_label_key;
-extern struct jump_label_key paravirt_steal_enabled;
-extern struct jump_label_key paravirt_steal_rq_enabled;
+struct static_key;
+extern struct static_key paravirt_steal_enabled;
+extern struct static_key paravirt_steal_rq_enabled;
static inline int
paravirt_do_steal_accounting(unsigned long *new_itm)
diff --git a/arch/ia64/kernel/paravirt.c b/arch/ia64/kernel/paravirt.c
index 1008682..1b22f6d 100644
--- a/arch/ia64/kernel/paravirt.c
+++ b/arch/ia64/kernel/paravirt.c
@@ -634,8 +634,8 @@ struct pv_irq_ops pv_irq_ops = {
* pv_time_ops
* time operations
*/
-struct jump_label_key paravirt_steal_enabled;
-struct jump_label_key paravirt_steal_rq_enabled;
+struct static_key paravirt_steal_enabled;
+struct static_key paravirt_steal_rq_enabled;
static int
ia64_native_do_steal_accounting(unsigned long *new_itm)
diff --git a/arch/mips/include/asm/jump_label.h b/arch/mips/include/asm/jump_label.h
index 1881b31..4d6d77e 100644
--- a/arch/mips/include/asm/jump_label.h
+++ b/arch/mips/include/asm/jump_label.h
@@ -20,7 +20,7 @@
#define WORD_INSN ".word"
#endif
-static __always_inline bool arch_static_branch(struct jump_label_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key)
{
asm goto("1:\tnop\n\t"
"nop\n\t"
diff --git a/arch/powerpc/include/asm/jump_label.h b/arch/powerpc/include/asm/jump_label.h
index 938986e..ae098c4 100644
--- a/arch/powerpc/include/asm/jump_label.h
+++ b/arch/powerpc/include/asm/jump_label.h
@@ -17,7 +17,7 @@
#define JUMP_ENTRY_TYPE stringify_in_c(FTR_ENTRY_LONG)
#define JUMP_LABEL_NOP_SIZE 4
-static __always_inline bool arch_static_branch(struct jump_label_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key)
{
asm goto("1:\n\t"
"nop\n\t"
diff --git a/arch/s390/include/asm/jump_label.h b/arch/s390/include/asm/jump_label.h
index 95a6cf2..6c32190 100644
--- a/arch/s390/include/asm/jump_label.h
+++ b/arch/s390/include/asm/jump_label.h
@@ -13,7 +13,7 @@
#define ASM_ALIGN ".balign 4"
#endif
-static __always_inline bool arch_static_branch(struct jump_label_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key)
{
asm goto("0: brcl 0,0\n"
".pushsection __jump_table, \"aw\"\n"
diff --git a/arch/sparc/include/asm/jump_label.h b/arch/sparc/include/asm/jump_label.h
index fc73a82..5080d16 100644
--- a/arch/sparc/include/asm/jump_label.h
+++ b/arch/sparc/include/asm/jump_label.h
@@ -7,7 +7,7 @@
#define JUMP_LABEL_NOP_SIZE 4
-static __always_inline bool arch_static_branch(struct jump_label_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key)
{
asm goto("1:\n\t"
"nop\n\t"
diff --git a/arch/x86/include/asm/jump_label.h b/arch/x86/include/asm/jump_label.h
index a32b18c..3a16c14 100644
--- a/arch/x86/include/asm/jump_label.h
+++ b/arch/x86/include/asm/jump_label.h
@@ -9,12 +9,12 @@
#define JUMP_LABEL_NOP_SIZE 5
-#define JUMP_LABEL_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t"
+#define STATIC_KEY_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t"
-static __always_inline bool arch_static_branch(struct jump_label_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key)
{
asm goto("1:"
- JUMP_LABEL_INITIAL_NOP
+ STATIC_KEY_INITIAL_NOP
".pushsection __jump_table, \"aw\" \n\t"
_ASM_ALIGN "\n\t"
_ASM_PTR "1b, %l[l_yes], %c0 \n\t"
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index a7d2db9..c0180fd 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -230,9 +230,9 @@ static inline unsigned long long paravirt_sched_clock(void)
return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock);
}
-struct jump_label_key;
-extern struct jump_label_key paravirt_steal_enabled;
-extern struct jump_label_key paravirt_steal_rq_enabled;
+struct static_key;
+extern struct static_key paravirt_steal_enabled;
+extern struct static_key paravirt_steal_rq_enabled;
static inline u64 paravirt_steal_clock(int cpu)
{
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index f0c6fd6..694d801 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -438,9 +438,9 @@ void __init kvm_guest_init(void)
static __init int activate_jump_labels(void)
{
if (has_steal_clock) {
- jump_label_inc(¶virt_steal_enabled);
+ static_key_slow_inc(¶virt_steal_enabled);
if (steal_acc)
- jump_label_inc(¶virt_steal_rq_enabled);
+ static_key_slow_inc(¶virt_steal_rq_enabled);
}
return 0;
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index d90272e..ada2f99 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -202,8 +202,8 @@ static void native_flush_tlb_single(unsigned long addr)
__native_flush_tlb_single(addr);
}
-struct jump_label_key paravirt_steal_enabled;
-struct jump_label_key paravirt_steal_rq_enabled;
+struct static_key paravirt_steal_enabled;
+struct static_key paravirt_steal_rq_enabled;
static u64 native_steal_clock(int cpu)
{
diff --git a/arch/x86/kvm/mmu_audit.c b/arch/x86/kvm/mmu_audit.c
index fe15dcc..ea7b4fd 100644
--- a/arch/x86/kvm/mmu_audit.c
+++ b/arch/x86/kvm/mmu_audit.c
@@ -234,7 +234,7 @@ static void audit_vcpu_spte(struct kvm_vcpu *vcpu)
}
static bool mmu_audit;
-static struct jump_label_key mmu_audit_key;
+static struct static_key mmu_audit_key;
static void __kvm_mmu_audit(struct kvm_vcpu *vcpu, int point)
{
@@ -250,7 +250,7 @@ static void __kvm_mmu_audit(struct kvm_vcpu *vcpu, int point)
static inline void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point)
{
- if (static_branch((&mmu_audit_key)))
+ if (static_key_false((&mmu_audit_key)))
__kvm_mmu_audit(vcpu, point);
}
@@ -259,7 +259,7 @@ static void mmu_audit_enable(void)
if (mmu_audit)
return;
- jump_label_inc(&mmu_audit_key);
+ static_key_slow_inc(&mmu_audit_key);
mmu_audit = true;
}
@@ -268,7 +268,7 @@ static void mmu_audit_disable(void)
if (!mmu_audit)
return;
- jump_label_dec(&mmu_audit_key);
+ static_key_slow_dec(&mmu_audit_key);
mmu_audit = false;
}
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index f7c6958..30d0023 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -9,15 +9,15 @@
*
* Jump labels provide an interface to generate dynamic branches using
* self-modifying code. Assuming toolchain and architecture support the result
- * of a "if (static_branch(&key))" statement is a unconditional branch (which
+ * of a "if (static_key_false(&key))" statement is a unconditional branch (which
* defaults to false - and the true block is placed out of line).
*
- * However at runtime we can change the 'static' branch target using
- * jump_label_{inc,dec}(). These function as a 'reference' count on the key
+ * However at runtime we can change the branch target using
+ * static_key_slow_{inc,dec}(). These function as a 'reference' count on the key
* object and for as long as there are references all branches referring to
* that particular key will point to the (out of line) true block.
*
- * Since this relies on modifying code the jump_label_{inc,dec}() functions
+ * Since this relies on modifying code the static_key_slow_{inc,dec}() functions
* must be considered absolute slow paths (machine wide synchronization etc.).
* OTOH, since the affected branches are unconditional their runtime overhead
* will be absolutely minimal, esp. in the default (off) case where the total
@@ -26,12 +26,26 @@
*
* When the control is directly exposed to userspace it is prudent to delay the
* decrement to avoid high frequency code modifications which can (and do)
- * cause significant performance degradation. Struct jump_label_key_deferred and
- * jump_label_dec_deferred() provide for this.
+ * cause significant performance degradation. Struct static_key_deferred and
+ * static_key_slow_dec_deferred() provide for this.
*
* Lacking toolchain and or architecture support, it falls back to a simple
* conditional branch.
- */
+ *
+ * struct static_key my_key = STATIC_KEY_INIT_TRUE;
+ *
+ * if (static_key_true(&my_key)) {
+ * }
+ *
+ * will result in the true case being in-line and starts the key with a single
+ * reference. Mixing static_key_true() and static_key_false() on the same key is not
+ * allowed.
+ *
+ * Not initializing the key (static data is initialized to 0s anyway) is the
+ * same as using STATIC_KEY_INIT_FALSE and static_key_false() is
+ * equivalent with static_branch().
+ *
+*/
#include <linux/types.h>
#include <linux/compiler.h>
@@ -39,16 +53,17 @@
#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
-struct jump_label_key {
+struct static_key {
atomic_t enabled;
+/* Set lsb bit to 1 if branch is default true, 0 ot */
struct jump_entry *entries;
#ifdef CONFIG_MODULES
- struct jump_label_mod *next;
+ struct static_key_mod *next;
#endif
};
-struct jump_label_key_deferred {
- struct jump_label_key key;
+struct static_key_deferred {
+ struct static_key key;
unsigned long timeout;
struct delayed_work work;
};
@@ -66,13 +81,34 @@ struct module;
#ifdef HAVE_JUMP_LABEL
-#ifdef CONFIG_MODULES
-#define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL, NULL}
-#else
-#define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL}
-#endif
+#define JUMP_LABEL_TRUE_BRANCH 1UL
+
+static
+inline struct jump_entry *jump_label_get_entries(struct static_key *key)
+{
+ return (struct jump_entry *)((unsigned long)key->entries
+ & ~JUMP_LABEL_TRUE_BRANCH);
+}
+
+static inline bool jump_label_get_branch_default(struct static_key *key)
+{
+ if ((unsigned long)key->entries & JUMP_LABEL_TRUE_BRANCH)
+ return true;
+ return false;
+}
+
+static __always_inline bool static_key_false(struct static_key *key)
+{
+ return arch_static_branch(key);
+}
-static __always_inline bool static_branch(struct jump_label_key *key)
+static __always_inline bool static_key_true(struct static_key *key)
+{
+ return !static_key_false(key);
+}
+
+/* Deprecated. Please use 'static_key_false() instead. */
+static __always_inline bool static_branch(struct static_key *key)
{
return arch_static_branch(key);
}
@@ -88,21 +124,24 @@ extern void arch_jump_label_transform(struct jump_entry *entry,
extern void arch_jump_label_transform_static(struct jump_entry *entry,
enum jump_label_type type);
extern int jump_label_text_reserved(void *start, void *end);
-extern void jump_label_inc(struct jump_label_key *key);
-extern void jump_label_dec(struct jump_label_key *key);
-extern void jump_label_dec_deferred(struct jump_label_key_deferred *key);
-extern bool jump_label_enabled(struct jump_label_key *key);
+extern void static_key_slow_inc(struct static_key *key);
+extern void static_key_slow_dec(struct static_key *key);
+extern void static_key_slow_dec_deferred(struct static_key_deferred *key);
+extern bool static_key_true(struct static_key *key);
extern void jump_label_apply_nops(struct module *mod);
-extern void jump_label_rate_limit(struct jump_label_key_deferred *key,
- unsigned long rl);
+extern void
+jump_label_rate_limit(struct static_key_deferred *key, unsigned long rl);
+
+#define STATIC_KEY_INIT_TRUE ((struct static_key) \
+ { .enabled = ATOMIC_INIT(1), .entries = (void *)1 })
+#define STATIC_KEY_INIT_FALSE ((struct static_key) \
+ { .enabled = ATOMIC_INIT(0), .entries = (void *)0 })
#else /* !HAVE_JUMP_LABEL */
#include <linux/atomic.h>
-#define JUMP_LABEL_INIT {ATOMIC_INIT(0)}
-
-struct jump_label_key {
+struct static_key {
atomic_t enabled;
};
@@ -110,30 +149,45 @@ static __always_inline void jump_label_init(void)
{
}
-struct jump_label_key_deferred {
- struct jump_label_key key;
+struct static_key_deferred {
+ struct static_key key;
};
-static __always_inline bool static_branch(struct jump_label_key *key)
+static __always_inline bool static_key_false(struct static_key *key)
+{
+ if (unlikely(atomic_read(&key->enabled)) > 0)
+ return true;
+ return false;
+}
+
+static __always_inline bool static_key_true(struct static_key *key)
{
- if (unlikely(atomic_read(&key->enabled)))
+ if (likely(atomic_read(&key->enabled)) > 0)
return true;
return false;
}
-static inline void jump_label_inc(struct jump_label_key *key)
+/* Deprecated. Please use 'static_key_false() instead. */
+static __always_inline bool static_branch(struct static_key *key)
+{
+ if (unlikely(atomic_read(&key->enabled)) > 0)
+ return true;
+ return false;
+}
+
+static inline void static_key_slow_inc(struct static_key *key)
{
atomic_inc(&key->enabled);
}
-static inline void jump_label_dec(struct jump_label_key *key)
+static inline void static_key_slow_dec(struct static_key *key)
{
atomic_dec(&key->enabled);
}
-static inline void jump_label_dec_deferred(struct jump_label_key_deferred *key)
+static inline void static_key_slow_dec_deferred(struct static_key_deferred *key)
{
- jump_label_dec(&key->key);
+ static_key_slow_dec(&key->key);
}
static inline int jump_label_text_reserved(void *start, void *end)
@@ -144,9 +198,9 @@ static inline int jump_label_text_reserved(void *start, void *end)
static inline void jump_label_lock(void) {}
static inline void jump_label_unlock(void) {}
-static inline bool jump_label_enabled(struct jump_label_key *key)
+static inline bool static_key_true(struct static_key *key)
{
- return !!atomic_read(&key->enabled);
+ return (atomic_read(&key->enabled) > 0);
}
static inline int jump_label_apply_nops(struct module *mod)
@@ -154,13 +208,20 @@ static inline int jump_label_apply_nops(struct module *mod)
return 0;
}
-static inline void jump_label_rate_limit(struct jump_label_key_deferred *key,
+static inline void
+jump_label_rate_limit(struct static_key_deferred *key,
unsigned long rl)
{
}
+
+#define STATIC_KEY_INIT_TRUE ((struct static_key) \
+ { .enabled = ATOMIC_INIT(1) })
+#define STATIC_KEY_INIT_FALSE ((struct static_key) \
+ { .enabled = ATOMIC_INIT(0) })
+
#endif /* HAVE_JUMP_LABEL */
-#define jump_label_key_enabled ((struct jump_label_key){ .enabled = ATOMIC_INIT(1), })
-#define jump_label_key_disabled ((struct jump_label_key){ .enabled = ATOMIC_INIT(0), })
+#define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
+#define jump_label_enabled static_key_true
#endif /* _LINUX_JUMP_LABEL_H */
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 0eac07c..7dfaae7 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -214,8 +214,8 @@ enum {
#include <linux/skbuff.h>
#ifdef CONFIG_RPS
-#include <linux/jump_label.h>
-extern struct jump_label_key rps_needed;
+#include <linux/static_key.h>
+extern struct static_key rps_needed;
#endif
struct neighbour;
diff --git a/include/linux/netfilter.h b/include/linux/netfilter.h
index b809265..29734be 100644
--- a/include/linux/netfilter.h
+++ b/include/linux/netfilter.h
@@ -163,13 +163,13 @@ extern struct ctl_path nf_net_ipv4_netfilter_sysctl_path[];
extern struct list_head nf_hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
#if defined(CONFIG_JUMP_LABEL)
-#include <linux/jump_label.h>
-extern struct jump_label_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
+#include <linux/static_key.h>
+extern struct static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
static inline bool nf_hooks_active(u_int8_t pf, unsigned int hook)
{
if (__builtin_constant_p(pf) &&
__builtin_constant_p(hook))
- return static_branch(&nf_hooks_needed[pf][hook]);
+ return static_key_false(&nf_hooks_needed[pf][hook]);
return !list_empty(&nf_hooks[pf][hook]);
}
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 412b790..0d21e6f 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -514,7 +514,7 @@ struct perf_guest_info_callbacks {
#include <linux/ftrace.h>
#include <linux/cpu.h>
#include <linux/irq_work.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
#include <linux/atomic.h>
#include <asm/local.h>
@@ -1038,7 +1038,7 @@ static inline int is_software_event(struct perf_event *event)
return event->pmu->task_ctx_nr == perf_sw_context;
}
-extern struct jump_label_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
+extern struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
extern void __perf_sw_event(u32, u64, struct pt_regs *, u64);
@@ -1066,7 +1066,7 @@ perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
{
struct pt_regs hot_regs;
- if (static_branch(&perf_swevent_enabled[event_id])) {
+ if (static_key_false(&perf_swevent_enabled[event_id])) {
if (!regs) {
perf_fetch_caller_regs(&hot_regs);
regs = &hot_regs;
@@ -1075,12 +1075,12 @@ perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
}
}
-extern struct jump_label_key_deferred perf_sched_events;
+extern struct static_key_deferred perf_sched_events;
static inline void perf_event_task_sched_in(struct task_struct *prev,
struct task_struct *task)
{
- if (static_branch(&perf_sched_events.key))
+ if (static_key_false(&perf_sched_events.key))
__perf_event_task_sched_in(prev, task);
}
@@ -1089,7 +1089,7 @@ static inline void perf_event_task_sched_out(struct task_struct *prev,
{
perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, NULL, 0);
- if (static_branch(&perf_sched_events.key))
+ if (static_key_false(&perf_sched_events.key))
__perf_event_task_sched_out(prev, next);
}
diff --git a/include/linux/static_key.h b/include/linux/static_key.h
new file mode 100644
index 0000000..27bd3f8
--- /dev/null
+++ b/include/linux/static_key.h
@@ -0,0 +1 @@
+#include <linux/jump_label.h>
diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h
index fc36da9..bd96ecd 100644
--- a/include/linux/tracepoint.h
+++ b/include/linux/tracepoint.h
@@ -17,7 +17,7 @@
#include <linux/errno.h>
#include <linux/types.h>
#include <linux/rcupdate.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
struct module;
struct tracepoint;
@@ -29,7 +29,7 @@ struct tracepoint_func {
struct tracepoint {
const char *name; /* Tracepoint name */
- struct jump_label_key key;
+ struct static_key key;
void (*regfunc)(void);
void (*unregfunc)(void);
struct tracepoint_func __rcu *funcs;
@@ -145,7 +145,7 @@ static inline void tracepoint_synchronize_unregister(void)
extern struct tracepoint __tracepoint_##name; \
static inline void trace_##name(proto) \
{ \
- if (static_branch(&__tracepoint_##name.key)) \
+ if (static_key_false(&__tracepoint_##name.key)) \
__DO_TRACE(&__tracepoint_##name, \
TP_PROTO(data_proto), \
TP_ARGS(data_args), \
@@ -188,7 +188,7 @@ static inline void tracepoint_synchronize_unregister(void)
__attribute__((section("__tracepoints_strings"))) = #name; \
struct tracepoint __tracepoint_##name \
__attribute__((section("__tracepoints"))) = \
- { __tpstrtab_##name, JUMP_LABEL_INIT, reg, unreg, NULL };\
+ { __tpstrtab_##name, STATIC_KEY_INIT_FALSE, reg, unreg, NULL };\
static struct tracepoint * const __tracepoint_ptr_##name __used \
__attribute__((section("__tracepoints_ptrs"))) = \
&__tracepoint_##name;
diff --git a/include/net/sock.h b/include/net/sock.h
index 91c1c8b..dcde2d9 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -55,7 +55,7 @@
#include <linux/uaccess.h>
#include <linux/memcontrol.h>
#include <linux/res_counter.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
#include <linux/filter.h>
#include <linux/rculist_nulls.h>
@@ -924,13 +924,13 @@ inline void sk_refcnt_debug_release(const struct sock *sk)
#endif /* SOCK_REFCNT_DEBUG */
#if defined(CONFIG_CGROUP_MEM_RES_CTLR_KMEM) && defined(CONFIG_NET)
-extern struct jump_label_key memcg_socket_limit_enabled;
+extern struct static_key memcg_socket_limit_enabled;
static inline struct cg_proto *parent_cg_proto(struct proto *proto,
struct cg_proto *cg_proto)
{
return proto->proto_cgroup(parent_mem_cgroup(cg_proto->memcg));
}
-#define mem_cgroup_sockets_enabled static_branch(&memcg_socket_limit_enabled)
+#define mem_cgroup_sockets_enabled static_key_false(&memcg_socket_limit_enabled)
#else
#define mem_cgroup_sockets_enabled 0
static inline struct cg_proto *parent_cg_proto(struct proto *proto,
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 7c3b9de..5e0f8bb 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -128,7 +128,7 @@ enum event_type_t {
* perf_sched_events : >0 events exist
* perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
*/
-struct jump_label_key_deferred perf_sched_events __read_mostly;
+struct static_key_deferred perf_sched_events __read_mostly;
static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
static atomic_t nr_mmap_events __read_mostly;
@@ -2769,7 +2769,7 @@ static void free_event(struct perf_event *event)
if (!event->parent) {
if (event->attach_state & PERF_ATTACH_TASK)
- jump_label_dec_deferred(&perf_sched_events);
+ static_key_slow_dec_deferred(&perf_sched_events);
if (event->attr.mmap || event->attr.mmap_data)
atomic_dec(&nr_mmap_events);
if (event->attr.comm)
@@ -2780,7 +2780,7 @@ static void free_event(struct perf_event *event)
put_callchain_buffers();
if (is_cgroup_event(event)) {
atomic_dec(&per_cpu(perf_cgroup_events, event->cpu));
- jump_label_dec_deferred(&perf_sched_events);
+ static_key_slow_dec_deferred(&perf_sched_events);
}
}
@@ -4982,7 +4982,7 @@ fail:
return err;
}
-struct jump_label_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
+struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
static void sw_perf_event_destroy(struct perf_event *event)
{
@@ -4990,7 +4990,7 @@ static void sw_perf_event_destroy(struct perf_event *event)
WARN_ON(event->parent);
- jump_label_dec(&perf_swevent_enabled[event_id]);
+ static_key_slow_dec(&perf_swevent_enabled[event_id]);
swevent_hlist_put(event);
}
@@ -5020,7 +5020,7 @@ static int perf_swevent_init(struct perf_event *event)
if (err)
return err;
- jump_label_inc(&perf_swevent_enabled[event_id]);
+ static_key_slow_inc(&perf_swevent_enabled[event_id]);
event->destroy = sw_perf_event_destroy;
}
@@ -5843,7 +5843,7 @@ done:
if (!event->parent) {
if (event->attach_state & PERF_ATTACH_TASK)
- jump_label_inc(&perf_sched_events.key);
+ static_key_slow_inc(&perf_sched_events.key);
if (event->attr.mmap || event->attr.mmap_data)
atomic_inc(&nr_mmap_events);
if (event->attr.comm)
@@ -6081,7 +6081,7 @@ SYSCALL_DEFINE5(perf_event_open,
* - that may need work on context switch
*/
atomic_inc(&per_cpu(perf_cgroup_events, event->cpu));
- jump_label_inc(&perf_sched_events.key);
+ static_key_slow_inc(&perf_sched_events.key);
}
/*
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 543782e..1cc32e3 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -12,7 +12,7 @@
#include <linux/slab.h>
#include <linux/sort.h>
#include <linux/err.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
#ifdef HAVE_JUMP_LABEL
@@ -29,10 +29,11 @@ void jump_label_unlock(void)
mutex_unlock(&jump_label_mutex);
}
-bool jump_label_enabled(struct jump_label_key *key)
+bool static_key_true(struct static_key *key)
{
- return !!atomic_read(&key->enabled);
+ return (atomic_read(&key->enabled) > 0);
}
+EXPORT_SYMBOL_GPL(static_key_true);
static int jump_label_cmp(const void *a, const void *b)
{
@@ -58,22 +59,26 @@ jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop)
sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL);
}
-static void jump_label_update(struct jump_label_key *key, int enable);
+static void jump_label_update(struct static_key *key, int enable);
-void jump_label_inc(struct jump_label_key *key)
+void static_key_slow_inc(struct static_key *key)
{
if (atomic_inc_not_zero(&key->enabled))
return;
jump_label_lock();
- if (atomic_read(&key->enabled) == 0)
- jump_label_update(key, JUMP_LABEL_ENABLE);
+ if (atomic_read(&key->enabled) == 0) {
+ if (!jump_label_get_branch_default(key))
+ jump_label_update(key, JUMP_LABEL_ENABLE);
+ else
+ jump_label_update(key, JUMP_LABEL_DISABLE);
+ }
atomic_inc(&key->enabled);
jump_label_unlock();
}
-EXPORT_SYMBOL_GPL(jump_label_inc);
+EXPORT_SYMBOL_GPL(static_key_slow_inc);
-static void __jump_label_dec(struct jump_label_key *key,
+static void __static_key_slow_dec(struct static_key *key,
unsigned long rate_limit, struct delayed_work *work)
{
if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) {
@@ -85,32 +90,35 @@ static void __jump_label_dec(struct jump_label_key *key,
if (rate_limit) {
atomic_inc(&key->enabled);
schedule_delayed_work(work, rate_limit);
- } else
- jump_label_update(key, JUMP_LABEL_DISABLE);
-
+ } else {
+ if (!jump_label_get_branch_default(key))
+ jump_label_update(key, JUMP_LABEL_DISABLE);
+ else
+ jump_label_update(key, JUMP_LABEL_ENABLE);
+ }
jump_label_unlock();
}
-EXPORT_SYMBOL_GPL(jump_label_dec);
static void jump_label_update_timeout(struct work_struct *work)
{
- struct jump_label_key_deferred *key =
- container_of(work, struct jump_label_key_deferred, work.work);
- __jump_label_dec(&key->key, 0, NULL);
+ struct static_key_deferred *key =
+ container_of(work, struct static_key_deferred, work.work);
+ __static_key_slow_dec(&key->key, 0, NULL);
}
-void jump_label_dec(struct jump_label_key *key)
+void static_key_slow_dec(struct static_key *key)
{
- __jump_label_dec(key, 0, NULL);
+ __static_key_slow_dec(key, 0, NULL);
}
+EXPORT_SYMBOL_GPL(static_key_slow_dec);
-void jump_label_dec_deferred(struct jump_label_key_deferred *key)
+void static_key_slow_dec_deferred(struct static_key_deferred *key)
{
- __jump_label_dec(&key->key, key->timeout, &key->work);
+ __static_key_slow_dec(&key->key, key->timeout, &key->work);
}
+EXPORT_SYMBOL_GPL(static_key_slow_dec_deferred);
-
-void jump_label_rate_limit(struct jump_label_key_deferred *key,
+void jump_label_rate_limit(struct static_key_deferred *key,
unsigned long rl)
{
key->timeout = rl;
@@ -153,7 +161,7 @@ void __weak __init_or_module arch_jump_label_transform_static(struct jump_entry
arch_jump_label_transform(entry, type);
}
-static void __jump_label_update(struct jump_label_key *key,
+static void __jump_label_update(struct static_key *key,
struct jump_entry *entry,
struct jump_entry *stop, int enable)
{
@@ -170,27 +178,40 @@ static void __jump_label_update(struct jump_label_key *key,
}
}
+static enum jump_label_type jump_label_type(struct static_key *key)
+{
+ bool true_branch = jump_label_get_branch_default(key);
+ bool state = static_key_true(key);
+
+ if ((!true_branch && state) || (true_branch && !state))
+ return JUMP_LABEL_ENABLE;
+
+ return JUMP_LABEL_DISABLE;
+}
+
void __init jump_label_init(void)
{
struct jump_entry *iter_start = __start___jump_table;
struct jump_entry *iter_stop = __stop___jump_table;
- struct jump_label_key *key = NULL;
+ struct static_key *key = NULL;
struct jump_entry *iter;
jump_label_lock();
jump_label_sort_entries(iter_start, iter_stop);
for (iter = iter_start; iter < iter_stop; iter++) {
- struct jump_label_key *iterk;
+ struct static_key *iterk;
- iterk = (struct jump_label_key *)(unsigned long)iter->key;
- arch_jump_label_transform_static(iter, jump_label_enabled(iterk) ?
- JUMP_LABEL_ENABLE : JUMP_LABEL_DISABLE);
+ iterk = (struct static_key *)(unsigned long)iter->key;
+ arch_jump_label_transform_static(iter, jump_label_type(iterk));
if (iterk == key)
continue;
key = iterk;
- key->entries = iter;
+ /*
+ * Set key->entries to iter, but preserve JUMP_LABEL_TRUE_BRANCH.
+ */
+ *((unsigned long *)&key->entries) += (unsigned long)iter;
#ifdef CONFIG_MODULES
key->next = NULL;
#endif
@@ -200,8 +221,8 @@ void __init jump_label_init(void)
#ifdef CONFIG_MODULES
-struct jump_label_mod {
- struct jump_label_mod *next;
+struct static_key_mod {
+ struct static_key_mod *next;
struct jump_entry *entries;
struct module *mod;
};
@@ -221,9 +242,9 @@ static int __jump_label_mod_text_reserved(void *start, void *end)
start, end);
}
-static void __jump_label_mod_update(struct jump_label_key *key, int enable)
+static void __jump_label_mod_update(struct static_key *key, int enable)
{
- struct jump_label_mod *mod = key->next;
+ struct static_key_mod *mod = key->next;
while (mod) {
struct module *m = mod->mod;
@@ -254,11 +275,7 @@ void jump_label_apply_nops(struct module *mod)
return;
for (iter = iter_start; iter < iter_stop; iter++) {
- struct jump_label_key *iterk;
-
- iterk = (struct jump_label_key *)(unsigned long)iter->key;
- arch_jump_label_transform_static(iter, jump_label_enabled(iterk) ?
- JUMP_LABEL_ENABLE : JUMP_LABEL_DISABLE);
+ arch_jump_label_transform_static(iter, JUMP_LABEL_DISABLE);
}
}
@@ -267,8 +284,8 @@ static int jump_label_add_module(struct module *mod)
struct jump_entry *iter_start = mod->jump_entries;
struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
struct jump_entry *iter;
- struct jump_label_key *key = NULL;
- struct jump_label_mod *jlm;
+ struct static_key *key = NULL;
+ struct static_key_mod *jlm;
/* if the module doesn't have jump label entries, just return */
if (iter_start == iter_stop)
@@ -277,28 +294,30 @@ static int jump_label_add_module(struct module *mod)
jump_label_sort_entries(iter_start, iter_stop);
for (iter = iter_start; iter < iter_stop; iter++) {
- if (iter->key == (jump_label_t)(unsigned long)key)
- continue;
+ struct static_key *iterk;
- key = (struct jump_label_key *)(unsigned long)iter->key;
+ iterk = (struct static_key *)(unsigned long)iter->key;
+ if (iterk == key)
+ continue;
+ key = iterk;
if (__module_address(iter->key) == mod) {
- atomic_set(&key->enabled, 0);
- key->entries = iter;
+ /*
+ * Set key->entries to iter, but preserve JUMP_LABEL_TRUE_BRANCH.
+ */
+ *((unsigned long *)&key->entries) += (unsigned long)iter;
key->next = NULL;
continue;
}
-
- jlm = kzalloc(sizeof(struct jump_label_mod), GFP_KERNEL);
+ jlm = kzalloc(sizeof(struct static_key_mod), GFP_KERNEL);
if (!jlm)
return -ENOMEM;
-
jlm->mod = mod;
jlm->entries = iter;
jlm->next = key->next;
key->next = jlm;
- if (jump_label_enabled(key))
+ if (jump_label_type(key) == JUMP_LABEL_ENABLE)
__jump_label_update(key, iter, iter_stop, JUMP_LABEL_ENABLE);
}
@@ -310,14 +329,14 @@ static void jump_label_del_module(struct module *mod)
struct jump_entry *iter_start = mod->jump_entries;
struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
struct jump_entry *iter;
- struct jump_label_key *key = NULL;
- struct jump_label_mod *jlm, **prev;
+ struct static_key *key = NULL;
+ struct static_key_mod *jlm, **prev;
for (iter = iter_start; iter < iter_stop; iter++) {
if (iter->key == (jump_label_t)(unsigned long)key)
continue;
- key = (struct jump_label_key *)(unsigned long)iter->key;
+ key = (struct static_key *)(unsigned long)iter->key;
if (__module_address(iter->key) == mod)
continue;
@@ -419,9 +438,10 @@ int jump_label_text_reserved(void *start, void *end)
return ret;
}
-static void jump_label_update(struct jump_label_key *key, int enable)
+static void jump_label_update(struct static_key *key, int enable)
{
- struct jump_entry *entry = key->entries, *stop = __stop___jump_table;
+ struct jump_entry *stop = __stop___jump_table;
+ struct jump_entry *entry = jump_label_get_entries(key);
#ifdef CONFIG_MODULES
struct module *mod = __module_address((unsigned long)key);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 5255c9d..05fb5df 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -162,13 +162,13 @@ static int sched_feat_show(struct seq_file *m, void *v)
#ifdef HAVE_JUMP_LABEL
-#define jump_label_key__true jump_label_key_enabled
-#define jump_label_key__false jump_label_key_disabled
+#define jump_label_key__true STATIC_KEY_INIT_TRUE
+#define jump_label_key__false STATIC_KEY_INIT_FALSE
#define SCHED_FEAT(name, enabled) \
jump_label_key__##enabled ,
-struct jump_label_key sched_feat_keys[__SCHED_FEAT_NR] = {
+struct static_key sched_feat_keys[__SCHED_FEAT_NR] = {
#include "features.h"
};
@@ -176,14 +176,14 @@ struct jump_label_key sched_feat_keys[__SCHED_FEAT_NR] = {
static void sched_feat_disable(int i)
{
- if (jump_label_enabled(&sched_feat_keys[i]))
- jump_label_dec(&sched_feat_keys[i]);
+ if (static_key_true(&sched_feat_keys[i]))
+ static_key_slow_dec(&sched_feat_keys[i]);
}
static void sched_feat_enable(int i)
{
- if (!jump_label_enabled(&sched_feat_keys[i]))
- jump_label_inc(&sched_feat_keys[i]);
+ if (!static_key_true(&sched_feat_keys[i]))
+ static_key_slow_inc(&sched_feat_keys[i]);
}
#else
static void sched_feat_disable(int i) { };
@@ -894,7 +894,7 @@ static void update_rq_clock_task(struct rq *rq, s64 delta)
delta -= irq_delta;
#endif
#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
- if (static_branch((¶virt_steal_rq_enabled))) {
+ if (static_key_false((¶virt_steal_rq_enabled))) {
u64 st;
steal = paravirt_steal_clock(cpu_of(rq));
@@ -2756,7 +2756,7 @@ void account_idle_time(cputime_t cputime)
static __always_inline bool steal_account_process_tick(void)
{
#ifdef CONFIG_PARAVIRT
- if (static_branch(¶virt_steal_enabled)) {
+ if (static_key_false(¶virt_steal_enabled)) {
u64 steal, st = 0;
steal = paravirt_steal_clock(smp_processor_id());
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 7c6414f..423547a 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1399,20 +1399,20 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued)
#ifdef CONFIG_CFS_BANDWIDTH
#ifdef HAVE_JUMP_LABEL
-static struct jump_label_key __cfs_bandwidth_used;
+static struct static_key __cfs_bandwidth_used;
static inline bool cfs_bandwidth_used(void)
{
- return static_branch(&__cfs_bandwidth_used);
+ return static_key_false(&__cfs_bandwidth_used);
}
void account_cfs_bandwidth_used(int enabled, int was_enabled)
{
/* only need to count groups transitioning between enabled/!enabled */
if (enabled && !was_enabled)
- jump_label_inc(&__cfs_bandwidth_used);
+ static_key_slow_inc(&__cfs_bandwidth_used);
else if (!enabled && was_enabled)
- jump_label_dec(&__cfs_bandwidth_used);
+ static_key_slow_dec(&__cfs_bandwidth_used);
}
#else /* HAVE_JUMP_LABEL */
static bool cfs_bandwidth_used(void)
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 98c0c26..b4cd6d8 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -611,7 +611,7 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
* Tunables that become constants when CONFIG_SCHED_DEBUG is off:
*/
#ifdef CONFIG_SCHED_DEBUG
-# include <linux/jump_label.h>
+# include <linux/static_key.h>
# define const_debug __read_mostly
#else
# define const_debug const
@@ -630,18 +630,18 @@ enum {
#undef SCHED_FEAT
#if defined(CONFIG_SCHED_DEBUG) && defined(HAVE_JUMP_LABEL)
-static __always_inline bool static_branch__true(struct jump_label_key *key)
+static __always_inline bool static_branch__true(struct static_key *key)
{
- return likely(static_branch(key)); /* Not out of line branch. */
+ return static_key_true(key); /* Not out of line branch. */
}
-static __always_inline bool static_branch__false(struct jump_label_key *key)
+static __always_inline bool static_branch__false(struct static_key *key)
{
- return unlikely(static_branch(key)); /* Out of line branch. */
+ return static_key_false(key); /* Out of line branch. */
}
#define SCHED_FEAT(name, enabled) \
-static __always_inline bool static_branch_##name(struct jump_label_key *key) \
+static __always_inline bool static_branch_##name(struct static_key *key) \
{ \
return static_branch__##enabled(key); \
}
@@ -650,7 +650,7 @@ static __always_inline bool static_branch_##name(struct jump_label_key *key) \
#undef SCHED_FEAT
-extern struct jump_label_key sched_feat_keys[__SCHED_FEAT_NR];
+extern struct static_key sched_feat_keys[__SCHED_FEAT_NR];
#define sched_feat(x) (static_branch_##x(&sched_feat_keys[__SCHED_FEAT_##x]))
#else /* !(SCHED_DEBUG && HAVE_JUMP_LABEL) */
#define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x))
diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c
index f1539de..5e37c26 100644
--- a/kernel/tracepoint.c
+++ b/kernel/tracepoint.c
@@ -25,7 +25,7 @@
#include <linux/err.h>
#include <linux/slab.h>
#include <linux/sched.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
extern struct tracepoint * const __start___tracepoints_ptrs[];
extern struct tracepoint * const __stop___tracepoints_ptrs[];
@@ -256,9 +256,9 @@ static void set_tracepoint(struct tracepoint_entry **entry,
{
WARN_ON(strcmp((*entry)->name, elem->name) != 0);
- if (elem->regfunc && !jump_label_enabled(&elem->key) && active)
+ if (elem->regfunc && !static_key_true(&elem->key) && active)
elem->regfunc();
- else if (elem->unregfunc && jump_label_enabled(&elem->key) && !active)
+ else if (elem->unregfunc && static_key_true(&elem->key) && !active)
elem->unregfunc();
/*
@@ -269,10 +269,10 @@ static void set_tracepoint(struct tracepoint_entry **entry,
* is used.
*/
rcu_assign_pointer(elem->funcs, (*entry)->funcs);
- if (active && !jump_label_enabled(&elem->key))
- jump_label_inc(&elem->key);
- else if (!active && jump_label_enabled(&elem->key))
- jump_label_dec(&elem->key);
+ if (active && !static_key_true(&elem->key))
+ static_key_slow_inc(&elem->key);
+ else if (!active && static_key_true(&elem->key))
+ static_key_slow_dec(&elem->key);
}
/*
@@ -283,11 +283,11 @@ static void set_tracepoint(struct tracepoint_entry **entry,
*/
static void disable_tracepoint(struct tracepoint *elem)
{
- if (elem->unregfunc && jump_label_enabled(&elem->key))
+ if (elem->unregfunc && static_key_true(&elem->key))
elem->unregfunc();
- if (jump_label_enabled(&elem->key))
- jump_label_dec(&elem->key);
+ if (static_key_true(&elem->key))
+ static_key_slow_dec(&elem->key);
rcu_assign_pointer(elem->funcs, NULL);
}
diff --git a/net/core/dev.c b/net/core/dev.c
index 115dee1..da7ce7f 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -134,7 +134,7 @@
#include <linux/inetdevice.h>
#include <linux/cpu_rmap.h>
#include <linux/net_tstamp.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
#include <net/flow_keys.h>
#include "net-sysfs.h"
@@ -1441,11 +1441,11 @@ int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
}
EXPORT_SYMBOL(call_netdevice_notifiers);
-static struct jump_label_key netstamp_needed __read_mostly;
+static struct static_key netstamp_needed __read_mostly;
#ifdef HAVE_JUMP_LABEL
-/* We are not allowed to call jump_label_dec() from irq context
+/* We are not allowed to call static_key_slow_dec() from irq context
* If net_disable_timestamp() is called from irq context, defer the
- * jump_label_dec() calls.
+ * static_key_slow_dec() calls.
*/
static atomic_t netstamp_needed_deferred;
#endif
@@ -1457,12 +1457,12 @@ void net_enable_timestamp(void)
if (deferred) {
while (--deferred)
- jump_label_dec(&netstamp_needed);
+ static_key_slow_dec(&netstamp_needed);
return;
}
#endif
WARN_ON(in_interrupt());
- jump_label_inc(&netstamp_needed);
+ static_key_slow_inc(&netstamp_needed);
}
EXPORT_SYMBOL(net_enable_timestamp);
@@ -1474,19 +1474,19 @@ void net_disable_timestamp(void)
return;
}
#endif
- jump_label_dec(&netstamp_needed);
+ static_key_slow_dec(&netstamp_needed);
}
EXPORT_SYMBOL(net_disable_timestamp);
static inline void net_timestamp_set(struct sk_buff *skb)
{
skb->tstamp.tv64 = 0;
- if (static_branch(&netstamp_needed))
+ if (static_key_false(&netstamp_needed))
__net_timestamp(skb);
}
#define net_timestamp_check(COND, SKB) \
- if (static_branch(&netstamp_needed)) { \
+ if (static_key_false(&netstamp_needed)) { \
if ((COND) && !(SKB)->tstamp.tv64) \
__net_timestamp(SKB); \
} \
@@ -2660,7 +2660,7 @@ EXPORT_SYMBOL(__skb_get_rxhash);
struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
EXPORT_SYMBOL(rps_sock_flow_table);
-struct jump_label_key rps_needed __read_mostly;
+struct static_key rps_needed __read_mostly;
static struct rps_dev_flow *
set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
@@ -2945,7 +2945,7 @@ int netif_rx(struct sk_buff *skb)
trace_netif_rx(skb);
#ifdef CONFIG_RPS
- if (static_branch(&rps_needed)) {
+ if (static_key_false(&rps_needed)) {
struct rps_dev_flow voidflow, *rflow = &voidflow;
int cpu;
@@ -3309,7 +3309,7 @@ int netif_receive_skb(struct sk_buff *skb)
return NET_RX_SUCCESS;
#ifdef CONFIG_RPS
- if (static_branch(&rps_needed)) {
+ if (static_key_false(&rps_needed)) {
struct rps_dev_flow voidflow, *rflow = &voidflow;
int cpu, ret;
diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c
index a1727cd..4955862 100644
--- a/net/core/net-sysfs.c
+++ b/net/core/net-sysfs.c
@@ -608,10 +608,10 @@ static ssize_t store_rps_map(struct netdev_rx_queue *queue,
spin_unlock(&rps_map_lock);
if (map)
- jump_label_inc(&rps_needed);
+ static_key_slow_inc(&rps_needed);
if (old_map) {
kfree_rcu(old_map, rcu);
- jump_label_dec(&rps_needed);
+ static_key_slow_dec(&rps_needed);
}
free_cpumask_var(mask);
return len;
diff --git a/net/core/sock.c b/net/core/sock.c
index 3e81fd2..3a4e581 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -111,7 +111,7 @@
#include <linux/init.h>
#include <linux/highmem.h>
#include <linux/user_namespace.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
#include <linux/memcontrol.h>
#include <asm/uaccess.h>
@@ -184,7 +184,7 @@ void mem_cgroup_sockets_destroy(struct cgroup *cgrp, struct cgroup_subsys *ss)
static struct lock_class_key af_family_keys[AF_MAX];
static struct lock_class_key af_family_slock_keys[AF_MAX];
-struct jump_label_key memcg_socket_limit_enabled;
+struct static_key memcg_socket_limit_enabled;
EXPORT_SYMBOL(memcg_socket_limit_enabled);
/*
diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c
index d05559d..0c28508 100644
--- a/net/core/sysctl_net_core.c
+++ b/net/core/sysctl_net_core.c
@@ -69,9 +69,9 @@ static int rps_sock_flow_sysctl(ctl_table *table, int write,
if (sock_table != orig_sock_table) {
rcu_assign_pointer(rps_sock_flow_table, sock_table);
if (sock_table)
- jump_label_inc(&rps_needed);
+ static_key_slow_inc(&rps_needed);
if (orig_sock_table) {
- jump_label_dec(&rps_needed);
+ static_key_slow_dec(&rps_needed);
synchronize_rcu();
vfree(orig_sock_table);
}
diff --git a/net/ipv4/tcp_memcontrol.c b/net/ipv4/tcp_memcontrol.c
index 4997878..602fb30 100644
--- a/net/ipv4/tcp_memcontrol.c
+++ b/net/ipv4/tcp_memcontrol.c
@@ -111,7 +111,7 @@ void tcp_destroy_cgroup(struct cgroup *cgrp, struct cgroup_subsys *ss)
val = res_counter_read_u64(&tcp->tcp_memory_allocated, RES_LIMIT);
if (val != RESOURCE_MAX)
- jump_label_dec(&memcg_socket_limit_enabled);
+ static_key_slow_dec(&memcg_socket_limit_enabled);
}
EXPORT_SYMBOL(tcp_destroy_cgroup);
@@ -143,9 +143,9 @@ static int tcp_update_limit(struct mem_cgroup *memcg, u64 val)
net->ipv4.sysctl_tcp_mem[i]);
if (val == RESOURCE_MAX && old_lim != RESOURCE_MAX)
- jump_label_dec(&memcg_socket_limit_enabled);
+ static_key_slow_dec(&memcg_socket_limit_enabled);
else if (old_lim == RESOURCE_MAX && val != RESOURCE_MAX)
- jump_label_inc(&memcg_socket_limit_enabled);
+ static_key_slow_inc(&memcg_socket_limit_enabled);
return 0;
}
diff --git a/net/netfilter/core.c b/net/netfilter/core.c
index b4e8ff0..e1b7e05 100644
--- a/net/netfilter/core.c
+++ b/net/netfilter/core.c
@@ -56,7 +56,7 @@ struct list_head nf_hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS] __read_mostly;
EXPORT_SYMBOL(nf_hooks);
#if defined(CONFIG_JUMP_LABEL)
-struct jump_label_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
+struct static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
EXPORT_SYMBOL(nf_hooks_needed);
#endif
@@ -77,7 +77,7 @@ int nf_register_hook(struct nf_hook_ops *reg)
list_add_rcu(®->list, elem->list.prev);
mutex_unlock(&nf_hook_mutex);
#if defined(CONFIG_JUMP_LABEL)
- jump_label_inc(&nf_hooks_needed[reg->pf][reg->hooknum]);
+ static_key_slow_inc(&nf_hooks_needed[reg->pf][reg->hooknum]);
#endif
return 0;
}
@@ -89,7 +89,7 @@ void nf_unregister_hook(struct nf_hook_ops *reg)
list_del_rcu(®->list);
mutex_unlock(&nf_hook_mutex);
#if defined(CONFIG_JUMP_LABEL)
- jump_label_dec(&nf_hooks_needed[reg->pf][reg->hooknum]);
+ static_key_slow_dec(&nf_hooks_needed[reg->pf][reg->hooknum]);
#endif
synchronize_net();
}
* Linus Torvalds <[email protected]> wrote:
> Seriously, I don't understand why you don't just use the
> obvious name. The data structure is named "static_key". The
> things that change it are named "static_key_inc()" or
> something. So a name like "static_key_true()" is simply
> *better*, isn't it?
Yes, you are right, in hindsight it's indeed obviously and
trivially better :-/
> It's not just about less confusion, it's actually about just
> having consistent naming.
Okay. I sent out the slightly reworked patch that gets rid of
this confusion and makes it all consistent.
Thanks,
Ingo
* Jason Baron <[email protected]> wrote:
> On Thu, Feb 23, 2012 at 06:18:42PM -0500, Mathieu Desnoyers wrote:
> > * Ingo Molnar ([email protected]) wrote:
> > >
> > > * Ingo Molnar <[email protected]> wrote:
> > >
> > > > So, a modified scheme would be:
> > > >
> > > > #include <linux/static_key.h>
> > > >
> > > > struct static_key key = STATIC_KEY_INIT_TRUE;
> > > >
> > > > if (static_key_false(&key))
> > > > do unlikely code
> > > > else
> > > > do likely code
> > > >
> > > > Or:
> > > >
> > > > if (static_key_true(&key))
> > > > do likely code
> > > > else
> > > > do unlikely code
> > > >
> > > > The static key is modified via:
> > > >
> > > > static_key_slow_inc(&key);
> > > > ...
> > > > static_key_slow_dec(&key);
> > > >
> > > > Is that API fine? I'll rework the series to such an effect if
> > > > everyone agrees.
> > >
> > > I.e. something like the patch below on top of
> > > tip:perf/jump-labels.
> > >
> > > Untested - will test it and will refactor the series if
> > > everyone's happy.
> >
> > Hi Ingo,
> >
> > Reading your documentation updates makes me realise that adding the
> > "inline" keyword in there would make the whole thing even clearer:
> >
> > struct static_key key = STATIC_KEY_INLINE_TRUE_INIT;
> > struct static_key key = STATIC_KEY_INLINE_FALSE_INIT;
> >
> > static_key_inline_true() / static_key_inline_false()
> >
> > to show that the "true/false" in there does not mean that the key will
> > always be true or false (the key value can indeed by changed by calling
> > static_key_slow_inc/dec), but that the inlined path is either the true
> > of false branch.
> >
>
> Also, as part of the API, there is a test to check the branch
> direction - which was 'jump_label_true(key)', but is now also
> 'static_key_true(key)', [...]
Yeah, there is such an overlap - I've renamed it to
static_key_enabled(), which makes sense anyway as the original
was jump_label_enabled()..
Btw., shouldnt it be an inline function? Currently it's:
bool static_key_enabled(struct static_key *key)
{
return (atomic_read(&key->enabled) > 0);
}
which is the perfect candidate for inlining. The difference to
static_key_true() is the lack of the jump label patching and the
lack of an unlikely() hint.
> [...] so we are going to have to change either the branch site
> or the test for true/false name. The above
> 'static_key_inline_true/false' solves that.
It's generally good practice to make the mostly commonly used
method names the simplest/shortest names - i.e. I don't think we
should make it longer via adding an _inline to every use.
In that sense static_key_true() has pretty optimal length - we'd
like these tests to also be visually unintrusive.
So in the latest patch (still under testing, will push it out
soon) we have:
static_key_true()
static_key_false()
static_key_enabled()
> Also, I do like having a consistent 'static_key_*' prefix for
> the entire API - definitely an improvement for me.
Yeah.
Thanks,
Ingo
* Jason Baron <[email protected]> wrote:
> So, we could get rid of the '&' with something as simple as:
>
> #define very_unlikely(key) __very_unlikely(&key)
>
> However, it does seem potentially more error prone, b/c if
> 'key' is passed to a function, and they we do the
> very_unlikely() we end up with the address parameter (due to
> pass by value). That said, it doesn't look like anybody is
> using very_unlikely() in that manner in the tree, and we could
> document the usage.
>
> In any case, I did the conversion, to see what it would look
> like, if anybody is interested:
I agree that it's still error-prone - it also departs from how
we typically use C APIs in the kernel. With the static_key_*()
naming there's no desire to make it work like unlikely() anymore
and there's no need to pass in the object by value - passing by
reference is fine.
So I don't think we need this.
Thanks,
Ingo
On Fri, Feb 24, 2012 at 10:08:11AM +0100, Ingo Molnar wrote:
> * Jason Baron <[email protected]> wrote:
>
> > On Thu, Feb 23, 2012 at 06:18:42PM -0500, Mathieu Desnoyers wrote:
> > > * Ingo Molnar ([email protected]) wrote:
> > > >
> > > > * Ingo Molnar <[email protected]> wrote:
> > > >
> > > > > So, a modified scheme would be:
> > > > >
> > > > > #include <linux/static_key.h>
> > > > >
> > > > > struct static_key key = STATIC_KEY_INIT_TRUE;
> > > > >
> > > > > if (static_key_false(&key))
> > > > > do unlikely code
> > > > > else
> > > > > do likely code
> > > > >
> > > > > Or:
> > > > >
> > > > > if (static_key_true(&key))
> > > > > do likely code
> > > > > else
> > > > > do unlikely code
> > > > >
> > > > > The static key is modified via:
> > > > >
> > > > > static_key_slow_inc(&key);
> > > > > ...
> > > > > static_key_slow_dec(&key);
> > > > >
> > > > > Is that API fine? I'll rework the series to such an effect if
> > > > > everyone agrees.
> > > >
> > > > I.e. something like the patch below on top of
> > > > tip:perf/jump-labels.
> > > >
> > > > Untested - will test it and will refactor the series if
> > > > everyone's happy.
> > >
> > > Hi Ingo,
> > >
> > > Reading your documentation updates makes me realise that adding the
> > > "inline" keyword in there would make the whole thing even clearer:
> > >
> > > struct static_key key = STATIC_KEY_INLINE_TRUE_INIT;
> > > struct static_key key = STATIC_KEY_INLINE_FALSE_INIT;
> > >
> > > static_key_inline_true() / static_key_inline_false()
> > >
> > > to show that the "true/false" in there does not mean that the key will
> > > always be true or false (the key value can indeed by changed by calling
> > > static_key_slow_inc/dec), but that the inlined path is either the true
> > > of false branch.
> > >
> >
> > Also, as part of the API, there is a test to check the branch
> > direction - which was 'jump_label_true(key)', but is now also
> > 'static_key_true(key)', [...]
>
> Yeah, there is such an overlap - I've renamed it to
> static_key_enabled(), which makes sense anyway as the original
> was jump_label_enabled()..
>
> Btw., shouldnt it be an inline function? Currently it's:
>
Yes. I've had thought that too. In fact, it is already 'static inline' for the
!JUMP_LABEL case. So we can probably just remove the function from the
.c and move the 'static inline' such that its defined for all cases.
> bool static_key_enabled(struct static_key *key)
> {
> return (atomic_read(&key->enabled) > 0);
> }
>
> which is the perfect candidate for inlining. The difference to
> static_key_true() is the lack of the jump label patching and the
> lack of an unlikely() hint.
>
> > [...] so we are going to have to change either the branch site
> > or the test for true/false name. The above
> > 'static_key_inline_true/false' solves that.
>
> It's generally good practice to make the mostly commonly used
> method names the simplest/shortest names - i.e. I don't think we
> should make it longer via adding an _inline to every use.
>
> In that sense static_key_true() has pretty optimal length - we'd
> like these tests to also be visually unintrusive.
>
> So in the latest patch (still under testing, will push it out
> soon) we have:
>
> static_key_true()
> static_key_false()
> static_key_enabled()
>
Ok. Looks good.
Acked-by: Jason Baron <[email protected]>
Thanks,
-Jason
* Ingo Molnar ([email protected]) wrote:
[...]
> > [...] so we are going to have to change either the branch site
> > or the test for true/false name. The above
> > 'static_key_inline_true/false' solves that.
>
> It's generally good practice to make the mostly commonly used
> method names the simplest/shortest names - i.e. I don't think we
> should make it longer via adding an _inline to every use.
>
> In that sense static_key_true() has pretty optimal length - we'd
> like these tests to also be visually unintrusive.
>
> So in the latest patch (still under testing, will push it out
> soon) we have:
>
> static_key_true()
> static_key_false()
> static_key_enabled()
Hi Ingo,
Yes, I think your proposal makes sense. The "_inline_" would add lots of
typing for little clarity value. And I guess people will get used to
these constructs.
Acked-by: Mathieu Desnoyers <[email protected]>
Thanks,
Mathieu
--
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com
On Fri, 2012-02-24 at 10:08 +0100, Ingo Molnar wrote:
> So in the latest patch (still under testing, will push it out
> soon) we have:
>
> static_key_true()
> static_key_false()
> static_key_enabled()
>
> > Also, I do like having a consistent 'static_key_*' prefix for
> > the entire API - definitely an improvement for me.
Acked-by: Steven Rostedt <[email protected]>
-- Steve
* Jason Baron <[email protected]> wrote:
> On Fri, Feb 24, 2012 at 10:08:11AM +0100, Ingo Molnar wrote:
> > * Jason Baron <[email protected]> wrote:
> >
> > > On Thu, Feb 23, 2012 at 06:18:42PM -0500, Mathieu Desnoyers wrote:
> > > > * Ingo Molnar ([email protected]) wrote:
> > > > >
> > > > > * Ingo Molnar <[email protected]> wrote:
> > > > >
> > > > > > So, a modified scheme would be:
> > > > > >
> > > > > > #include <linux/static_key.h>
> > > > > >
> > > > > > struct static_key key = STATIC_KEY_INIT_TRUE;
> > > > > >
> > > > > > if (static_key_false(&key))
> > > > > > do unlikely code
> > > > > > else
> > > > > > do likely code
> > > > > >
> > > > > > Or:
> > > > > >
> > > > > > if (static_key_true(&key))
> > > > > > do likely code
> > > > > > else
> > > > > > do unlikely code
> > > > > >
> > > > > > The static key is modified via:
> > > > > >
> > > > > > static_key_slow_inc(&key);
> > > > > > ...
> > > > > > static_key_slow_dec(&key);
> > > > > >
> > > > > > Is that API fine? I'll rework the series to such an effect if
> > > > > > everyone agrees.
> > > > >
> > > > > I.e. something like the patch below on top of
> > > > > tip:perf/jump-labels.
> > > > >
> > > > > Untested - will test it and will refactor the series if
> > > > > everyone's happy.
> > > >
> > > > Hi Ingo,
> > > >
> > > > Reading your documentation updates makes me realise that adding the
> > > > "inline" keyword in there would make the whole thing even clearer:
> > > >
> > > > struct static_key key = STATIC_KEY_INLINE_TRUE_INIT;
> > > > struct static_key key = STATIC_KEY_INLINE_FALSE_INIT;
> > > >
> > > > static_key_inline_true() / static_key_inline_false()
> > > >
> > > > to show that the "true/false" in there does not mean that the key will
> > > > always be true or false (the key value can indeed by changed by calling
> > > > static_key_slow_inc/dec), but that the inlined path is either the true
> > > > of false branch.
> > > >
> > >
> > > Also, as part of the API, there is a test to check the branch
> > > direction - which was 'jump_label_true(key)', but is now also
> > > 'static_key_true(key)', [...]
> >
> > Yeah, there is such an overlap - I've renamed it to
> > static_key_enabled(), which makes sense anyway as the original
> > was jump_label_enabled()..
> >
> > Btw., shouldnt it be an inline function? Currently it's:
> >
>
> Yes. I've had thought that too. In fact, it is already 'static
> inline' for the !JUMP_LABEL case. So we can probably just
> remove the function from the .c and move the 'static inline'
> such that its defined for all cases.
Yep. Mind sending a patch for that, against latest -tip?
Thanks,
Ingo
Commit-ID: fadf0464b83f91ba021a358c0238a0810c0d2a0b
Gitweb: http://git.kernel.org/tip/fadf0464b83f91ba021a358c0238a0810c0d2a0b
Author: Jason Baron <[email protected]>
AuthorDate: Tue, 21 Feb 2012 15:02:53 -0500
Committer: Ingo Molnar <[email protected]>
CommitDate: Wed, 22 Feb 2012 07:59:39 +0100
jump label: Add a WARN() if jump label key count goes negative
The count on a jump label key should never go negative. Add a
WARN() to check for this condition.
Signed-off-by: Jason Baron <[email protected]>
Cc: Gleb Natapov <[email protected]>
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Link: http://lkml.kernel.org/r/3c68556121be4d1920417a3fe367da1ec38246b4.1329851692.git.jbaron@redhat.com
Signed-off-by: Ingo Molnar <[email protected]>
---
kernel/jump_label.c | 5 ++++-
1 files changed, 4 insertions(+), 1 deletions(-)
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 01d3b70..ed9654f 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -76,8 +76,11 @@ EXPORT_SYMBOL_GPL(jump_label_inc);
static void __jump_label_dec(struct jump_label_key *key,
unsigned long rate_limit, struct delayed_work *work)
{
- if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex))
+ if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) {
+ WARN(atomic_read(&key->enabled) < 0,
+ "jump label: negative count!\n");
return;
+ }
if (rate_limit) {
atomic_inc(&key->enabled);
Commit-ID: a746e3cc984b0aa5b620dd07c1a433283b1835cf
Gitweb: http://git.kernel.org/tip/a746e3cc984b0aa5b620dd07c1a433283b1835cf
Author: Jason Baron <[email protected]>
AuthorDate: Tue, 21 Feb 2012 15:02:57 -0500
Committer: Ingo Molnar <[email protected]>
CommitDate: Wed, 22 Feb 2012 07:59:40 +0100
jump label: Fix compiler warning
While cross-compiling on sparc64, I found:
kernel/jump_label.c: In function 'jump_label_update':
kernel/jump_label.c:447:40: warning: cast from pointer to
integer of different size [-Wpointer-to-int-cast]
Fix by casting to 'unsigned long'.
Signed-off-by: Jason Baron <[email protected]>
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Link: http://lkml.kernel.org/r/08026cbc6df80619cae833ef1ebbbc43efab69ab.1329851692.git.jbaron@redhat.com
Signed-off-by: Ingo Molnar <[email protected]>
---
kernel/jump_label.c | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index ed9654f..543782e 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -424,7 +424,7 @@ static void jump_label_update(struct jump_label_key *key, int enable)
struct jump_entry *entry = key->entries, *stop = __stop___jump_table;
#ifdef CONFIG_MODULES
- struct module *mod = __module_address((jump_label_t)key);
+ struct module *mod = __module_address((unsigned long)key);
__jump_label_mod_update(key, enable);
Commit-ID: 1cfa60dc7d7c7cc774a44eee47ff135a644a1f31
Gitweb: http://git.kernel.org/tip/1cfa60dc7d7c7cc774a44eee47ff135a644a1f31
Author: Jason Baron <[email protected]>
AuthorDate: Tue, 21 Feb 2012 15:03:30 -0500
Committer: Ingo Molnar <[email protected]>
CommitDate: Fri, 24 Feb 2012 09:12:19 +0100
static keys: Add docs better explaining the whole 'struct static_key' mechanism
Add better documentation for static keys.
Signed-off-by: Jason Baron <[email protected]>
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Link: http://lkml.kernel.org/r/52570e566e5f1914f27b67e4eafb5781b8f9f9db.1329851692.git.jbaron@redhat.com
[ Added a 'Summary' section and rewrote it to explain static keys ]
Signed-off-by: Ingo Molnar <[email protected]>
---
Documentation/static-keys.txt | 286 +++++++++++++++++++++++++++++++++++++++++
1 files changed, 286 insertions(+), 0 deletions(-)
diff --git a/Documentation/static-keys.txt b/Documentation/static-keys.txt
new file mode 100644
index 0000000..d93f3c0
--- /dev/null
+++ b/Documentation/static-keys.txt
@@ -0,0 +1,286 @@
+ Static Keys
+ -----------
+
+By: Jason Baron <[email protected]>
+
+0) Abstract
+
+Static keys allows the inclusion of seldom used features in
+performance-sensitive fast-path kernel code, via a GCC feature and a code
+patching technique. A quick example:
+
+ struct static_key key = STATIC_KEY_INIT_FALSE;
+
+ ...
+
+ if (static_key_false(&key))
+ do unlikely code
+ else
+ do likely code
+
+ ...
+ static_key_slow_inc();
+ ...
+ static_key_slow_inc();
+ ...
+
+The static_key_false() branch will be generated into the code with as little
+impact to the likely code path as possible.
+
+
+1) Motivation
+
+
+Currently, tracepoints are implemented using a conditional branch. The
+conditional check requires checking a global variable for each tracepoint.
+Although the overhead of this check is small, it increases when the memory
+cache comes under pressure (memory cache lines for these global variables may
+be shared with other memory accesses). As we increase the number of tracepoints
+in the kernel this overhead may become more of an issue. In addition,
+tracepoints are often dormant (disabled) and provide no direct kernel
+functionality. Thus, it is highly desirable to reduce their impact as much as
+possible. Although tracepoints are the original motivation for this work, other
+kernel code paths should be able to make use of the static keys facility.
+
+
+2) Solution
+
+
+gcc (v4.5) adds a new 'asm goto' statement that allows branching to a label:
+
+http://gcc.gnu.org/ml/gcc-patches/2009-07/msg01556.html
+
+Using the 'asm goto', we can create branches that are either taken or not taken
+by default, without the need to check memory. Then, at run-time, we can patch
+the branch site to change the branch direction.
+
+For example, if we have a simple branch that is disabled by default:
+
+ if (static_key_false(&key))
+ printk("I am the true branch\n");
+
+Thus, by default the 'printk' will not be emitted. And the code generated will
+consist of a single atomic 'no-op' instruction (5 bytes on x86), in the
+straight-line code path. When the branch is 'flipped', we will patch the
+'no-op' in the straight-line codepath with a 'jump' instruction to the
+out-of-line true branch. Thus, changing branch direction is expensive but
+branch selection is basically 'free'. That is the basic tradeoff of this
+optimization.
+
+This lowlevel patching mechanism is called 'jump label patching', and it gives
+the basis for the static keys facility.
+
+3) Static key label API, usage and examples:
+
+
+In order to make use of this optimization you must first define a key:
+
+ struct static_key key;
+
+Which is initialized as:
+
+ struct static_key key = STATIC_KEY_INIT_TRUE;
+
+or:
+
+ struct static_key key = STATIC_KEY_INIT_FALSE;
+
+If the key is not initialized, it is default false. The 'struct static_key',
+must be a 'global'. That is, it can't be allocated on the stack or dynamically
+allocated at run-time.
+
+The key is then used in code as:
+
+ if (static_key_false(&key))
+ do unlikely code
+ else
+ do likely code
+
+Or:
+
+ if (static_key_true(&key))
+ do likely code
+ else
+ do unlikely code
+
+A key that is initialized via 'STATIC_KEY_INIT_FALSE', must be used in a
+'static_key_false()' construct. Likewise, a key initialized via
+'STATIC_KEY_INIT_TRUE' must be used in a 'static_key_true()' construct. A
+single key can be used in many branches, but all the branches must match the
+way that the key has been initialized.
+
+The branch(es) can then be switched via:
+
+ static_key_slow_inc(&key);
+ ...
+ static_key_slow_dec(&key);
+
+Thus, 'static_key_slow_inc()' means 'make the branch true', and
+'static_key_slow_dec()' means 'make the the branch false' with appropriate
+reference counting. For example, if the key is initialized true, a
+static_key_slow_dec(), will switch the branch to false. And a subsequent
+static_key_slow_inc(), will change the branch back to true. Likewise, if the
+key is initialized false, a 'static_key_slow_inc()', will change the branch to
+true. And then a 'static_key_slow_dec()', will again make the branch false.
+
+An example usage in the kernel is the implementation of tracepoints:
+
+ static inline void trace_##name(proto) \
+ { \
+ if (static_key_false(&__tracepoint_##name.key)) \
+ __DO_TRACE(&__tracepoint_##name, \
+ TP_PROTO(data_proto), \
+ TP_ARGS(data_args), \
+ TP_CONDITION(cond)); \
+ }
+
+Tracepoints are disabled by default, and can be placed in performance critical
+pieces of the kernel. Thus, by using a static key, the tracepoints can have
+absolutely minimal impact when not in use.
+
+
+4) Architecture level code patching interface, 'jump labels'
+
+
+There are a few functions and macros that architectures must implement in order
+to take advantage of this optimization. If there is no architecture support, we
+simply fall back to a traditional, load, test, and jump sequence.
+
+* select HAVE_ARCH_JUMP_LABEL, see: arch/x86/Kconfig
+
+* #define JUMP_LABEL_NOP_SIZE, see: arch/x86/include/asm/jump_label.h
+
+* __always_inline bool arch_static_branch(struct static_key *key), see:
+ arch/x86/include/asm/jump_label.h
+
+* void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type),
+ see: arch/x86/kernel/jump_label.c
+
+* __init_or_module void arch_jump_label_transform_static(struct jump_entry *entry, enum jump_label_type type),
+ see: arch/x86/kernel/jump_label.c
+
+
+* struct jump_entry, see: arch/x86/include/asm/jump_label.h
+
+
+5) Static keys / jump label analysis, results (x86_64):
+
+
+As an example, let's add the following branch to 'getppid()', such that the
+system call now looks like:
+
+SYSCALL_DEFINE0(getppid)
+{
+ int pid;
+
++ if (static_key_false(&key))
++ printk("I am the true branch\n");
+
+ rcu_read_lock();
+ pid = task_tgid_vnr(rcu_dereference(current->real_parent));
+ rcu_read_unlock();
+
+ return pid;
+}
+
+The resulting instructions with jump labels generated by GCC is:
+
+ffffffff81044290 <sys_getppid>:
+ffffffff81044290: 55 push %rbp
+ffffffff81044291: 48 89 e5 mov %rsp,%rbp
+ffffffff81044294: e9 00 00 00 00 jmpq ffffffff81044299 <sys_getppid+0x9>
+ffffffff81044299: 65 48 8b 04 25 c0 b6 mov %gs:0xb6c0,%rax
+ffffffff810442a0: 00 00
+ffffffff810442a2: 48 8b 80 80 02 00 00 mov 0x280(%rax),%rax
+ffffffff810442a9: 48 8b 80 b0 02 00 00 mov 0x2b0(%rax),%rax
+ffffffff810442b0: 48 8b b8 e8 02 00 00 mov 0x2e8(%rax),%rdi
+ffffffff810442b7: e8 f4 d9 00 00 callq ffffffff81051cb0 <pid_vnr>
+ffffffff810442bc: 5d pop %rbp
+ffffffff810442bd: 48 98 cltq
+ffffffff810442bf: c3 retq
+ffffffff810442c0: 48 c7 c7 e3 54 98 81 mov $0xffffffff819854e3,%rdi
+ffffffff810442c7: 31 c0 xor %eax,%eax
+ffffffff810442c9: e8 71 13 6d 00 callq ffffffff8171563f <printk>
+ffffffff810442ce: eb c9 jmp ffffffff81044299 <sys_getppid+0x9>
+
+Without the jump label optimization it looks like:
+
+ffffffff810441f0 <sys_getppid>:
+ffffffff810441f0: 8b 05 8a 52 d8 00 mov 0xd8528a(%rip),%eax # ffffffff81dc9480 <key>
+ffffffff810441f6: 55 push %rbp
+ffffffff810441f7: 48 89 e5 mov %rsp,%rbp
+ffffffff810441fa: 85 c0 test %eax,%eax
+ffffffff810441fc: 75 27 jne ffffffff81044225 <sys_getppid+0x35>
+ffffffff810441fe: 65 48 8b 04 25 c0 b6 mov %gs:0xb6c0,%rax
+ffffffff81044205: 00 00
+ffffffff81044207: 48 8b 80 80 02 00 00 mov 0x280(%rax),%rax
+ffffffff8104420e: 48 8b 80 b0 02 00 00 mov 0x2b0(%rax),%rax
+ffffffff81044215: 48 8b b8 e8 02 00 00 mov 0x2e8(%rax),%rdi
+ffffffff8104421c: e8 2f da 00 00 callq ffffffff81051c50 <pid_vnr>
+ffffffff81044221: 5d pop %rbp
+ffffffff81044222: 48 98 cltq
+ffffffff81044224: c3 retq
+ffffffff81044225: 48 c7 c7 13 53 98 81 mov $0xffffffff81985313,%rdi
+ffffffff8104422c: 31 c0 xor %eax,%eax
+ffffffff8104422e: e8 60 0f 6d 00 callq ffffffff81715193 <printk>
+ffffffff81044233: eb c9 jmp ffffffff810441fe <sys_getppid+0xe>
+ffffffff81044235: 66 66 2e 0f 1f 84 00 data32 nopw %cs:0x0(%rax,%rax,1)
+ffffffff8104423c: 00 00 00 00
+
+Thus, the disable jump label case adds a 'mov', 'test' and 'jne' instruction
+vs. the jump label case just has a 'no-op' or 'jmp 0'. (The jmp 0, is patched
+to a 5 byte atomic no-op instruction at boot-time.) Thus, the disabled jump
+label case adds:
+
+6 (mov) + 2 (test) + 2 (jne) = 10 - 5 (5 byte jump 0) = 5 addition bytes.
+
+If we then include the padding bytes, the jump label code saves, 16 total bytes
+of instruction memory for this small fucntion. In this case the non-jump label
+function is 80 bytes long. Thus, we have have saved 20% of the instruction
+footprint. We can in fact improve this even further, since the 5-byte no-op
+really can be a 2-byte no-op since we can reach the branch with a 2-byte jmp.
+However, we have not yet implemented optimal no-op sizes (they are currently
+hard-coded).
+
+Since there are a number of static key API uses in the scheduler paths,
+'pipe-test' (also known as 'perf bench sched pipe') can be used to show the
+performance improvement. Testing done on 3.3.0-rc2:
+
+jump label disabled:
+
+ Performance counter stats for 'bash -c /tmp/pipe-test' (50 runs):
+
+ 855.700314 task-clock # 0.534 CPUs utilized ( +- 0.11% )
+ 200,003 context-switches # 0.234 M/sec ( +- 0.00% )
+ 0 CPU-migrations # 0.000 M/sec ( +- 39.58% )
+ 487 page-faults # 0.001 M/sec ( +- 0.02% )
+ 1,474,374,262 cycles # 1.723 GHz ( +- 0.17% )
+ <not supported> stalled-cycles-frontend
+ <not supported> stalled-cycles-backend
+ 1,178,049,567 instructions # 0.80 insns per cycle ( +- 0.06% )
+ 208,368,926 branches # 243.507 M/sec ( +- 0.06% )
+ 5,569,188 branch-misses # 2.67% of all branches ( +- 0.54% )
+
+ 1.601607384 seconds time elapsed ( +- 0.07% )
+
+jump label enabled:
+
+ Performance counter stats for 'bash -c /tmp/pipe-test' (50 runs):
+
+ 841.043185 task-clock # 0.533 CPUs utilized ( +- 0.12% )
+ 200,004 context-switches # 0.238 M/sec ( +- 0.00% )
+ 0 CPU-migrations # 0.000 M/sec ( +- 40.87% )
+ 487 page-faults # 0.001 M/sec ( +- 0.05% )
+ 1,432,559,428 cycles # 1.703 GHz ( +- 0.18% )
+ <not supported> stalled-cycles-frontend
+ <not supported> stalled-cycles-backend
+ 1,175,363,994 instructions # 0.82 insns per cycle ( +- 0.04% )
+ 206,859,359 branches # 245.956 M/sec ( +- 0.04% )
+ 4,884,119 branch-misses # 2.36% of all branches ( +- 0.85% )
+
+ 1.579384366 seconds time elapsed
+
+The percentage of saved branches is .7%, and we've saved 12% on
+'branch-misses'. This is where we would expect to get the most savings, since
+this optimization is about reducing the number of branches. In addition, we've
+saved .2% on instructions, and 2.8% on cycles and 1.4% on elapsed time.
Commit-ID: c5905afb0ee6550b42c49213da1c22d67316c194
Gitweb: http://git.kernel.org/tip/c5905afb0ee6550b42c49213da1c22d67316c194
Author: Ingo Molnar <[email protected]>
AuthorDate: Fri, 24 Feb 2012 08:31:31 +0100
Committer: Ingo Molnar <[email protected]>
CommitDate: Fri, 24 Feb 2012 10:05:59 +0100
static keys: Introduce 'struct static_key', static_key_true()/false() and static_key_slow_[inc|dec]()
So here's a boot tested patch on top of Jason's series that does
all the cleanups I talked about and turns jump labels into a
more intuitive to use facility. It should also address the
various misconceptions and confusions that surround jump labels.
Typical usage scenarios:
#include <linux/static_key.h>
struct static_key key = STATIC_KEY_INIT_TRUE;
if (static_key_false(&key))
do unlikely code
else
do likely code
Or:
if (static_key_true(&key))
do likely code
else
do unlikely code
The static key is modified via:
static_key_slow_inc(&key);
...
static_key_slow_dec(&key);
The 'slow' prefix makes it abundantly clear that this is an
expensive operation.
I've updated all in-kernel code to use this everywhere. Note
that I (intentionally) have not pushed through the rename
blindly through to the lowest levels: the actual jump-label
patching arch facility should be named like that, so we want to
decouple jump labels from the static-key facility a bit.
On non-jump-label enabled architectures static keys default to
likely()/unlikely() branches.
Signed-off-by: Ingo Molnar <[email protected]>
Acked-by: Jason Baron <[email protected]>
Acked-by: Steven Rostedt <[email protected]>
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: Linus Torvalds <[email protected]>
Link: http://lkml.kernel.org/r/[email protected]
Signed-off-by: Ingo Molnar <[email protected]>
---
arch/Kconfig | 27 +++++--
arch/ia64/include/asm/paravirt.h | 6 +-
arch/ia64/kernel/paravirt.c | 4 +-
arch/mips/include/asm/jump_label.h | 2 +-
arch/powerpc/include/asm/jump_label.h | 2 +-
arch/s390/include/asm/jump_label.h | 2 +-
arch/sparc/include/asm/jump_label.h | 2 +-
arch/x86/include/asm/jump_label.h | 6 +-
arch/x86/include/asm/paravirt.h | 6 +-
arch/x86/kernel/kvm.c | 4 +-
arch/x86/kernel/paravirt.c | 4 +-
arch/x86/kvm/mmu_audit.c | 8 +-
include/linux/jump_label.h | 139 +++++++++++++++++++++++---------
include/linux/netdevice.h | 4 +-
include/linux/netfilter.h | 6 +-
include/linux/perf_event.h | 12 ++--
include/linux/static_key.h | 1 +
include/linux/tracepoint.h | 8 +-
include/net/sock.h | 6 +-
kernel/events/core.c | 16 ++--
kernel/jump_label.c | 128 +++++++++++++++++-------------
kernel/sched/core.c | 18 ++--
kernel/sched/fair.c | 8 +-
kernel/sched/sched.h | 14 ++--
kernel/tracepoint.c | 20 +++---
net/core/dev.c | 24 +++---
net/core/net-sysfs.c | 4 +-
net/core/sock.c | 4 +-
net/core/sysctl_net_core.c | 4 +-
net/ipv4/tcp_memcontrol.c | 6 +-
net/netfilter/core.c | 6 +-
31 files changed, 297 insertions(+), 204 deletions(-)
diff --git a/arch/Kconfig b/arch/Kconfig
index 4f55c73..5b448a7 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -47,18 +47,29 @@ config KPROBES
If in doubt, say "N".
config JUMP_LABEL
- bool "Optimize trace point call sites"
+ bool "Optimize very unlikely/likely branches"
depends on HAVE_ARCH_JUMP_LABEL
help
+ This option enables a transparent branch optimization that
+ makes certain almost-always-true or almost-always-false branch
+ conditions even cheaper to execute within the kernel.
+
+ Certain performance-sensitive kernel code, such as trace points,
+ scheduler functionality, networking code and KVM have such
+ branches and include support for this optimization technique.
+
If it is detected that the compiler has support for "asm goto",
- the kernel will compile trace point locations with just a
- nop instruction. When trace points are enabled, the nop will
- be converted to a jump to the trace function. This technique
- lowers overhead and stress on the branch prediction of the
- processor.
-
- On i386, options added to the compiler flags may increase
- the size of the kernel slightly.
+ the kernel will compile such branches with just a nop
+ instruction. When the condition flag is toggled to true, the
+ nop will be converted to a jump instruction to execute the
+ conditional block of instructions.
+
+ This technique lowers overhead and stress on the branch prediction
+ of the processor and generally makes the kernel faster. The update
+ of the condition is slower, but those are always very rare.
+
+ ( On 32-bit x86, the necessary options added to the compiler
+ flags may increase the size of the kernel slightly. )
config OPTPROBES
def_bool y
diff --git a/arch/ia64/include/asm/paravirt.h b/arch/ia64/include/asm/paravirt.h
index 32551d3..b149b88 100644
--- a/arch/ia64/include/asm/paravirt.h
+++ b/arch/ia64/include/asm/paravirt.h
@@ -281,9 +281,9 @@ paravirt_init_missing_ticks_accounting(int cpu)
pv_time_ops.init_missing_ticks_accounting(cpu);
}
-struct jump_label_key;
-extern struct jump_label_key paravirt_steal_enabled;
-extern struct jump_label_key paravirt_steal_rq_enabled;
+struct static_key;
+extern struct static_key paravirt_steal_enabled;
+extern struct static_key paravirt_steal_rq_enabled;
static inline int
paravirt_do_steal_accounting(unsigned long *new_itm)
diff --git a/arch/ia64/kernel/paravirt.c b/arch/ia64/kernel/paravirt.c
index 1008682..1b22f6d 100644
--- a/arch/ia64/kernel/paravirt.c
+++ b/arch/ia64/kernel/paravirt.c
@@ -634,8 +634,8 @@ struct pv_irq_ops pv_irq_ops = {
* pv_time_ops
* time operations
*/
-struct jump_label_key paravirt_steal_enabled;
-struct jump_label_key paravirt_steal_rq_enabled;
+struct static_key paravirt_steal_enabled;
+struct static_key paravirt_steal_rq_enabled;
static int
ia64_native_do_steal_accounting(unsigned long *new_itm)
diff --git a/arch/mips/include/asm/jump_label.h b/arch/mips/include/asm/jump_label.h
index 1881b31..4d6d77e 100644
--- a/arch/mips/include/asm/jump_label.h
+++ b/arch/mips/include/asm/jump_label.h
@@ -20,7 +20,7 @@
#define WORD_INSN ".word"
#endif
-static __always_inline bool arch_static_branch(struct jump_label_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key)
{
asm goto("1:\tnop\n\t"
"nop\n\t"
diff --git a/arch/powerpc/include/asm/jump_label.h b/arch/powerpc/include/asm/jump_label.h
index 938986e..ae098c4 100644
--- a/arch/powerpc/include/asm/jump_label.h
+++ b/arch/powerpc/include/asm/jump_label.h
@@ -17,7 +17,7 @@
#define JUMP_ENTRY_TYPE stringify_in_c(FTR_ENTRY_LONG)
#define JUMP_LABEL_NOP_SIZE 4
-static __always_inline bool arch_static_branch(struct jump_label_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key)
{
asm goto("1:\n\t"
"nop\n\t"
diff --git a/arch/s390/include/asm/jump_label.h b/arch/s390/include/asm/jump_label.h
index 95a6cf2..6c32190 100644
--- a/arch/s390/include/asm/jump_label.h
+++ b/arch/s390/include/asm/jump_label.h
@@ -13,7 +13,7 @@
#define ASM_ALIGN ".balign 4"
#endif
-static __always_inline bool arch_static_branch(struct jump_label_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key)
{
asm goto("0: brcl 0,0\n"
".pushsection __jump_table, \"aw\"\n"
diff --git a/arch/sparc/include/asm/jump_label.h b/arch/sparc/include/asm/jump_label.h
index fc73a82..5080d16 100644
--- a/arch/sparc/include/asm/jump_label.h
+++ b/arch/sparc/include/asm/jump_label.h
@@ -7,7 +7,7 @@
#define JUMP_LABEL_NOP_SIZE 4
-static __always_inline bool arch_static_branch(struct jump_label_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key)
{
asm goto("1:\n\t"
"nop\n\t"
diff --git a/arch/x86/include/asm/jump_label.h b/arch/x86/include/asm/jump_label.h
index a32b18c..3a16c14 100644
--- a/arch/x86/include/asm/jump_label.h
+++ b/arch/x86/include/asm/jump_label.h
@@ -9,12 +9,12 @@
#define JUMP_LABEL_NOP_SIZE 5
-#define JUMP_LABEL_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t"
+#define STATIC_KEY_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t"
-static __always_inline bool arch_static_branch(struct jump_label_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key)
{
asm goto("1:"
- JUMP_LABEL_INITIAL_NOP
+ STATIC_KEY_INITIAL_NOP
".pushsection __jump_table, \"aw\" \n\t"
_ASM_ALIGN "\n\t"
_ASM_PTR "1b, %l[l_yes], %c0 \n\t"
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index a7d2db9..c0180fd 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -230,9 +230,9 @@ static inline unsigned long long paravirt_sched_clock(void)
return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock);
}
-struct jump_label_key;
-extern struct jump_label_key paravirt_steal_enabled;
-extern struct jump_label_key paravirt_steal_rq_enabled;
+struct static_key;
+extern struct static_key paravirt_steal_enabled;
+extern struct static_key paravirt_steal_rq_enabled;
static inline u64 paravirt_steal_clock(int cpu)
{
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index f0c6fd6..694d801 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -438,9 +438,9 @@ void __init kvm_guest_init(void)
static __init int activate_jump_labels(void)
{
if (has_steal_clock) {
- jump_label_inc(¶virt_steal_enabled);
+ static_key_slow_inc(¶virt_steal_enabled);
if (steal_acc)
- jump_label_inc(¶virt_steal_rq_enabled);
+ static_key_slow_inc(¶virt_steal_rq_enabled);
}
return 0;
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index d90272e..ada2f99 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -202,8 +202,8 @@ static void native_flush_tlb_single(unsigned long addr)
__native_flush_tlb_single(addr);
}
-struct jump_label_key paravirt_steal_enabled;
-struct jump_label_key paravirt_steal_rq_enabled;
+struct static_key paravirt_steal_enabled;
+struct static_key paravirt_steal_rq_enabled;
static u64 native_steal_clock(int cpu)
{
diff --git a/arch/x86/kvm/mmu_audit.c b/arch/x86/kvm/mmu_audit.c
index fe15dcc..ea7b4fd 100644
--- a/arch/x86/kvm/mmu_audit.c
+++ b/arch/x86/kvm/mmu_audit.c
@@ -234,7 +234,7 @@ static void audit_vcpu_spte(struct kvm_vcpu *vcpu)
}
static bool mmu_audit;
-static struct jump_label_key mmu_audit_key;
+static struct static_key mmu_audit_key;
static void __kvm_mmu_audit(struct kvm_vcpu *vcpu, int point)
{
@@ -250,7 +250,7 @@ static void __kvm_mmu_audit(struct kvm_vcpu *vcpu, int point)
static inline void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point)
{
- if (static_branch((&mmu_audit_key)))
+ if (static_key_false((&mmu_audit_key)))
__kvm_mmu_audit(vcpu, point);
}
@@ -259,7 +259,7 @@ static void mmu_audit_enable(void)
if (mmu_audit)
return;
- jump_label_inc(&mmu_audit_key);
+ static_key_slow_inc(&mmu_audit_key);
mmu_audit = true;
}
@@ -268,7 +268,7 @@ static void mmu_audit_disable(void)
if (!mmu_audit)
return;
- jump_label_dec(&mmu_audit_key);
+ static_key_slow_dec(&mmu_audit_key);
mmu_audit = false;
}
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index f7c6958..2172da2 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -9,15 +9,15 @@
*
* Jump labels provide an interface to generate dynamic branches using
* self-modifying code. Assuming toolchain and architecture support the result
- * of a "if (static_branch(&key))" statement is a unconditional branch (which
+ * of a "if (static_key_false(&key))" statement is a unconditional branch (which
* defaults to false - and the true block is placed out of line).
*
- * However at runtime we can change the 'static' branch target using
- * jump_label_{inc,dec}(). These function as a 'reference' count on the key
+ * However at runtime we can change the branch target using
+ * static_key_slow_{inc,dec}(). These function as a 'reference' count on the key
* object and for as long as there are references all branches referring to
* that particular key will point to the (out of line) true block.
*
- * Since this relies on modifying code the jump_label_{inc,dec}() functions
+ * Since this relies on modifying code the static_key_slow_{inc,dec}() functions
* must be considered absolute slow paths (machine wide synchronization etc.).
* OTOH, since the affected branches are unconditional their runtime overhead
* will be absolutely minimal, esp. in the default (off) case where the total
@@ -26,12 +26,26 @@
*
* When the control is directly exposed to userspace it is prudent to delay the
* decrement to avoid high frequency code modifications which can (and do)
- * cause significant performance degradation. Struct jump_label_key_deferred and
- * jump_label_dec_deferred() provide for this.
+ * cause significant performance degradation. Struct static_key_deferred and
+ * static_key_slow_dec_deferred() provide for this.
*
* Lacking toolchain and or architecture support, it falls back to a simple
* conditional branch.
- */
+ *
+ * struct static_key my_key = STATIC_KEY_INIT_TRUE;
+ *
+ * if (static_key_true(&my_key)) {
+ * }
+ *
+ * will result in the true case being in-line and starts the key with a single
+ * reference. Mixing static_key_true() and static_key_false() on the same key is not
+ * allowed.
+ *
+ * Not initializing the key (static data is initialized to 0s anyway) is the
+ * same as using STATIC_KEY_INIT_FALSE and static_key_false() is
+ * equivalent with static_branch().
+ *
+*/
#include <linux/types.h>
#include <linux/compiler.h>
@@ -39,16 +53,17 @@
#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
-struct jump_label_key {
+struct static_key {
atomic_t enabled;
+/* Set lsb bit to 1 if branch is default true, 0 ot */
struct jump_entry *entries;
#ifdef CONFIG_MODULES
- struct jump_label_mod *next;
+ struct static_key_mod *next;
#endif
};
-struct jump_label_key_deferred {
- struct jump_label_key key;
+struct static_key_deferred {
+ struct static_key key;
unsigned long timeout;
struct delayed_work work;
};
@@ -66,13 +81,34 @@ struct module;
#ifdef HAVE_JUMP_LABEL
-#ifdef CONFIG_MODULES
-#define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL, NULL}
-#else
-#define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL}
-#endif
+#define JUMP_LABEL_TRUE_BRANCH 1UL
+
+static
+inline struct jump_entry *jump_label_get_entries(struct static_key *key)
+{
+ return (struct jump_entry *)((unsigned long)key->entries
+ & ~JUMP_LABEL_TRUE_BRANCH);
+}
+
+static inline bool jump_label_get_branch_default(struct static_key *key)
+{
+ if ((unsigned long)key->entries & JUMP_LABEL_TRUE_BRANCH)
+ return true;
+ return false;
+}
+
+static __always_inline bool static_key_false(struct static_key *key)
+{
+ return arch_static_branch(key);
+}
-static __always_inline bool static_branch(struct jump_label_key *key)
+static __always_inline bool static_key_true(struct static_key *key)
+{
+ return !static_key_false(key);
+}
+
+/* Deprecated. Please use 'static_key_false() instead. */
+static __always_inline bool static_branch(struct static_key *key)
{
return arch_static_branch(key);
}
@@ -88,21 +124,24 @@ extern void arch_jump_label_transform(struct jump_entry *entry,
extern void arch_jump_label_transform_static(struct jump_entry *entry,
enum jump_label_type type);
extern int jump_label_text_reserved(void *start, void *end);
-extern void jump_label_inc(struct jump_label_key *key);
-extern void jump_label_dec(struct jump_label_key *key);
-extern void jump_label_dec_deferred(struct jump_label_key_deferred *key);
-extern bool jump_label_enabled(struct jump_label_key *key);
+extern void static_key_slow_inc(struct static_key *key);
+extern void static_key_slow_dec(struct static_key *key);
+extern void static_key_slow_dec_deferred(struct static_key_deferred *key);
+extern bool static_key_enabled(struct static_key *key);
extern void jump_label_apply_nops(struct module *mod);
-extern void jump_label_rate_limit(struct jump_label_key_deferred *key,
- unsigned long rl);
+extern void
+jump_label_rate_limit(struct static_key_deferred *key, unsigned long rl);
+
+#define STATIC_KEY_INIT_TRUE ((struct static_key) \
+ { .enabled = ATOMIC_INIT(1), .entries = (void *)1 })
+#define STATIC_KEY_INIT_FALSE ((struct static_key) \
+ { .enabled = ATOMIC_INIT(0), .entries = (void *)0 })
#else /* !HAVE_JUMP_LABEL */
#include <linux/atomic.h>
-#define JUMP_LABEL_INIT {ATOMIC_INIT(0)}
-
-struct jump_label_key {
+struct static_key {
atomic_t enabled;
};
@@ -110,30 +149,45 @@ static __always_inline void jump_label_init(void)
{
}
-struct jump_label_key_deferred {
- struct jump_label_key key;
+struct static_key_deferred {
+ struct static_key key;
};
-static __always_inline bool static_branch(struct jump_label_key *key)
+static __always_inline bool static_key_false(struct static_key *key)
+{
+ if (unlikely(atomic_read(&key->enabled)) > 0)
+ return true;
+ return false;
+}
+
+static __always_inline bool static_key_true(struct static_key *key)
{
- if (unlikely(atomic_read(&key->enabled)))
+ if (likely(atomic_read(&key->enabled)) > 0)
return true;
return false;
}
-static inline void jump_label_inc(struct jump_label_key *key)
+/* Deprecated. Please use 'static_key_false() instead. */
+static __always_inline bool static_branch(struct static_key *key)
+{
+ if (unlikely(atomic_read(&key->enabled)) > 0)
+ return true;
+ return false;
+}
+
+static inline void static_key_slow_inc(struct static_key *key)
{
atomic_inc(&key->enabled);
}
-static inline void jump_label_dec(struct jump_label_key *key)
+static inline void static_key_slow_dec(struct static_key *key)
{
atomic_dec(&key->enabled);
}
-static inline void jump_label_dec_deferred(struct jump_label_key_deferred *key)
+static inline void static_key_slow_dec_deferred(struct static_key_deferred *key)
{
- jump_label_dec(&key->key);
+ static_key_slow_dec(&key->key);
}
static inline int jump_label_text_reserved(void *start, void *end)
@@ -144,9 +198,9 @@ static inline int jump_label_text_reserved(void *start, void *end)
static inline void jump_label_lock(void) {}
static inline void jump_label_unlock(void) {}
-static inline bool jump_label_enabled(struct jump_label_key *key)
+static inline bool static_key_enabled(struct static_key *key)
{
- return !!atomic_read(&key->enabled);
+ return (atomic_read(&key->enabled) > 0);
}
static inline int jump_label_apply_nops(struct module *mod)
@@ -154,13 +208,20 @@ static inline int jump_label_apply_nops(struct module *mod)
return 0;
}
-static inline void jump_label_rate_limit(struct jump_label_key_deferred *key,
+static inline void
+jump_label_rate_limit(struct static_key_deferred *key,
unsigned long rl)
{
}
+
+#define STATIC_KEY_INIT_TRUE ((struct static_key) \
+ { .enabled = ATOMIC_INIT(1) })
+#define STATIC_KEY_INIT_FALSE ((struct static_key) \
+ { .enabled = ATOMIC_INIT(0) })
+
#endif /* HAVE_JUMP_LABEL */
-#define jump_label_key_enabled ((struct jump_label_key){ .enabled = ATOMIC_INIT(1), })
-#define jump_label_key_disabled ((struct jump_label_key){ .enabled = ATOMIC_INIT(0), })
+#define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
+#define jump_label_enabled static_key_enabled
#endif /* _LINUX_JUMP_LABEL_H */
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 0eac07c..7dfaae7 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -214,8 +214,8 @@ enum {
#include <linux/skbuff.h>
#ifdef CONFIG_RPS
-#include <linux/jump_label.h>
-extern struct jump_label_key rps_needed;
+#include <linux/static_key.h>
+extern struct static_key rps_needed;
#endif
struct neighbour;
diff --git a/include/linux/netfilter.h b/include/linux/netfilter.h
index b809265..29734be 100644
--- a/include/linux/netfilter.h
+++ b/include/linux/netfilter.h
@@ -163,13 +163,13 @@ extern struct ctl_path nf_net_ipv4_netfilter_sysctl_path[];
extern struct list_head nf_hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
#if defined(CONFIG_JUMP_LABEL)
-#include <linux/jump_label.h>
-extern struct jump_label_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
+#include <linux/static_key.h>
+extern struct static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
static inline bool nf_hooks_active(u_int8_t pf, unsigned int hook)
{
if (__builtin_constant_p(pf) &&
__builtin_constant_p(hook))
- return static_branch(&nf_hooks_needed[pf][hook]);
+ return static_key_false(&nf_hooks_needed[pf][hook]);
return !list_empty(&nf_hooks[pf][hook]);
}
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 412b790..0d21e6f 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -514,7 +514,7 @@ struct perf_guest_info_callbacks {
#include <linux/ftrace.h>
#include <linux/cpu.h>
#include <linux/irq_work.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
#include <linux/atomic.h>
#include <asm/local.h>
@@ -1038,7 +1038,7 @@ static inline int is_software_event(struct perf_event *event)
return event->pmu->task_ctx_nr == perf_sw_context;
}
-extern struct jump_label_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
+extern struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
extern void __perf_sw_event(u32, u64, struct pt_regs *, u64);
@@ -1066,7 +1066,7 @@ perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
{
struct pt_regs hot_regs;
- if (static_branch(&perf_swevent_enabled[event_id])) {
+ if (static_key_false(&perf_swevent_enabled[event_id])) {
if (!regs) {
perf_fetch_caller_regs(&hot_regs);
regs = &hot_regs;
@@ -1075,12 +1075,12 @@ perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
}
}
-extern struct jump_label_key_deferred perf_sched_events;
+extern struct static_key_deferred perf_sched_events;
static inline void perf_event_task_sched_in(struct task_struct *prev,
struct task_struct *task)
{
- if (static_branch(&perf_sched_events.key))
+ if (static_key_false(&perf_sched_events.key))
__perf_event_task_sched_in(prev, task);
}
@@ -1089,7 +1089,7 @@ static inline void perf_event_task_sched_out(struct task_struct *prev,
{
perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, NULL, 0);
- if (static_branch(&perf_sched_events.key))
+ if (static_key_false(&perf_sched_events.key))
__perf_event_task_sched_out(prev, next);
}
diff --git a/include/linux/static_key.h b/include/linux/static_key.h
new file mode 100644
index 0000000..27bd3f8
--- /dev/null
+++ b/include/linux/static_key.h
@@ -0,0 +1 @@
+#include <linux/jump_label.h>
diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h
index fc36da9..bd96ecd 100644
--- a/include/linux/tracepoint.h
+++ b/include/linux/tracepoint.h
@@ -17,7 +17,7 @@
#include <linux/errno.h>
#include <linux/types.h>
#include <linux/rcupdate.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
struct module;
struct tracepoint;
@@ -29,7 +29,7 @@ struct tracepoint_func {
struct tracepoint {
const char *name; /* Tracepoint name */
- struct jump_label_key key;
+ struct static_key key;
void (*regfunc)(void);
void (*unregfunc)(void);
struct tracepoint_func __rcu *funcs;
@@ -145,7 +145,7 @@ static inline void tracepoint_synchronize_unregister(void)
extern struct tracepoint __tracepoint_##name; \
static inline void trace_##name(proto) \
{ \
- if (static_branch(&__tracepoint_##name.key)) \
+ if (static_key_false(&__tracepoint_##name.key)) \
__DO_TRACE(&__tracepoint_##name, \
TP_PROTO(data_proto), \
TP_ARGS(data_args), \
@@ -188,7 +188,7 @@ static inline void tracepoint_synchronize_unregister(void)
__attribute__((section("__tracepoints_strings"))) = #name; \
struct tracepoint __tracepoint_##name \
__attribute__((section("__tracepoints"))) = \
- { __tpstrtab_##name, JUMP_LABEL_INIT, reg, unreg, NULL };\
+ { __tpstrtab_##name, STATIC_KEY_INIT_FALSE, reg, unreg, NULL };\
static struct tracepoint * const __tracepoint_ptr_##name __used \
__attribute__((section("__tracepoints_ptrs"))) = \
&__tracepoint_##name;
diff --git a/include/net/sock.h b/include/net/sock.h
index 91c1c8b..dcde2d9 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -55,7 +55,7 @@
#include <linux/uaccess.h>
#include <linux/memcontrol.h>
#include <linux/res_counter.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
#include <linux/filter.h>
#include <linux/rculist_nulls.h>
@@ -924,13 +924,13 @@ inline void sk_refcnt_debug_release(const struct sock *sk)
#endif /* SOCK_REFCNT_DEBUG */
#if defined(CONFIG_CGROUP_MEM_RES_CTLR_KMEM) && defined(CONFIG_NET)
-extern struct jump_label_key memcg_socket_limit_enabled;
+extern struct static_key memcg_socket_limit_enabled;
static inline struct cg_proto *parent_cg_proto(struct proto *proto,
struct cg_proto *cg_proto)
{
return proto->proto_cgroup(parent_mem_cgroup(cg_proto->memcg));
}
-#define mem_cgroup_sockets_enabled static_branch(&memcg_socket_limit_enabled)
+#define mem_cgroup_sockets_enabled static_key_false(&memcg_socket_limit_enabled)
#else
#define mem_cgroup_sockets_enabled 0
static inline struct cg_proto *parent_cg_proto(struct proto *proto,
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 7c3b9de..5e0f8bb 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -128,7 +128,7 @@ enum event_type_t {
* perf_sched_events : >0 events exist
* perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
*/
-struct jump_label_key_deferred perf_sched_events __read_mostly;
+struct static_key_deferred perf_sched_events __read_mostly;
static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
static atomic_t nr_mmap_events __read_mostly;
@@ -2769,7 +2769,7 @@ static void free_event(struct perf_event *event)
if (!event->parent) {
if (event->attach_state & PERF_ATTACH_TASK)
- jump_label_dec_deferred(&perf_sched_events);
+ static_key_slow_dec_deferred(&perf_sched_events);
if (event->attr.mmap || event->attr.mmap_data)
atomic_dec(&nr_mmap_events);
if (event->attr.comm)
@@ -2780,7 +2780,7 @@ static void free_event(struct perf_event *event)
put_callchain_buffers();
if (is_cgroup_event(event)) {
atomic_dec(&per_cpu(perf_cgroup_events, event->cpu));
- jump_label_dec_deferred(&perf_sched_events);
+ static_key_slow_dec_deferred(&perf_sched_events);
}
}
@@ -4982,7 +4982,7 @@ fail:
return err;
}
-struct jump_label_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
+struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
static void sw_perf_event_destroy(struct perf_event *event)
{
@@ -4990,7 +4990,7 @@ static void sw_perf_event_destroy(struct perf_event *event)
WARN_ON(event->parent);
- jump_label_dec(&perf_swevent_enabled[event_id]);
+ static_key_slow_dec(&perf_swevent_enabled[event_id]);
swevent_hlist_put(event);
}
@@ -5020,7 +5020,7 @@ static int perf_swevent_init(struct perf_event *event)
if (err)
return err;
- jump_label_inc(&perf_swevent_enabled[event_id]);
+ static_key_slow_inc(&perf_swevent_enabled[event_id]);
event->destroy = sw_perf_event_destroy;
}
@@ -5843,7 +5843,7 @@ done:
if (!event->parent) {
if (event->attach_state & PERF_ATTACH_TASK)
- jump_label_inc(&perf_sched_events.key);
+ static_key_slow_inc(&perf_sched_events.key);
if (event->attr.mmap || event->attr.mmap_data)
atomic_inc(&nr_mmap_events);
if (event->attr.comm)
@@ -6081,7 +6081,7 @@ SYSCALL_DEFINE5(perf_event_open,
* - that may need work on context switch
*/
atomic_inc(&per_cpu(perf_cgroup_events, event->cpu));
- jump_label_inc(&perf_sched_events.key);
+ static_key_slow_inc(&perf_sched_events.key);
}
/*
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 543782e..bf9dcad 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -12,7 +12,7 @@
#include <linux/slab.h>
#include <linux/sort.h>
#include <linux/err.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
#ifdef HAVE_JUMP_LABEL
@@ -29,10 +29,11 @@ void jump_label_unlock(void)
mutex_unlock(&jump_label_mutex);
}
-bool jump_label_enabled(struct jump_label_key *key)
+bool static_key_enabled(struct static_key *key)
{
- return !!atomic_read(&key->enabled);
+ return (atomic_read(&key->enabled) > 0);
}
+EXPORT_SYMBOL_GPL(static_key_enabled);
static int jump_label_cmp(const void *a, const void *b)
{
@@ -58,22 +59,26 @@ jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop)
sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL);
}
-static void jump_label_update(struct jump_label_key *key, int enable);
+static void jump_label_update(struct static_key *key, int enable);
-void jump_label_inc(struct jump_label_key *key)
+void static_key_slow_inc(struct static_key *key)
{
if (atomic_inc_not_zero(&key->enabled))
return;
jump_label_lock();
- if (atomic_read(&key->enabled) == 0)
- jump_label_update(key, JUMP_LABEL_ENABLE);
+ if (atomic_read(&key->enabled) == 0) {
+ if (!jump_label_get_branch_default(key))
+ jump_label_update(key, JUMP_LABEL_ENABLE);
+ else
+ jump_label_update(key, JUMP_LABEL_DISABLE);
+ }
atomic_inc(&key->enabled);
jump_label_unlock();
}
-EXPORT_SYMBOL_GPL(jump_label_inc);
+EXPORT_SYMBOL_GPL(static_key_slow_inc);
-static void __jump_label_dec(struct jump_label_key *key,
+static void __static_key_slow_dec(struct static_key *key,
unsigned long rate_limit, struct delayed_work *work)
{
if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) {
@@ -85,32 +90,35 @@ static void __jump_label_dec(struct jump_label_key *key,
if (rate_limit) {
atomic_inc(&key->enabled);
schedule_delayed_work(work, rate_limit);
- } else
- jump_label_update(key, JUMP_LABEL_DISABLE);
-
+ } else {
+ if (!jump_label_get_branch_default(key))
+ jump_label_update(key, JUMP_LABEL_DISABLE);
+ else
+ jump_label_update(key, JUMP_LABEL_ENABLE);
+ }
jump_label_unlock();
}
-EXPORT_SYMBOL_GPL(jump_label_dec);
static void jump_label_update_timeout(struct work_struct *work)
{
- struct jump_label_key_deferred *key =
- container_of(work, struct jump_label_key_deferred, work.work);
- __jump_label_dec(&key->key, 0, NULL);
+ struct static_key_deferred *key =
+ container_of(work, struct static_key_deferred, work.work);
+ __static_key_slow_dec(&key->key, 0, NULL);
}
-void jump_label_dec(struct jump_label_key *key)
+void static_key_slow_dec(struct static_key *key)
{
- __jump_label_dec(key, 0, NULL);
+ __static_key_slow_dec(key, 0, NULL);
}
+EXPORT_SYMBOL_GPL(static_key_slow_dec);
-void jump_label_dec_deferred(struct jump_label_key_deferred *key)
+void static_key_slow_dec_deferred(struct static_key_deferred *key)
{
- __jump_label_dec(&key->key, key->timeout, &key->work);
+ __static_key_slow_dec(&key->key, key->timeout, &key->work);
}
+EXPORT_SYMBOL_GPL(static_key_slow_dec_deferred);
-
-void jump_label_rate_limit(struct jump_label_key_deferred *key,
+void jump_label_rate_limit(struct static_key_deferred *key,
unsigned long rl)
{
key->timeout = rl;
@@ -153,7 +161,7 @@ void __weak __init_or_module arch_jump_label_transform_static(struct jump_entry
arch_jump_label_transform(entry, type);
}
-static void __jump_label_update(struct jump_label_key *key,
+static void __jump_label_update(struct static_key *key,
struct jump_entry *entry,
struct jump_entry *stop, int enable)
{
@@ -170,27 +178,40 @@ static void __jump_label_update(struct jump_label_key *key,
}
}
+static enum jump_label_type jump_label_type(struct static_key *key)
+{
+ bool true_branch = jump_label_get_branch_default(key);
+ bool state = static_key_enabled(key);
+
+ if ((!true_branch && state) || (true_branch && !state))
+ return JUMP_LABEL_ENABLE;
+
+ return JUMP_LABEL_DISABLE;
+}
+
void __init jump_label_init(void)
{
struct jump_entry *iter_start = __start___jump_table;
struct jump_entry *iter_stop = __stop___jump_table;
- struct jump_label_key *key = NULL;
+ struct static_key *key = NULL;
struct jump_entry *iter;
jump_label_lock();
jump_label_sort_entries(iter_start, iter_stop);
for (iter = iter_start; iter < iter_stop; iter++) {
- struct jump_label_key *iterk;
+ struct static_key *iterk;
- iterk = (struct jump_label_key *)(unsigned long)iter->key;
- arch_jump_label_transform_static(iter, jump_label_enabled(iterk) ?
- JUMP_LABEL_ENABLE : JUMP_LABEL_DISABLE);
+ iterk = (struct static_key *)(unsigned long)iter->key;
+ arch_jump_label_transform_static(iter, jump_label_type(iterk));
if (iterk == key)
continue;
key = iterk;
- key->entries = iter;
+ /*
+ * Set key->entries to iter, but preserve JUMP_LABEL_TRUE_BRANCH.
+ */
+ *((unsigned long *)&key->entries) += (unsigned long)iter;
#ifdef CONFIG_MODULES
key->next = NULL;
#endif
@@ -200,8 +221,8 @@ void __init jump_label_init(void)
#ifdef CONFIG_MODULES
-struct jump_label_mod {
- struct jump_label_mod *next;
+struct static_key_mod {
+ struct static_key_mod *next;
struct jump_entry *entries;
struct module *mod;
};
@@ -221,9 +242,9 @@ static int __jump_label_mod_text_reserved(void *start, void *end)
start, end);
}
-static void __jump_label_mod_update(struct jump_label_key *key, int enable)
+static void __jump_label_mod_update(struct static_key *key, int enable)
{
- struct jump_label_mod *mod = key->next;
+ struct static_key_mod *mod = key->next;
while (mod) {
struct module *m = mod->mod;
@@ -254,11 +275,7 @@ void jump_label_apply_nops(struct module *mod)
return;
for (iter = iter_start; iter < iter_stop; iter++) {
- struct jump_label_key *iterk;
-
- iterk = (struct jump_label_key *)(unsigned long)iter->key;
- arch_jump_label_transform_static(iter, jump_label_enabled(iterk) ?
- JUMP_LABEL_ENABLE : JUMP_LABEL_DISABLE);
+ arch_jump_label_transform_static(iter, JUMP_LABEL_DISABLE);
}
}
@@ -267,8 +284,8 @@ static int jump_label_add_module(struct module *mod)
struct jump_entry *iter_start = mod->jump_entries;
struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
struct jump_entry *iter;
- struct jump_label_key *key = NULL;
- struct jump_label_mod *jlm;
+ struct static_key *key = NULL;
+ struct static_key_mod *jlm;
/* if the module doesn't have jump label entries, just return */
if (iter_start == iter_stop)
@@ -277,28 +294,30 @@ static int jump_label_add_module(struct module *mod)
jump_label_sort_entries(iter_start, iter_stop);
for (iter = iter_start; iter < iter_stop; iter++) {
- if (iter->key == (jump_label_t)(unsigned long)key)
- continue;
+ struct static_key *iterk;
- key = (struct jump_label_key *)(unsigned long)iter->key;
+ iterk = (struct static_key *)(unsigned long)iter->key;
+ if (iterk == key)
+ continue;
+ key = iterk;
if (__module_address(iter->key) == mod) {
- atomic_set(&key->enabled, 0);
- key->entries = iter;
+ /*
+ * Set key->entries to iter, but preserve JUMP_LABEL_TRUE_BRANCH.
+ */
+ *((unsigned long *)&key->entries) += (unsigned long)iter;
key->next = NULL;
continue;
}
-
- jlm = kzalloc(sizeof(struct jump_label_mod), GFP_KERNEL);
+ jlm = kzalloc(sizeof(struct static_key_mod), GFP_KERNEL);
if (!jlm)
return -ENOMEM;
-
jlm->mod = mod;
jlm->entries = iter;
jlm->next = key->next;
key->next = jlm;
- if (jump_label_enabled(key))
+ if (jump_label_type(key) == JUMP_LABEL_ENABLE)
__jump_label_update(key, iter, iter_stop, JUMP_LABEL_ENABLE);
}
@@ -310,14 +329,14 @@ static void jump_label_del_module(struct module *mod)
struct jump_entry *iter_start = mod->jump_entries;
struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
struct jump_entry *iter;
- struct jump_label_key *key = NULL;
- struct jump_label_mod *jlm, **prev;
+ struct static_key *key = NULL;
+ struct static_key_mod *jlm, **prev;
for (iter = iter_start; iter < iter_stop; iter++) {
if (iter->key == (jump_label_t)(unsigned long)key)
continue;
- key = (struct jump_label_key *)(unsigned long)iter->key;
+ key = (struct static_key *)(unsigned long)iter->key;
if (__module_address(iter->key) == mod)
continue;
@@ -419,9 +438,10 @@ int jump_label_text_reserved(void *start, void *end)
return ret;
}
-static void jump_label_update(struct jump_label_key *key, int enable)
+static void jump_label_update(struct static_key *key, int enable)
{
- struct jump_entry *entry = key->entries, *stop = __stop___jump_table;
+ struct jump_entry *stop = __stop___jump_table;
+ struct jump_entry *entry = jump_label_get_entries(key);
#ifdef CONFIG_MODULES
struct module *mod = __module_address((unsigned long)key);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 5255c9d..112c682 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -162,13 +162,13 @@ static int sched_feat_show(struct seq_file *m, void *v)
#ifdef HAVE_JUMP_LABEL
-#define jump_label_key__true jump_label_key_enabled
-#define jump_label_key__false jump_label_key_disabled
+#define jump_label_key__true STATIC_KEY_INIT_TRUE
+#define jump_label_key__false STATIC_KEY_INIT_FALSE
#define SCHED_FEAT(name, enabled) \
jump_label_key__##enabled ,
-struct jump_label_key sched_feat_keys[__SCHED_FEAT_NR] = {
+struct static_key sched_feat_keys[__SCHED_FEAT_NR] = {
#include "features.h"
};
@@ -176,14 +176,14 @@ struct jump_label_key sched_feat_keys[__SCHED_FEAT_NR] = {
static void sched_feat_disable(int i)
{
- if (jump_label_enabled(&sched_feat_keys[i]))
- jump_label_dec(&sched_feat_keys[i]);
+ if (static_key_enabled(&sched_feat_keys[i]))
+ static_key_slow_dec(&sched_feat_keys[i]);
}
static void sched_feat_enable(int i)
{
- if (!jump_label_enabled(&sched_feat_keys[i]))
- jump_label_inc(&sched_feat_keys[i]);
+ if (!static_key_enabled(&sched_feat_keys[i]))
+ static_key_slow_inc(&sched_feat_keys[i]);
}
#else
static void sched_feat_disable(int i) { };
@@ -894,7 +894,7 @@ static void update_rq_clock_task(struct rq *rq, s64 delta)
delta -= irq_delta;
#endif
#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
- if (static_branch((¶virt_steal_rq_enabled))) {
+ if (static_key_false((¶virt_steal_rq_enabled))) {
u64 st;
steal = paravirt_steal_clock(cpu_of(rq));
@@ -2756,7 +2756,7 @@ void account_idle_time(cputime_t cputime)
static __always_inline bool steal_account_process_tick(void)
{
#ifdef CONFIG_PARAVIRT
- if (static_branch(¶virt_steal_enabled)) {
+ if (static_key_false(¶virt_steal_enabled)) {
u64 steal, st = 0;
steal = paravirt_steal_clock(smp_processor_id());
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 7c6414f..423547a 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1399,20 +1399,20 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued)
#ifdef CONFIG_CFS_BANDWIDTH
#ifdef HAVE_JUMP_LABEL
-static struct jump_label_key __cfs_bandwidth_used;
+static struct static_key __cfs_bandwidth_used;
static inline bool cfs_bandwidth_used(void)
{
- return static_branch(&__cfs_bandwidth_used);
+ return static_key_false(&__cfs_bandwidth_used);
}
void account_cfs_bandwidth_used(int enabled, int was_enabled)
{
/* only need to count groups transitioning between enabled/!enabled */
if (enabled && !was_enabled)
- jump_label_inc(&__cfs_bandwidth_used);
+ static_key_slow_inc(&__cfs_bandwidth_used);
else if (!enabled && was_enabled)
- jump_label_dec(&__cfs_bandwidth_used);
+ static_key_slow_dec(&__cfs_bandwidth_used);
}
#else /* HAVE_JUMP_LABEL */
static bool cfs_bandwidth_used(void)
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 98c0c26..b4cd6d8 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -611,7 +611,7 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
* Tunables that become constants when CONFIG_SCHED_DEBUG is off:
*/
#ifdef CONFIG_SCHED_DEBUG
-# include <linux/jump_label.h>
+# include <linux/static_key.h>
# define const_debug __read_mostly
#else
# define const_debug const
@@ -630,18 +630,18 @@ enum {
#undef SCHED_FEAT
#if defined(CONFIG_SCHED_DEBUG) && defined(HAVE_JUMP_LABEL)
-static __always_inline bool static_branch__true(struct jump_label_key *key)
+static __always_inline bool static_branch__true(struct static_key *key)
{
- return likely(static_branch(key)); /* Not out of line branch. */
+ return static_key_true(key); /* Not out of line branch. */
}
-static __always_inline bool static_branch__false(struct jump_label_key *key)
+static __always_inline bool static_branch__false(struct static_key *key)
{
- return unlikely(static_branch(key)); /* Out of line branch. */
+ return static_key_false(key); /* Out of line branch. */
}
#define SCHED_FEAT(name, enabled) \
-static __always_inline bool static_branch_##name(struct jump_label_key *key) \
+static __always_inline bool static_branch_##name(struct static_key *key) \
{ \
return static_branch__##enabled(key); \
}
@@ -650,7 +650,7 @@ static __always_inline bool static_branch_##name(struct jump_label_key *key) \
#undef SCHED_FEAT
-extern struct jump_label_key sched_feat_keys[__SCHED_FEAT_NR];
+extern struct static_key sched_feat_keys[__SCHED_FEAT_NR];
#define sched_feat(x) (static_branch_##x(&sched_feat_keys[__SCHED_FEAT_##x]))
#else /* !(SCHED_DEBUG && HAVE_JUMP_LABEL) */
#define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x))
diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c
index f1539de..d96ba22 100644
--- a/kernel/tracepoint.c
+++ b/kernel/tracepoint.c
@@ -25,7 +25,7 @@
#include <linux/err.h>
#include <linux/slab.h>
#include <linux/sched.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
extern struct tracepoint * const __start___tracepoints_ptrs[];
extern struct tracepoint * const __stop___tracepoints_ptrs[];
@@ -256,9 +256,9 @@ static void set_tracepoint(struct tracepoint_entry **entry,
{
WARN_ON(strcmp((*entry)->name, elem->name) != 0);
- if (elem->regfunc && !jump_label_enabled(&elem->key) && active)
+ if (elem->regfunc && !static_key_enabled(&elem->key) && active)
elem->regfunc();
- else if (elem->unregfunc && jump_label_enabled(&elem->key) && !active)
+ else if (elem->unregfunc && static_key_enabled(&elem->key) && !active)
elem->unregfunc();
/*
@@ -269,10 +269,10 @@ static void set_tracepoint(struct tracepoint_entry **entry,
* is used.
*/
rcu_assign_pointer(elem->funcs, (*entry)->funcs);
- if (active && !jump_label_enabled(&elem->key))
- jump_label_inc(&elem->key);
- else if (!active && jump_label_enabled(&elem->key))
- jump_label_dec(&elem->key);
+ if (active && !static_key_enabled(&elem->key))
+ static_key_slow_inc(&elem->key);
+ else if (!active && static_key_enabled(&elem->key))
+ static_key_slow_dec(&elem->key);
}
/*
@@ -283,11 +283,11 @@ static void set_tracepoint(struct tracepoint_entry **entry,
*/
static void disable_tracepoint(struct tracepoint *elem)
{
- if (elem->unregfunc && jump_label_enabled(&elem->key))
+ if (elem->unregfunc && static_key_enabled(&elem->key))
elem->unregfunc();
- if (jump_label_enabled(&elem->key))
- jump_label_dec(&elem->key);
+ if (static_key_enabled(&elem->key))
+ static_key_slow_dec(&elem->key);
rcu_assign_pointer(elem->funcs, NULL);
}
diff --git a/net/core/dev.c b/net/core/dev.c
index 115dee1..da7ce7f 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -134,7 +134,7 @@
#include <linux/inetdevice.h>
#include <linux/cpu_rmap.h>
#include <linux/net_tstamp.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
#include <net/flow_keys.h>
#include "net-sysfs.h"
@@ -1441,11 +1441,11 @@ int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
}
EXPORT_SYMBOL(call_netdevice_notifiers);
-static struct jump_label_key netstamp_needed __read_mostly;
+static struct static_key netstamp_needed __read_mostly;
#ifdef HAVE_JUMP_LABEL
-/* We are not allowed to call jump_label_dec() from irq context
+/* We are not allowed to call static_key_slow_dec() from irq context
* If net_disable_timestamp() is called from irq context, defer the
- * jump_label_dec() calls.
+ * static_key_slow_dec() calls.
*/
static atomic_t netstamp_needed_deferred;
#endif
@@ -1457,12 +1457,12 @@ void net_enable_timestamp(void)
if (deferred) {
while (--deferred)
- jump_label_dec(&netstamp_needed);
+ static_key_slow_dec(&netstamp_needed);
return;
}
#endif
WARN_ON(in_interrupt());
- jump_label_inc(&netstamp_needed);
+ static_key_slow_inc(&netstamp_needed);
}
EXPORT_SYMBOL(net_enable_timestamp);
@@ -1474,19 +1474,19 @@ void net_disable_timestamp(void)
return;
}
#endif
- jump_label_dec(&netstamp_needed);
+ static_key_slow_dec(&netstamp_needed);
}
EXPORT_SYMBOL(net_disable_timestamp);
static inline void net_timestamp_set(struct sk_buff *skb)
{
skb->tstamp.tv64 = 0;
- if (static_branch(&netstamp_needed))
+ if (static_key_false(&netstamp_needed))
__net_timestamp(skb);
}
#define net_timestamp_check(COND, SKB) \
- if (static_branch(&netstamp_needed)) { \
+ if (static_key_false(&netstamp_needed)) { \
if ((COND) && !(SKB)->tstamp.tv64) \
__net_timestamp(SKB); \
} \
@@ -2660,7 +2660,7 @@ EXPORT_SYMBOL(__skb_get_rxhash);
struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
EXPORT_SYMBOL(rps_sock_flow_table);
-struct jump_label_key rps_needed __read_mostly;
+struct static_key rps_needed __read_mostly;
static struct rps_dev_flow *
set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
@@ -2945,7 +2945,7 @@ int netif_rx(struct sk_buff *skb)
trace_netif_rx(skb);
#ifdef CONFIG_RPS
- if (static_branch(&rps_needed)) {
+ if (static_key_false(&rps_needed)) {
struct rps_dev_flow voidflow, *rflow = &voidflow;
int cpu;
@@ -3309,7 +3309,7 @@ int netif_receive_skb(struct sk_buff *skb)
return NET_RX_SUCCESS;
#ifdef CONFIG_RPS
- if (static_branch(&rps_needed)) {
+ if (static_key_false(&rps_needed)) {
struct rps_dev_flow voidflow, *rflow = &voidflow;
int cpu, ret;
diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c
index a1727cd..4955862 100644
--- a/net/core/net-sysfs.c
+++ b/net/core/net-sysfs.c
@@ -608,10 +608,10 @@ static ssize_t store_rps_map(struct netdev_rx_queue *queue,
spin_unlock(&rps_map_lock);
if (map)
- jump_label_inc(&rps_needed);
+ static_key_slow_inc(&rps_needed);
if (old_map) {
kfree_rcu(old_map, rcu);
- jump_label_dec(&rps_needed);
+ static_key_slow_dec(&rps_needed);
}
free_cpumask_var(mask);
return len;
diff --git a/net/core/sock.c b/net/core/sock.c
index 3e81fd2..3a4e581 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -111,7 +111,7 @@
#include <linux/init.h>
#include <linux/highmem.h>
#include <linux/user_namespace.h>
-#include <linux/jump_label.h>
+#include <linux/static_key.h>
#include <linux/memcontrol.h>
#include <asm/uaccess.h>
@@ -184,7 +184,7 @@ void mem_cgroup_sockets_destroy(struct cgroup *cgrp, struct cgroup_subsys *ss)
static struct lock_class_key af_family_keys[AF_MAX];
static struct lock_class_key af_family_slock_keys[AF_MAX];
-struct jump_label_key memcg_socket_limit_enabled;
+struct static_key memcg_socket_limit_enabled;
EXPORT_SYMBOL(memcg_socket_limit_enabled);
/*
diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c
index d05559d..0c28508 100644
--- a/net/core/sysctl_net_core.c
+++ b/net/core/sysctl_net_core.c
@@ -69,9 +69,9 @@ static int rps_sock_flow_sysctl(ctl_table *table, int write,
if (sock_table != orig_sock_table) {
rcu_assign_pointer(rps_sock_flow_table, sock_table);
if (sock_table)
- jump_label_inc(&rps_needed);
+ static_key_slow_inc(&rps_needed);
if (orig_sock_table) {
- jump_label_dec(&rps_needed);
+ static_key_slow_dec(&rps_needed);
synchronize_rcu();
vfree(orig_sock_table);
}
diff --git a/net/ipv4/tcp_memcontrol.c b/net/ipv4/tcp_memcontrol.c
index 4997878..602fb30 100644
--- a/net/ipv4/tcp_memcontrol.c
+++ b/net/ipv4/tcp_memcontrol.c
@@ -111,7 +111,7 @@ void tcp_destroy_cgroup(struct cgroup *cgrp, struct cgroup_subsys *ss)
val = res_counter_read_u64(&tcp->tcp_memory_allocated, RES_LIMIT);
if (val != RESOURCE_MAX)
- jump_label_dec(&memcg_socket_limit_enabled);
+ static_key_slow_dec(&memcg_socket_limit_enabled);
}
EXPORT_SYMBOL(tcp_destroy_cgroup);
@@ -143,9 +143,9 @@ static int tcp_update_limit(struct mem_cgroup *memcg, u64 val)
net->ipv4.sysctl_tcp_mem[i]);
if (val == RESOURCE_MAX && old_lim != RESOURCE_MAX)
- jump_label_dec(&memcg_socket_limit_enabled);
+ static_key_slow_dec(&memcg_socket_limit_enabled);
else if (old_lim == RESOURCE_MAX && val != RESOURCE_MAX)
- jump_label_inc(&memcg_socket_limit_enabled);
+ static_key_slow_inc(&memcg_socket_limit_enabled);
return 0;
}
diff --git a/net/netfilter/core.c b/net/netfilter/core.c
index b4e8ff0..e1b7e05 100644
--- a/net/netfilter/core.c
+++ b/net/netfilter/core.c
@@ -56,7 +56,7 @@ struct list_head nf_hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS] __read_mostly;
EXPORT_SYMBOL(nf_hooks);
#if defined(CONFIG_JUMP_LABEL)
-struct jump_label_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
+struct static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
EXPORT_SYMBOL(nf_hooks_needed);
#endif
@@ -77,7 +77,7 @@ int nf_register_hook(struct nf_hook_ops *reg)
list_add_rcu(®->list, elem->list.prev);
mutex_unlock(&nf_hook_mutex);
#if defined(CONFIG_JUMP_LABEL)
- jump_label_inc(&nf_hooks_needed[reg->pf][reg->hooknum]);
+ static_key_slow_inc(&nf_hooks_needed[reg->pf][reg->hooknum]);
#endif
return 0;
}
@@ -89,7 +89,7 @@ void nf_unregister_hook(struct nf_hook_ops *reg)
list_del_rcu(®->list);
mutex_unlock(&nf_hook_mutex);
#if defined(CONFIG_JUMP_LABEL)
- jump_label_dec(&nf_hooks_needed[reg->pf][reg->hooknum]);
+ static_key_slow_dec(&nf_hooks_needed[reg->pf][reg->hooknum]);
#endif
synchronize_net();
}