Static calls are a replacement for global function pointers. They use
code patching to allow direct calls to be used instead of indirect
calls. They give the flexibility of function pointers, but with
improved performance. This is especially important for cases where
retpolines would otherwise be used, as retpolines can significantly
impact performance.
The concept and code are an extension of previous work done by Ard
Biesheuvel and Steven Rostedt:
https://lkml.kernel.org/r/[email protected]
https://lkml.kernel.org/r/[email protected]
There are two implementations, depending on arch support:
1) out-of-line: patched trampolines (CONFIG_HAVE_STATIC_CALL)
2) basic function pointers
For more details, see the comments in include/linux/static_call.h.
[peterz: simplified interface]
Signed-off-by: Josh Poimboeuf <[email protected]>
Signed-off-by: Peter Zijlstra (Intel) <[email protected]>
---
arch/Kconfig | 3
include/linux/static_call.h | 137 ++++++++++++++++++++++++++++++++++++++
include/linux/static_call_types.h | 15 ++++
3 files changed, 155 insertions(+)
create mode 100644 include/linux/static_call.h
create mode 100644 include/linux/static_call_types.h
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -941,6 +941,9 @@ config HAVE_SPARSE_SYSCALL_NR
entries at 4000, 5000 and 6000 locations. This option turns on syscall
related optimizations for a given architecture.
+config HAVE_STATIC_CALL
+ bool
+
source "kernel/gcov/Kconfig"
source "scripts/gcc-plugins/Kconfig"
--- /dev/null
+++ b/include/linux/static_call.h
@@ -0,0 +1,137 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_STATIC_CALL_H
+#define _LINUX_STATIC_CALL_H
+
+/*
+ * Static call support
+ *
+ * Static calls use code patching to hard-code function pointers into direct
+ * branch instructions. They give the flexibility of function pointers, but
+ * with improved performance. This is especially important for cases where
+ * retpolines would otherwise be used, as retpolines can significantly impact
+ * performance.
+ *
+ *
+ * API overview:
+ *
+ * DECLARE_STATIC_CALL(name, func);
+ * DEFINE_STATIC_CALL(name, func);
+ * static_call(name)(args...);
+ * static_call_update(name, func);
+ *
+ * Usage example:
+ *
+ * # Start with the following functions (with identical prototypes):
+ * int func_a(int arg1, int arg2);
+ * int func_b(int arg1, int arg2);
+ *
+ * # Define a 'my_name' reference, associated with func_a() by default
+ * DEFINE_STATIC_CALL(my_name, func_a);
+ *
+ * # Call func_a()
+ * static_call(my_name)(arg1, arg2);
+ *
+ * # Update 'my_name' to point to func_b()
+ * static_call_update(my_name, &func_b);
+ *
+ * # Call func_b()
+ * static_call(my_name)(arg1, arg2);
+ *
+ *
+ * Implementation details:
+ *
+ * This requires some arch-specific code (CONFIG_HAVE_STATIC_CALL).
+ * Otherwise basic indirect calls are used (with function pointers).
+ *
+ * Each static_call() site calls into a trampoline associated with the name.
+ * The trampoline has a direct branch to the default function. Updates to a
+ * name will modify the trampoline's branch destination.
+ *
+ * If the arch has CONFIG_HAVE_STATIC_CALL_INLINE, then the call sites
+ * themselves will be patched at runtime to call the functions directly,
+ * rather than calling through the trampoline. This requires objtool or a
+ * compiler plugin to detect all the static_call() sites and annotate them
+ * in the .static_call_sites section.
+ */
+
+#include <linux/types.h>
+#include <linux/cpu.h>
+#include <linux/static_call_types.h>
+
+#ifdef CONFIG_HAVE_STATIC_CALL
+#include <asm/static_call.h>
+/*
+ * Either @site or @tramp can be NULL.
+ */
+extern void arch_static_call_transform(void *site, void *tramp, void *func);
+#define STATIC_CALL_TRAMP_ADDR(name) &STATIC_CALL_TRAMP(name)
+#else
+#define STATIC_CALL_TRAMP_ADDR(name) NULL
+#endif
+
+
+#define DECLARE_STATIC_CALL(name, func) \
+ extern struct static_call_key STATIC_CALL_NAME(name); \
+ extern typeof(func) STATIC_CALL_TRAMP(name)
+
+#define static_call_update(name, func) \
+({ \
+ BUILD_BUG_ON(!__same_type(*(func), STATIC_CALL_TRAMP(name))); \
+ __static_call_update(&STATIC_CALL_NAME(name), \
+ STATIC_CALL_TRAMP_ADDR(name), func); \
+})
+
+#if defined(CONFIG_HAVE_STATIC_CALL)
+
+struct static_call_key {
+ void *func;
+};
+
+#define DEFINE_STATIC_CALL(name, _func) \
+ DECLARE_STATIC_CALL(name, _func); \
+ struct static_call_key STATIC_CALL_NAME(name) = { \
+ .func = _func, \
+ }; \
+ ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
+
+#define static_call(name) STATIC_CALL_TRAMP(name)
+
+static inline
+void __static_call_update(struct static_call_key *key, void *tramp, void *func)
+{
+ cpus_read_lock();
+ WRITE_ONCE(key->func, func);
+ arch_static_call_transform(NULL, tramp, func);
+ cpus_read_unlock();
+}
+
+#define EXPORT_STATIC_CALL(name) EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
+#define EXPORT_STATIC_CALL_GPL(name) EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
+
+#else /* Generic implementation */
+
+struct static_call_key {
+ void *func;
+};
+
+#define DEFINE_STATIC_CALL(name, _func) \
+ DECLARE_STATIC_CALL(name, _func); \
+ struct static_call_key STATIC_CALL_NAME(name) = { \
+ .func = _func, \
+ }
+
+#define static_call(name) \
+ ((typeof(STATIC_CALL_TRAMP(name))*)(STATIC_CALL_NAME(name).func))
+
+static inline
+void __static_call_update(struct static_call_key *key, void *tramp, void *func)
+{
+ WRITE_ONCE(key->func, func);
+}
+
+#define EXPORT_STATIC_CALL(name) EXPORT_SYMBOL(STATIC_CALL_NAME(name))
+#define EXPORT_STATIC_CALL_GPL(name) EXPORT_SYMBOL_GPL(STATIC_CALL_NAME(name))
+
+#endif /* CONFIG_HAVE_STATIC_CALL */
+
+#endif /* _LINUX_STATIC_CALL_H */
--- /dev/null
+++ b/include/linux/static_call_types.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _STATIC_CALL_TYPES_H
+#define _STATIC_CALL_TYPES_H
+
+#include <linux/stringify.h>
+
+#define STATIC_CALL_PREFIX __SC__
+#define STATIC_CALL_PREFIX_STR __stringify(STATIC_CALL_PREFIX)
+
+#define STATIC_CALL_NAME(name) __PASTE(STATIC_CALL_PREFIX, name)
+
+#define STATIC_CALL_TRAMP(name) STATIC_CALL_NAME(name##_tramp)
+#define STATIC_CALL_TRAMP_STR(name) __stringify(STATIC_CALL_TRAMP(name))
+
+#endif /* _STATIC_CALL_TYPES_H */
> On Mar 24, 2020, at 6:56 AM, Peter Zijlstra <[email protected]> wrote:
>
> Static calls are a replacement for global function pointers. They use
> code patching to allow direct calls to be used instead of indirect
> calls. They give the flexibility of function pointers, but with
> improved performance. This is especially important for cases where
> retpolines would otherwise be used, as retpolines can significantly
> impact performance.
>
> The concept and code are an extension of previous work done by Ard
> Biesheuvel and Steven Rostedt:
>
> https://nam04.safelinks.protection.outlook.com/?url=https%3A%2F%2Flkml.kernel.org%2Fr%2F20181005081333.15018-1-ard.biesheuvel%40linaro.org&data=02%7C01%7Cnamit%40vmware.com%7C48cb49e3bd65479a440008d7cfff207d%7Cb39138ca3cee4b4aa4d6cd83d9dd62f0%7C0%7C0%7C637206567005697724&sdata=jLAdeYFX8C67QFeYtiguq9BfeN8zxjSa8gmtY%2F2nJQ8%3D&reserved=0
> https://nam04.safelinks.protection.outlook.com/?url=https%3A%2F%2Flkml.kernel.org%2Fr%2F20181006015110.653946300%40goodmis.org&data=02%7C01%7Cnamit%40vmware.com%7C48cb49e3bd65479a440008d7cfff207d%7Cb39138ca3cee4b4aa4d6cd83d9dd62f0%7C0%7C0%7C637206567005697724&sdata=KBVx7EJo4EgzKaCccAaMP7C4L%2BYlQ6liO4wO7vv27MU%3D&reserved=0
>
> There are two implementations, depending on arch support:
>
> 1) out-of-line: patched trampolines (CONFIG_HAVE_STATIC_CALL)
> 2) basic function pointers
>
> For more details, see the comments in include/linux/static_call.h.
>
> [peterz: simplified interface]
> Signed-off-by: Josh Poimboeuf <[email protected]>
> Signed-off-by: Peter Zijlstra (Intel) <[email protected]>
> ---
> arch/Kconfig | 3
> include/linux/static_call.h | 137 ++++++++++++++++++++++++++++++++++++++
> include/linux/static_call_types.h | 15 ++++
> 3 files changed, 155 insertions(+)
> create mode 100644 include/linux/static_call.h
> create mode 100644 include/linux/static_call_types.h
>
> --- a/arch/Kconfig
> +++ b/arch/Kconfig
> @@ -941,6 +941,9 @@ config HAVE_SPARSE_SYSCALL_NR
> entries at 4000, 5000 and 6000 locations. This option turns on syscall
> related optimizations for a given architecture.
>
> +config HAVE_STATIC_CALL
> + bool
> +
> source "kernel/gcov/Kconfig"
>
> source "scripts/gcc-plugins/Kconfig"
> --- /dev/null
> +++ b/include/linux/static_call.h
> @@ -0,0 +1,137 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#ifndef _LINUX_STATIC_CALL_H
> +#define _LINUX_STATIC_CALL_H
> +
> +/*
> + * Static call support
> + *
> + * Static calls use code patching to hard-code function pointers into direct
> + * branch instructions. They give the flexibility of function pointers, but
> + * with improved performance. This is especially important for cases where
> + * retpolines would otherwise be used, as retpolines can significantly impact
> + * performance.
> + *
> + *
> + * API overview:
> + *
> + * DECLARE_STATIC_CALL(name, func);
> + * DEFINE_STATIC_CALL(name, func);
> + * static_call(name)(args...);
> + * static_call_update(name, func);
> + *
> + * Usage example:
> + *
> + * # Start with the following functions (with identical prototypes):
> + * int func_a(int arg1, int arg2);
> + * int func_b(int arg1, int arg2);
> + *
> + * # Define a 'my_name' reference, associated with func_a() by default
> + * DEFINE_STATIC_CALL(my_name, func_a);
Do you want to support optional function attributes, such as “pure” and
“const”?
On Thu, Mar 26, 2020 at 04:42:07PM +0000, Nadav Amit wrote:
> > On Mar 24, 2020, at 6:56 AM, Peter Zijlstra <[email protected]> wrote:
> > + * API overview:
> > + *
> > + * DECLARE_STATIC_CALL(name, func);
> > + * DEFINE_STATIC_CALL(name, func);
> > + * static_call(name)(args...);
> > + * static_call_update(name, func);
> > + *
> > + * Usage example:
> > + *
> > + * # Start with the following functions (with identical prototypes):
> > + * int func_a(int arg1, int arg2);
> > + * int func_b(int arg1, int arg2);
> > + *
> > + * # Define a 'my_name' reference, associated with func_a() by default
> > + * DEFINE_STATIC_CALL(my_name, func_a);
>
> Do you want to support optional function attributes, such as “pure” and
> “const”?
Do you see a need for that? And what is the syntax for a pointer to a
pure function?
> On Mar 26, 2020, at 10:01 AM, Peter Zijlstra <[email protected]> wrote:
>
> On Thu, Mar 26, 2020 at 04:42:07PM +0000, Nadav Amit wrote:
>>> On Mar 24, 2020, at 6:56 AM, Peter Zijlstra <[email protected]> wrote:
>
>>> + * API overview:
>>> + *
>>> + * DECLARE_STATIC_CALL(name, func);
>>> + * DEFINE_STATIC_CALL(name, func);
>>> + * static_call(name)(args...);
>>> + * static_call_update(name, func);
>>> + *
>>> + * Usage example:
>>> + *
>>> + * # Start with the following functions (with identical prototypes):
>>> + * int func_a(int arg1, int arg2);
>>> + * int func_b(int arg1, int arg2);
>>> + *
>>> + * # Define a 'my_name' reference, associated with func_a() by default
>>> + * DEFINE_STATIC_CALL(my_name, func_a);
>>
>> Do you want to support optional function attributes, such as “pure” and
>> “const”?
>
> Do you see a need for that? And what is the syntax for a pointer to a
> pure function?
I think that the kernel underutilizes the pure attribute in general.
Building it with "-Wsuggest-attribute=pure” results in many warnings.
Function pointers such kvm_x86_ops.get_XXX() could have been candidates to
use the “pure” attribute.
The syntax is what you would expect:
static void __attribute__((pure))(*ptr)(void);
However, you have a point, gcc does not appear to respect “pure” for
function pointers and emits a warning it is ignored. GCC apparently only
respects “const”. In contrast clang appears to respect the pure attribute
for function pointers.
On Thu, Mar 26, 2020 at 06:09:07PM +0000, Nadav Amit wrote:
> I think that the kernel underutilizes the pure attribute in general.
> Building it with "-Wsuggest-attribute=pure” results in many warnings.
> Function pointers such kvm_x86_ops.get_XXX() could have been candidates to
> use the “pure” attribute.
>
> The syntax is what you would expect:
>
> static void __attribute__((pure))(*ptr)(void);
>
Well, I didn't in fact expect that, because an attribute is not a
type qualifier.
> However, you have a point, gcc does not appear to respect “pure” for
> function pointers and emits a warning it is ignored. GCC apparently only
> respects “const”. In contrast clang appears to respect the pure attribute
> for function pointers.
Still, we can probably make it happen for static_call(), since it is a
direct call to the trampoline, all we need to do is make sure the
trampoline is declared pure.
It does however mean that static_call() inherits all the dangers and
pit-falls of function pointers with some extra on top. It will be
impossible to validate this stuff.
That is, you can static_call_update() with a pointer to a !pure function
and you get to keep the pieces.
> On Mar 26, 2020, at 11:28 AM, Peter Zijlstra <[email protected]> wrote:
>
> On Thu, Mar 26, 2020 at 06:09:07PM +0000, Nadav Amit wrote:
>
>> I think that the kernel underutilizes the pure attribute in general.
>> Building it with "-Wsuggest-attribute=pure” results in many warnings.
>> Function pointers such kvm_x86_ops.get_XXX() could have been candidates to
>> use the “pure” attribute.
>>
>> The syntax is what you would expect:
>>
>> static void __attribute__((pure))(*ptr)(void);
>
> Well, I didn't in fact expect that, because an attribute is not a
> type qualifier.
Just a small correction for my stupid example - pure function should always
return a value.
>> However, you have a point, gcc does not appear to respect “pure” for
>> function pointers and emits a warning it is ignored. GCC apparently only
>> respects “const”. In contrast clang appears to respect the pure attribute
>> for function pointers.
>
> Still, we can probably make it happen for static_call(), since it is a
> direct call to the trampoline, all we need to do is make sure the
> trampoline is declared pure.
>
> It does however mean that static_call() inherits all the dangers and
> pit-falls of function pointers with some extra on top. It will be
> impossible to validate this stuff.
>
> That is, you can static_call_update() with a pointer to a !pure function
> and you get to keep the pieces.
I understand. Well, perhaps it can be added later, as anyhow GCC does not
support it.
On another note - it may be beneficial to see if the infrastructure that you
built can accommodate notifier-chains. It is not the most painful point, but
it would be nice to deal with those as well. Since many of those are changed
asynchronously, I am not sure it is the easiest thing to do.
On Thu, Mar 26, 2020 at 07:02:33PM +0000, Nadav Amit wrote:
> On another note - it may be beneficial to see if the infrastructure that you
> built can accommodate notifier-chains. It is not the most painful point, but
> it would be nice to deal with those as well. Since many of those are changed
> asynchronously, I am not sure it is the easiest thing to do.
You mean, like patch 12 does?