From: "Steven Rostedt (VMware)" <[email protected]>
Now that the sched_class descriptors are defined by the linker script, and
this needs to be aware of the existance of stop_sched_class when SMP is
enabled or not, as it is used as the "highest" priority when defined. Move
the declaration of sched_class_highest to the same location in the linker
script that inserts stop_sched_class, and this will also make it easier to
see what should be defined as the highest class, as this linker script
location defines the priorities as well.
Signed-off-by: Steven Rostedt (VMware) <[email protected]>
---
include/asm-generic/vmlinux.lds.h | 11 ++++++++++-
kernel/sched/sched.h | 9 +++------
2 files changed, 13 insertions(+), 7 deletions(-)
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index 772d961c69a5..1c14c4ddf785 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -109,9 +109,16 @@
#endif
#ifdef CONFIG_SMP
-#define STOP_SCHED_CLASS *(__stop_sched_class)
+#define STOP_SCHED_CLASS \
+ STRUCT_ALIGN(); \
+ sched_class_highest = .; \
+ *(__stop_sched_class)
+#define BEFORE_DL_SCHED_CLASS
#else
#define STOP_SCHED_CLASS
+#define BEFORE_DL_SCHED_CLASS \
+ STRUCT_ALIGN(); \
+ sched_class_highest = .;
#endif
/*
@@ -120,9 +127,11 @@
* relation to each other.
*/
#define SCHED_DATA \
+ STRUCT_ALIGN(); \
*(__idle_sched_class) \
*(__fair_sched_class) \
*(__rt_sched_class) \
+ BEFORE_DL_SCHED_CLASS \
*(__dl_sched_class) \
STOP_SCHED_CLASS
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 280a3c735935..0554c588ad85 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1771,17 +1771,14 @@ static inline void set_next_task(struct rq *rq, struct task_struct *next)
next->sched_class->set_next_task(rq, next, false);
}
-#ifdef CONFIG_SMP
-#define sched_class_highest (&stop_sched_class)
-#else
-#define sched_class_highest (&dl_sched_class)
-#endif
+/* Defined in include/asm-generic/vmlinux.lds.h */
+extern struct sched_class sched_class_highest;
#define for_class_range(class, _from, _to) \
for (class = (_from); class != (_to); class = class->next)
#define for_each_class(class) \
- for_class_range(class, sched_class_highest, NULL)
+ for_class_range(class, &sched_class_highest, NULL)
extern const struct sched_class stop_sched_class;
extern const struct sched_class dl_sched_class;
--
2.24.0
On 19/12/2019 22.44, Steven Rostedt wrote:
> From: "Steven Rostedt (VMware)" <[email protected]>
>
> Now that the sched_class descriptors are defined by the linker script, and
> this needs to be aware of the existance of stop_sched_class when SMP is
> enabled or not, as it is used as the "highest" priority when defined. Move
> the declaration of sched_class_highest to the same location in the linker
> script that inserts stop_sched_class, and this will also make it easier to
> see what should be defined as the highest class, as this linker script
> location defines the priorities as well.
>
> Signed-off-by: Steven Rostedt (VMware) <[email protected]>
> ---
> include/asm-generic/vmlinux.lds.h | 11 ++++++++++-
> kernel/sched/sched.h | 9 +++------
> 2 files changed, 13 insertions(+), 7 deletions(-)
>
> diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
> index 772d961c69a5..1c14c4ddf785 100644
> --- a/include/asm-generic/vmlinux.lds.h
> +++ b/include/asm-generic/vmlinux.lds.h
> @@ -109,9 +109,16 @@
> #endif
>
> #ifdef CONFIG_SMP
> -#define STOP_SCHED_CLASS *(__stop_sched_class)
> +#define STOP_SCHED_CLASS \
> + STRUCT_ALIGN(); \
> + sched_class_highest = .; \
> + *(__stop_sched_class)
> +#define BEFORE_DL_SCHED_CLASS
> #else
> #define STOP_SCHED_CLASS
> +#define BEFORE_DL_SCHED_CLASS \
> + STRUCT_ALIGN(); \
> + sched_class_highest = .;
> #endif
>
> /*
> @@ -120,9 +127,11 @@
> * relation to each other.
> */
> #define SCHED_DATA \
> + STRUCT_ALIGN(); \
> *(__idle_sched_class) \
> *(__fair_sched_class) \
> *(__rt_sched_class) \
> + BEFORE_DL_SCHED_CLASS \
> *(__dl_sched_class) \
> STOP_SCHED_CLASS
If you reverse the ordering so the highest sched class comes first, this
can just be
sched_class_highest = .
STOP_SCHED_CLASS
*(__dl_sched_class)
...
*(__idle_sched_class)
__end_sched_classes = .
and this from patch 3/4
#define for_class_range(class, _from, _to) \
- for (class = (_from); class != (_to); class = class->next)
+ for (class = (_from); class > (_to); class--)
#define for_each_class(class) \
- for_class_range(class, &sched_class_highest, NULL)
+ for_class_range(class, &sched_class_highest, (&__start_sched_classes) - 1)
can instead become
+ for (class = (_from); class < (_to); class++)
and
+ for_class_range(class, &sched_class_highest, &__end_sched_classes)
which seem somewhat more readable.
And actually, I don't think you need the STOP_SCHED_CLASS define at all
- in non-SMP, no object file has a __stop_sched_class section, so
including *(__stop_sched_class) unconditionally will DTRT. (BTW, it's a
bit confusing that stop_task.o is compiled in if CONFIG_SMP, but
stop_task.c also has a #ifdef CONFIG_SMP).
Rasmus
The following commit has been merged into the sched/core branch of tip:
Commit-ID: c3a340f7e7eadac7662ab104ceb16432e5a4c6b2
Gitweb: https://git.kernel.org/tip/c3a340f7e7eadac7662ab104ceb16432e5a4c6b2
Author: Steven Rostedt (VMware) <[email protected]>
AuthorDate: Thu, 19 Dec 2019 16:44:53 -05:00
Committer: Peter Zijlstra <[email protected]>
CommitterDate: Thu, 25 Jun 2020 13:45:44 +02:00
sched: Have sched_class_highest define by vmlinux.lds.h
Now that the sched_class descriptors are defined by the linker script, and
this needs to be aware of the existance of stop_sched_class when SMP is
enabled or not, as it is used as the "highest" priority when defined. Move
the declaration of sched_class_highest to the same location in the linker
script that inserts stop_sched_class, and this will also make it easier to
see what should be defined as the highest class, as this linker script
location defines the priorities as well.
Signed-off-by: Steven Rostedt (VMware) <[email protected]>
Signed-off-by: Peter Zijlstra (Intel) <[email protected]>
Link: https://lkml.kernel.org/r/[email protected]
---
include/asm-generic/vmlinux.lds.h | 5 ++++-
kernel/sched/core.c | 8 ++++++++
kernel/sched/sched.h | 17 +++++++++--------
3 files changed, 21 insertions(+), 9 deletions(-)
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index 2186d7b..66fb84c 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -114,11 +114,14 @@
* relation to each other.
*/
#define SCHED_DATA \
+ STRUCT_ALIGN(); \
+ __begin_sched_classes = .; \
*(__idle_sched_class) \
*(__fair_sched_class) \
*(__rt_sched_class) \
*(__dl_sched_class) \
- *(__stop_sched_class)
+ *(__stop_sched_class) \
+ __end_sched_classes = .;
/*
* Align to a 32 byte boundary equal to the
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 0208b71..81640fe 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -6646,6 +6646,14 @@ void __init sched_init(void)
unsigned long ptr = 0;
int i;
+ /* Make sure the linker didn't screw up */
+ BUG_ON(&idle_sched_class + 1 != &fair_sched_class ||
+ &fair_sched_class + 1 != &rt_sched_class ||
+ &rt_sched_class + 1 != &dl_sched_class);
+#ifdef CONFIG_SMP
+ BUG_ON(&dl_sched_class + 1 != &stop_sched_class);
+#endif
+
wait_bit_init();
#ifdef CONFIG_FAIR_GROUP_SCHED
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 3368876..4165c06 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1811,7 +1811,7 @@ struct sched_class {
#ifdef CONFIG_FAIR_GROUP_SCHED
void (*task_change_group)(struct task_struct *p, int type);
#endif
-};
+} __aligned(32); /* STRUCT_ALIGN(), vmlinux.lds.h */
static inline void put_prev_task(struct rq *rq, struct task_struct *prev)
{
@@ -1825,17 +1825,18 @@ static inline void set_next_task(struct rq *rq, struct task_struct *next)
next->sched_class->set_next_task(rq, next, false);
}
-#ifdef CONFIG_SMP
-#define sched_class_highest (&stop_sched_class)
-#else
-#define sched_class_highest (&dl_sched_class)
-#endif
+/* Defined in include/asm-generic/vmlinux.lds.h */
+extern struct sched_class __begin_sched_classes[];
+extern struct sched_class __end_sched_classes[];
+
+#define sched_class_highest (__end_sched_classes - 1)
+#define sched_class_lowest (__begin_sched_classes - 1)
#define for_class_range(class, _from, _to) \
- for (class = (_from); class != (_to); class = class->next)
+ for (class = (_from); class != (_to); class--)
#define for_each_class(class) \
- for_class_range(class, sched_class_highest, NULL)
+ for_class_range(class, sched_class_highest, sched_class_lowest)
extern const struct sched_class stop_sched_class;
extern const struct sched_class dl_sched_class;