Introduce a cpumask that indicates (for each CPU) what direction the
CPU hotplug is currently going. Notably, it tracks rollbacks. Eg. when
an up fails and we do a roll-back down, it will accurately reflect the
direction.
Signed-off-by: Peter Zijlstra (Intel) <[email protected]>
---
include/linux/cpumask.h | 20 ++++++++++++++++++++
kernel/cpu.c | 6 ++++++
2 files changed, 26 insertions(+)
--- a/include/linux/cpumask.h
+++ b/include/linux/cpumask.h
@@ -91,10 +91,12 @@ extern struct cpumask __cpu_possible_mas
extern struct cpumask __cpu_online_mask;
extern struct cpumask __cpu_present_mask;
extern struct cpumask __cpu_active_mask;
+extern struct cpumask __cpu_dying_mask;
#define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask)
#define cpu_online_mask ((const struct cpumask *)&__cpu_online_mask)
#define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask)
#define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask)
+#define cpu_dying_mask ((const struct cpumask *)&__cpu_dying_mask)
extern atomic_t __num_online_cpus;
@@ -826,6 +828,14 @@ set_cpu_active(unsigned int cpu, bool ac
cpumask_clear_cpu(cpu, &__cpu_active_mask);
}
+static inline void
+set_cpu_dying(unsigned int cpu, bool dying)
+{
+ if (dying)
+ cpumask_set_cpu(cpu, &__cpu_dying_mask);
+ else
+ cpumask_clear_cpu(cpu, &__cpu_dying_mask);
+}
/**
* to_cpumask - convert an NR_CPUS bitmap to a struct cpumask *
@@ -900,6 +910,11 @@ static inline bool cpu_active(unsigned i
return cpumask_test_cpu(cpu, cpu_active_mask);
}
+static inline bool cpu_dying(unsigned int cpu)
+{
+ return cpumask_test_cpu(cpu, cpu_dying_mask);
+}
+
#else
#define num_online_cpus() 1U
@@ -927,6 +942,11 @@ static inline bool cpu_active(unsigned i
return cpu == 0;
}
+static inline bool cpu_dying(unsigned int cpu)
+{
+ return false;
+}
+
#endif /* NR_CPUS > 1 */
#define cpu_is_offline(cpu) unlikely(!cpu_online(cpu))
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -160,6 +160,9 @@ static int cpuhp_invoke_callback(unsigne
int (*cb)(unsigned int cpu);
int ret, cnt;
+ if (bringup != !cpu_dying(cpu))
+ set_cpu_dying(cpu, !bringup);
+
if (st->fail == state) {
st->fail = CPUHP_INVALID;
return -EAGAIN;
@@ -2512,6 +2515,9 @@ EXPORT_SYMBOL(__cpu_present_mask);
struct cpumask __cpu_active_mask __read_mostly;
EXPORT_SYMBOL(__cpu_active_mask);
+struct cpumask __cpu_dying_mask __read_mostly;
+EXPORT_SYMBOL(__cpu_dying_mask);
+
atomic_t __num_online_cpus __read_mostly;
EXPORT_SYMBOL(__num_online_cpus);
On 03/10/21 15:53, Peter Zijlstra wrote:
> --- a/kernel/cpu.c
> +++ b/kernel/cpu.c
> @@ -160,6 +160,9 @@ static int cpuhp_invoke_callback(unsigne
> int (*cb)(unsigned int cpu);
> int ret, cnt;
>
> + if (bringup != !cpu_dying(cpu))
nit: this condition is hard to read
> + set_cpu_dying(cpu, !bringup);
since cpu_dying() will do cpumask_test_cpu(), are we saving much if we
unconditionally call set_cpu_dying(cpu, !bringup) which performs
cpumask_{set, clear}_cpu()?
> +
> if (st->fail == state) {
> st->fail = CPUHP_INVALID;
> return -EAGAIN;
Thanks
--
Qais yousef
On Sun, 21 Mar 2021 19:30:37 +0000
Qais Yousef <[email protected]> wrote:
> On 03/10/21 15:53, Peter Zijlstra wrote:
> > --- a/kernel/cpu.c
> > +++ b/kernel/cpu.c
> > @@ -160,6 +160,9 @@ static int cpuhp_invoke_callback(unsigne
> > int (*cb)(unsigned int cpu);
> > int ret, cnt;
> >
> > + if (bringup != !cpu_dying(cpu))
>
> nit: this condition is hard to read
Would
if (bringup == !!cpu_dying(cpu))
read better?
-- Steve
>
> > + set_cpu_dying(cpu, !bringup);
On Sun, Mar 21, 2021 at 07:30:37PM +0000, Qais Yousef wrote:
> On 03/10/21 15:53, Peter Zijlstra wrote:
> > --- a/kernel/cpu.c
> > +++ b/kernel/cpu.c
> > @@ -160,6 +160,9 @@ static int cpuhp_invoke_callback(unsigne
> > int (*cb)(unsigned int cpu);
> > int ret, cnt;
> >
> > + if (bringup != !cpu_dying(cpu))
>
> nit: this condition is hard to read
>
> > + set_cpu_dying(cpu, !bringup);
How's:
if (cpu_dying(cpu) != !bringup)
set_cpu_dying(cpu, !bringup);
> since cpu_dying() will do cpumask_test_cpu(), are we saving much if we
> unconditionally call set_cpu_dying(cpu, !bringup) which performs
> cpumask_{set, clear}_cpu()?
This is hotplug, it's all slow, endlessly rewriting that bit shouldn't
be a problem I suppose.
On 04/12/21 12:55, Peter Zijlstra wrote:
> On Sun, Mar 21, 2021 at 07:30:37PM +0000, Qais Yousef wrote:
> > On 03/10/21 15:53, Peter Zijlstra wrote:
> > > --- a/kernel/cpu.c
> > > +++ b/kernel/cpu.c
> > > @@ -160,6 +160,9 @@ static int cpuhp_invoke_callback(unsigne
> > > int (*cb)(unsigned int cpu);
> > > int ret, cnt;
> > >
> > > + if (bringup != !cpu_dying(cpu))
> >
> > nit: this condition is hard to read
> >
> > > + set_cpu_dying(cpu, !bringup);
>
> How's:
>
> if (cpu_dying(cpu) != !bringup)
> set_cpu_dying(cpu, !bringup);
Slightly better I suppose :)
>
> > since cpu_dying() will do cpumask_test_cpu(), are we saving much if we
> > unconditionally call set_cpu_dying(cpu, !bringup) which performs
> > cpumask_{set, clear}_cpu()?
>
> This is hotplug, it's all slow, endlessly rewriting that bit shouldn't
> be a problem I suppose.
True. Beside I doubt there's a performance hit really, cpu_dying() will read
the bit in the cpumask anyway, unconditionally writing will be as fast since
both will fetch the cacheline anyway?
Regardless, not really a big deal. It's not really the hardest thing to stare
at here ;-)
Thanks
--
Qais Yousef
The following commit has been merged into the sched/core branch of tip:
Commit-ID: e40f74c535b8a0ecf3ef0388b51a34cdadb34fb5
Gitweb: https://git.kernel.org/tip/e40f74c535b8a0ecf3ef0388b51a34cdadb34fb5
Author: Peter Zijlstra <[email protected]>
AuthorDate: Tue, 19 Jan 2021 18:43:45 +01:00
Committer: Peter Zijlstra <[email protected]>
CommitterDate: Fri, 16 Apr 2021 17:06:32 +02:00
cpumask: Introduce DYING mask
Introduce a cpumask that indicates (for each CPU) what direction the
CPU hotplug is currently going. Notably, it tracks rollbacks. Eg. when
an up fails and we do a roll-back down, it will accurately reflect the
direction.
Signed-off-by: Peter Zijlstra (Intel) <[email protected]>
Reviewed-by: Valentin Schneider <[email protected]>
Link: https://lkml.kernel.org/r/[email protected]
---
include/linux/cpumask.h | 20 ++++++++++++++++++++
kernel/cpu.c | 6 ++++++
2 files changed, 26 insertions(+)
diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h
index a584336..e6b948a 100644
--- a/include/linux/cpumask.h
+++ b/include/linux/cpumask.h
@@ -91,10 +91,12 @@ extern struct cpumask __cpu_possible_mask;
extern struct cpumask __cpu_online_mask;
extern struct cpumask __cpu_present_mask;
extern struct cpumask __cpu_active_mask;
+extern struct cpumask __cpu_dying_mask;
#define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask)
#define cpu_online_mask ((const struct cpumask *)&__cpu_online_mask)
#define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask)
#define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask)
+#define cpu_dying_mask ((const struct cpumask *)&__cpu_dying_mask)
extern atomic_t __num_online_cpus;
@@ -826,6 +828,14 @@ set_cpu_active(unsigned int cpu, bool active)
cpumask_clear_cpu(cpu, &__cpu_active_mask);
}
+static inline void
+set_cpu_dying(unsigned int cpu, bool dying)
+{
+ if (dying)
+ cpumask_set_cpu(cpu, &__cpu_dying_mask);
+ else
+ cpumask_clear_cpu(cpu, &__cpu_dying_mask);
+}
/**
* to_cpumask - convert an NR_CPUS bitmap to a struct cpumask *
@@ -900,6 +910,11 @@ static inline bool cpu_active(unsigned int cpu)
return cpumask_test_cpu(cpu, cpu_active_mask);
}
+static inline bool cpu_dying(unsigned int cpu)
+{
+ return cpumask_test_cpu(cpu, cpu_dying_mask);
+}
+
#else
#define num_online_cpus() 1U
@@ -927,6 +942,11 @@ static inline bool cpu_active(unsigned int cpu)
return cpu == 0;
}
+static inline bool cpu_dying(unsigned int cpu)
+{
+ return false;
+}
+
#endif /* NR_CPUS > 1 */
#define cpu_is_offline(cpu) unlikely(!cpu_online(cpu))
diff --git a/kernel/cpu.c b/kernel/cpu.c
index 23505d6..838dcf2 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -160,6 +160,9 @@ static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,
int (*cb)(unsigned int cpu);
int ret, cnt;
+ if (cpu_dying(cpu) != !bringup)
+ set_cpu_dying(cpu, !bringup);
+
if (st->fail == state) {
st->fail = CPUHP_INVALID;
return -EAGAIN;
@@ -2512,6 +2515,9 @@ EXPORT_SYMBOL(__cpu_present_mask);
struct cpumask __cpu_active_mask __read_mostly;
EXPORT_SYMBOL(__cpu_active_mask);
+struct cpumask __cpu_dying_mask __read_mostly;
+EXPORT_SYMBOL(__cpu_dying_mask);
+
atomic_t __num_online_cpus __read_mostly;
EXPORT_SYMBOL(__num_online_cpus);