Consider a system with some NOHZ-idle CPUs, such that
nohz.idle_cpus_mask = S
nohz.next_balance = T
When a new CPU k goes NOHZ idle (nohz_balance_enter_idle()), we end up
with:
nohz.idle_cpus_mask = S \U {k}
nohz.next_balance = T
Note that the nohz.next_balance hasn't changed - it won't be updated until
a NOHZ balance is triggered. This is problematic if the newly NOHZ idle CPU
has an earlier rq.next_balance than the other NOHZ idle CPUs, IOW if:
cpu_rq(k).next_balance < nohz.next_balance
In such scenarios, the existing nohz.next_balance will prevent any NOHZ
balance from happening, which itself will prevent nohz.next_balance from
being updated to this new cpu_rq(k).next_balance. Unnecessary load balance
delays of over 12ms caused by this were observed on an arm64 RB5 board.
Track which CPUs are iterated over during a NOHZ idle balance with a new
cpumask. When considering whether to kick a NOHZ idle balance, use this
cpumask to determine if any CPU has entered NOHZ idle but hasn't had its
rq.next_balance collated into nohz.next_balance yet, and kick a NOHZ_STATS
balance if it is the case.
Signed-off-by: Valentin Schneider <[email protected]>
---
kernel/sched/core.c | 8 ++++++++
kernel/sched/fair.c | 19 +++++++++++++++++--
2 files changed, 25 insertions(+), 2 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 0c22cd026440..1bc4cbc1f85e 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -8893,6 +8893,10 @@ static struct kmem_cache *task_group_cache __read_mostly;
DECLARE_PER_CPU(cpumask_var_t, load_balance_mask);
DECLARE_PER_CPU(cpumask_var_t, select_idle_mask);
+#ifdef CONFIG_NOHZ_COMMON
+DECLARE_PER_CPU(cpumask_var_t, nohz_balance_mask);
+#endif /* CONFIG_NOHZ_COMMON */
+
void __init sched_init(void)
{
unsigned long ptr = 0;
@@ -8942,6 +8946,10 @@ void __init sched_init(void)
cpumask_size(), GFP_KERNEL, cpu_to_node(i));
per_cpu(select_idle_mask, i) = (cpumask_var_t)kzalloc_node(
cpumask_size(), GFP_KERNEL, cpu_to_node(i));
+#ifdef CONFIG_NOHZ_COMMON
+ per_cpu(nohz_balance_mask, i) = (cpumask_var_t)kzalloc_node(
+ cpumask_size(), GFP_KERNEL, cpu_to_node(i));
+#endif /* CONFIG_NOHZ_COMMON */
}
#endif /* CONFIG_CPUMASK_OFFSTACK */
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 11d22943753f..497208a1afb8 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -5694,8 +5694,11 @@ DEFINE_PER_CPU(cpumask_var_t, select_idle_mask);
#ifdef CONFIG_NO_HZ_COMMON
+DEFINE_PER_CPU(cpumask_var_t, nohz_balance_mask);
+
static struct {
- cpumask_var_t idle_cpus_mask;
+ cpumask_var_t idle_cpus_mask; /* CPUs in NOHZ idle */
+ cpumask_var_t last_balance_mask; /* CPUs covered by last NOHZ balance */
atomic_t nr_cpus;
int has_blocked; /* Idle CPUS has blocked load */
unsigned long next_balance; /* in jiffy units */
@@ -10351,6 +10354,13 @@ static void nohz_balancer_kick(struct rq *rq)
unlock:
rcu_read_unlock();
out:
+ /*
+ * Some CPUs have recently gone into NOHZ idle; kick a balance to
+ * collate the proper next balance interval.
+ */
+ if (!cpumask_subset(nohz.idle_cpus_mask, nohz.last_balance_mask))
+ flags |= NOHZ_STATS_KICK;
+
if (flags)
kick_ilb(flags);
}
@@ -10487,6 +10497,7 @@ static bool update_nohz_stats(struct rq *rq)
static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags,
enum cpu_idle_type idle)
{
+ struct cpumask *cpus = this_cpu_cpumask_var_ptr(nohz_balance_mask);
/* Earliest time when we have to do rebalance again */
unsigned long now = jiffies;
unsigned long next_balance = now + 60*HZ;
@@ -10518,7 +10529,8 @@ static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags,
* Start with the next CPU after this_cpu so we will end with this_cpu and let a
* chance for other idle cpu to pull load.
*/
- for_each_cpu_wrap(balance_cpu, nohz.idle_cpus_mask, this_cpu+1) {
+ cpumask_copy(cpus, nohz.idle_cpus_mask);
+ for_each_cpu_wrap(balance_cpu, cpus, this_cpu+1) {
if (!idle_cpu(balance_cpu))
continue;
@@ -10565,6 +10577,8 @@ static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags,
if (likely(update_next_balance))
nohz.next_balance = next_balance;
+ cpumask_copy(nohz.last_balance_mask, cpus);
+
WRITE_ONCE(nohz.next_blocked,
now + msecs_to_jiffies(LOAD_AVG_PERIOD));
@@ -11550,6 +11564,7 @@ __init void init_sched_fair_class(void)
nohz.next_balance = jiffies;
nohz.next_blocked = jiffies;
zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT);
+ zalloc_cpumask_var(&nohz.last_balance_mask, GFP_NOWAIT);
#endif
#endif /* SMP */
--
2.25.1
kernel/sched/fair.c:5697:1: warning: symbol '__pcpu_scope_nohz_balance_mask' was not declared. Should it be static?
Reported-by: kernel test robot <[email protected]>
Signed-off-by: kernel test robot <[email protected]>
---
fair.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 497208a1afb88..c6a515d469dd8 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -5694,7 +5694,7 @@ DEFINE_PER_CPU(cpumask_var_t, select_idle_mask);
#ifdef CONFIG_NO_HZ_COMMON
-DEFINE_PER_CPU(cpumask_var_t, nohz_balance_mask);
+static DEFINE_PER_CPU(cpumask_var_t, nohz_balance_mask);
static struct {
cpumask_var_t idle_cpus_mask; /* CPUs in NOHZ idle */
Hi Valentin,
Thank you for the patch! Perhaps something to improve:
[auto build test WARNING on tip/sched/core]
[also build test WARNING on tip/master linux/master linus/master v5.14-rc1 next-20210714]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Valentin-Schneider/sched-fair-Update-nohz-next_balance-for-newly-NOHZ-idle-CPUs/20210714-194021
base: https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git 031e3bd8986fffe31e1ddbf5264cccfe30c9abd7
config: x86_64-randconfig-s022-20210714 (attached as .config)
compiler: gcc-10 (Debian 10.2.1-6) 10.2.1 20210110
reproduce:
# apt-get install sparse
# sparse version: v0.6.3-341-g8af24329-dirty
# https://github.com/0day-ci/linux/commit/cbd87e97caf59c1a9d06d35e5a59404e4d7c8660
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Valentin-Schneider/sched-fair-Update-nohz-next_balance-for-newly-NOHZ-idle-CPUs/20210714-194021
git checkout cbd87e97caf59c1a9d06d35e5a59404e4d7c8660
# save the attached .config to linux build tree
make W=1 C=1 CF='-fdiagnostic-prefix -D__CHECK_ENDIAN__' O=build_dir ARCH=x86_64 SHELL=/bin/bash kernel/sched/
If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <[email protected]>
sparse warnings: (new ones prefixed by >>)
kernel/sched/fair.c:830:34: sparse: sparse: incorrect type in argument 1 (different address spaces) @@ expected struct sched_entity *se @@ got struct sched_entity [noderef] __rcu * @@
kernel/sched/fair.c:830:34: sparse: expected struct sched_entity *se
kernel/sched/fair.c:830:34: sparse: got struct sched_entity [noderef] __rcu *
kernel/sched/fair.c:2477:13: sparse: sparse: incorrect type in assignment (different address spaces) @@ expected struct task_struct *tsk @@ got struct task_struct [noderef] __rcu * @@
kernel/sched/fair.c:2477:13: sparse: expected struct task_struct *tsk
kernel/sched/fair.c:2477:13: sparse: got struct task_struct [noderef] __rcu *
kernel/sched/fair.c:10730:9: sparse: sparse: incorrect type in assignment (different address spaces) @@ expected struct sched_domain *[assigned] sd @@ got struct sched_domain [noderef] __rcu *parent @@
kernel/sched/fair.c:10730:9: sparse: expected struct sched_domain *[assigned] sd
kernel/sched/fair.c:10730:9: sparse: got struct sched_domain [noderef] __rcu *parent
kernel/sched/fair.c:4932:22: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/sched/fair.c:4932:22: sparse: struct task_struct [noderef] __rcu *
kernel/sched/fair.c:4932:22: sparse: struct task_struct *
kernel/sched/fair.c:5692:1: sparse: sparse: symbol '__pcpu_scope_load_balance_mask' was not declared. Should it be static?
kernel/sched/fair.c:5693:1: sparse: sparse: symbol '__pcpu_scope_select_idle_mask' was not declared. Should it be static?
>> kernel/sched/fair.c:5697:1: sparse: sparse: symbol '__pcpu_scope_nohz_balance_mask' was not declared. Should it be static?
kernel/sched/fair.c:6719:20: sparse: sparse: incorrect type in assignment (different address spaces) @@ expected struct sched_domain *[assigned] sd @@ got struct sched_domain [noderef] __rcu *parent @@
kernel/sched/fair.c:6719:20: sparse: expected struct sched_domain *[assigned] sd
kernel/sched/fair.c:6719:20: sparse: got struct sched_domain [noderef] __rcu *parent
kernel/sched/fair.c:6853:9: sparse: sparse: incorrect type in assignment (different address spaces) @@ expected struct sched_domain *[assigned] tmp @@ got struct sched_domain [noderef] __rcu *parent @@
kernel/sched/fair.c:6853:9: sparse: expected struct sched_domain *[assigned] tmp
kernel/sched/fair.c:6853:9: sparse: got struct sched_domain [noderef] __rcu *parent
kernel/sched/fair.c:7051:38: sparse: sparse: incorrect type in initializer (different address spaces) @@ expected struct task_struct *curr @@ got struct task_struct [noderef] __rcu *curr @@
kernel/sched/fair.c:7051:38: sparse: expected struct task_struct *curr
kernel/sched/fair.c:7051:38: sparse: got struct task_struct [noderef] __rcu *curr
kernel/sched/fair.c:7335:38: sparse: sparse: incorrect type in initializer (different address spaces) @@ expected struct task_struct *curr @@ got struct task_struct [noderef] __rcu *curr @@
kernel/sched/fair.c:7335:38: sparse: expected struct task_struct *curr
kernel/sched/fair.c:7335:38: sparse: got struct task_struct [noderef] __rcu *curr
kernel/sched/fair.c:8320:40: sparse: sparse: incorrect type in initializer (different address spaces) @@ expected struct sched_domain *child @@ got struct sched_domain [noderef] __rcu *child @@
kernel/sched/fair.c:8320:40: sparse: expected struct sched_domain *child
kernel/sched/fair.c:8320:40: sparse: got struct sched_domain [noderef] __rcu *child
kernel/sched/fair.c:8768:22: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/sched/fair.c:8768:22: sparse: struct task_struct [noderef] __rcu *
kernel/sched/fair.c:8768:22: sparse: struct task_struct *
kernel/sched/fair.c:10031:9: sparse: sparse: incorrect type in assignment (different address spaces) @@ expected struct sched_domain *[assigned] sd @@ got struct sched_domain [noderef] __rcu *parent @@
kernel/sched/fair.c:10031:9: sparse: expected struct sched_domain *[assigned] sd
kernel/sched/fair.c:10031:9: sparse: got struct sched_domain [noderef] __rcu *parent
kernel/sched/fair.c:9691:44: sparse: sparse: incorrect type in initializer (different address spaces) @@ expected struct sched_domain *sd_parent @@ got struct sched_domain [noderef] __rcu *parent @@
kernel/sched/fair.c:9691:44: sparse: expected struct sched_domain *sd_parent
kernel/sched/fair.c:9691:44: sparse: got struct sched_domain [noderef] __rcu *parent
kernel/sched/fair.c:10103:9: sparse: sparse: incorrect type in assignment (different address spaces) @@ expected struct sched_domain *[assigned] sd @@ got struct sched_domain [noderef] __rcu *parent @@
kernel/sched/fair.c:10103:9: sparse: expected struct sched_domain *[assigned] sd
kernel/sched/fair.c:10103:9: sparse: got struct sched_domain [noderef] __rcu *parent
kernel/sched/fair.c:2421:9: sparse: sparse: context imbalance in 'task_numa_placement' - different lock contexts for basic block
kernel/sched/fair.c:4597:31: sparse: sparse: marked inline, but without a definition
kernel/sched/fair.c: note: in included file:
kernel/sched/sched.h:2169:9: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/sched/sched.h:2169:9: sparse: struct task_struct [noderef] __rcu *
kernel/sched/sched.h:2169:9: sparse: struct task_struct *
kernel/sched/sched.h:2011:25: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/sched/sched.h:2011:25: sparse: struct task_struct [noderef] __rcu *
kernel/sched/sched.h:2011:25: sparse: struct task_struct *
kernel/sched/sched.h:2011:25: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/sched/sched.h:2011:25: sparse: struct task_struct [noderef] __rcu *
kernel/sched/sched.h:2011:25: sparse: struct task_struct *
Please review and possibly fold the followup patch.
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/[email protected]
On Wed, 14 Jul 2021 at 13:39, Valentin Schneider
<[email protected]> wrote:
>
> Consider a system with some NOHZ-idle CPUs, such that
>
> nohz.idle_cpus_mask = S
> nohz.next_balance = T
>
> When a new CPU k goes NOHZ idle (nohz_balance_enter_idle()), we end up
> with:
>
> nohz.idle_cpus_mask = S \U {k}
> nohz.next_balance = T
>
> Note that the nohz.next_balance hasn't changed - it won't be updated until
> a NOHZ balance is triggered. This is problematic if the newly NOHZ idle CPU
> has an earlier rq.next_balance than the other NOHZ idle CPUs, IOW if:
>
> cpu_rq(k).next_balance < nohz.next_balance
>
> In such scenarios, the existing nohz.next_balance will prevent any NOHZ
> balance from happening, which itself will prevent nohz.next_balance from
> being updated to this new cpu_rq(k).next_balance. Unnecessary load balance
> delays of over 12ms caused by this were observed on an arm64 RB5 board.
How many CPUs has the arm64 RB5 ?
>
> Track which CPUs are iterated over during a NOHZ idle balance with a new
> cpumask. When considering whether to kick a NOHZ idle balance, use this
> cpumask to determine if any CPU has entered NOHZ idle but hasn't had its
> rq.next_balance collated into nohz.next_balance yet, and kick a NOHZ_STATS
> balance if it is the case.
>
> Signed-off-by: Valentin Schneider <[email protected]>
> ---
> kernel/sched/core.c | 8 ++++++++
> kernel/sched/fair.c | 19 +++++++++++++++++--
> 2 files changed, 25 insertions(+), 2 deletions(-)
>
> diff --git a/kernel/sched/core.c b/kernel/sched/core.c
> index 0c22cd026440..1bc4cbc1f85e 100644
> --- a/kernel/sched/core.c
> +++ b/kernel/sched/core.c
> @@ -8893,6 +8893,10 @@ static struct kmem_cache *task_group_cache __read_mostly;
> DECLARE_PER_CPU(cpumask_var_t, load_balance_mask);
> DECLARE_PER_CPU(cpumask_var_t, select_idle_mask);
>
> +#ifdef CONFIG_NOHZ_COMMON
> +DECLARE_PER_CPU(cpumask_var_t, nohz_balance_mask);
> +#endif /* CONFIG_NOHZ_COMMON */
> +
> void __init sched_init(void)
> {
> unsigned long ptr = 0;
> @@ -8942,6 +8946,10 @@ void __init sched_init(void)
> cpumask_size(), GFP_KERNEL, cpu_to_node(i));
> per_cpu(select_idle_mask, i) = (cpumask_var_t)kzalloc_node(
> cpumask_size(), GFP_KERNEL, cpu_to_node(i));
> +#ifdef CONFIG_NOHZ_COMMON
> + per_cpu(nohz_balance_mask, i) = (cpumask_var_t)kzalloc_node(
> + cpumask_size(), GFP_KERNEL, cpu_to_node(i));
> +#endif /* CONFIG_NOHZ_COMMON */
> }
> #endif /* CONFIG_CPUMASK_OFFSTACK */
>
> diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
> index 11d22943753f..497208a1afb8 100644
> --- a/kernel/sched/fair.c
> +++ b/kernel/sched/fair.c
> @@ -5694,8 +5694,11 @@ DEFINE_PER_CPU(cpumask_var_t, select_idle_mask);
>
> #ifdef CONFIG_NO_HZ_COMMON
>
> +DEFINE_PER_CPU(cpumask_var_t, nohz_balance_mask);
> +
> static struct {
> - cpumask_var_t idle_cpus_mask;
> + cpumask_var_t idle_cpus_mask; /* CPUs in NOHZ idle */
> + cpumask_var_t last_balance_mask; /* CPUs covered by last NOHZ balance */
> atomic_t nr_cpus;
> int has_blocked; /* Idle CPUS has blocked load */
> unsigned long next_balance; /* in jiffy units */
> @@ -10351,6 +10354,13 @@ static void nohz_balancer_kick(struct rq *rq)
> unlock:
> rcu_read_unlock();
> out:
> + /*
> + * Some CPUs have recently gone into NOHZ idle; kick a balance to
> + * collate the proper next balance interval.
> + */
> + if (!cpumask_subset(nohz.idle_cpus_mask, nohz.last_balance_mask))
I don't really like having to manipulate a cpumask just to trigger an
ILB and force the update of nohz.next_balance. Could we use something
similar to nohz.has_blocked and introduce a nohz.force_update.
manipulating cpumask will even be more complex if we start to have a
per node idle_cpus_mask like proposed here:
https://lore.kernel.org/lkml/[email protected]/
Also
Something like below is simpler
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 44e44c235f1f..91c314f58982 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -10657,6 +10657,9 @@ static void nohz_newidle_balance(struct rq *this_rq)
if (this_rq->avg_idle < sysctl_sched_migration_cost)
return;
+ if (time_before(this_rq->next_balance, READ_ONCE(nohz.next_balance))
+ WRITE_ONCE(nohz.need_update, 1);
+
/* Don't need to update blocked load of idle CPUs*/
if (!READ_ONCE(nohz.has_blocked) ||
time_before(jiffies, READ_ONCE(nohz.next_blocked)))
Then we have to test nohz.need_update in nohz_balancer_kick()
> + flags |= NOHZ_STATS_KICK;
people complain that an update of blocked load is time consuming so we
should not kick this update unnecessarily.
We should introduce a new bit like NOHZ_NEXT_KICK that will only go
through idle cpus and update nohz.next_balance
> +
> if (flags)
> kick_ilb(flags);
> }
> @@ -10487,6 +10497,7 @@ static bool update_nohz_stats(struct rq *rq)
> static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags,
> enum cpu_idle_type idle)
> {
> + struct cpumask *cpus = this_cpu_cpumask_var_ptr(nohz_balance_mask);
> /* Earliest time when we have to do rebalance again */
> unsigned long now = jiffies;
> unsigned long next_balance = now + 60*HZ;
> @@ -10518,7 +10529,8 @@ static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags,
> * Start with the next CPU after this_cpu so we will end with this_cpu and let a
> * chance for other idle cpu to pull load.
> */
> - for_each_cpu_wrap(balance_cpu, nohz.idle_cpus_mask, this_cpu+1) {
> + cpumask_copy(cpus, nohz.idle_cpus_mask);
we are not sure to go through all idle cpus and ilb can abort
> + for_each_cpu_wrap(balance_cpu, cpus, this_cpu+1) {
> if (!idle_cpu(balance_cpu))
> continue;
>
> @@ -10565,6 +10577,8 @@ static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags,
> if (likely(update_next_balance))
> nohz.next_balance = next_balance;
>
> + cpumask_copy(nohz.last_balance_mask, cpus);
> +
> WRITE_ONCE(nohz.next_blocked,
> now + msecs_to_jiffies(LOAD_AVG_PERIOD));
>
> @@ -11550,6 +11564,7 @@ __init void init_sched_fair_class(void)
> nohz.next_balance = jiffies;
> nohz.next_blocked = jiffies;
> zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT);
> + zalloc_cpumask_var(&nohz.last_balance_mask, GFP_NOWAIT);
> #endif
> #endif /* SMP */
>
> --
> 2.25.1
>
Hi Vincent,
Thanks for taking a look.
On 15/07/21 09:42, Vincent Guittot wrote:
> On Wed, 14 Jul 2021 at 13:39, Valentin Schneider
> <[email protected]> wrote:
>>
>> Consider a system with some NOHZ-idle CPUs, such that
>>
>> nohz.idle_cpus_mask = S
>> nohz.next_balance = T
>>
>> When a new CPU k goes NOHZ idle (nohz_balance_enter_idle()), we end up
>> with:
>>
>> nohz.idle_cpus_mask = S \U {k}
>> nohz.next_balance = T
>>
>> Note that the nohz.next_balance hasn't changed - it won't be updated until
>> a NOHZ balance is triggered. This is problematic if the newly NOHZ idle CPU
>> has an earlier rq.next_balance than the other NOHZ idle CPUs, IOW if:
>>
>> cpu_rq(k).next_balance < nohz.next_balance
>>
>> In such scenarios, the existing nohz.next_balance will prevent any NOHZ
>> balance from happening, which itself will prevent nohz.next_balance from
>> being updated to this new cpu_rq(k).next_balance. Unnecessary load balance
>> delays of over 12ms caused by this were observed on an arm64 RB5 board.
>
> How many CPUs has the arm64 RB5 ?
That's an 8 CPU DynamIQ system - 4 littles, 3 bigs and one "huge". That
should give us a regular balance_interval of 8ms, but our tests have picked
up CPUs staying idle for >20ms when they really have stuff to pull. In this
case balance_interval increases are involved.
>> @@ -10351,6 +10354,13 @@ static void nohz_balancer_kick(struct rq *rq)
>> unlock:
>> rcu_read_unlock();
>> out:
>> + /*
>> + * Some CPUs have recently gone into NOHZ idle; kick a balance to
>> + * collate the proper next balance interval.
>> + */
>> + if (!cpumask_subset(nohz.idle_cpus_mask, nohz.last_balance_mask))
>
> I don't really like having to manipulate a cpumask just to trigger an
> ILB and force the update of nohz.next_balance. Could we use something
> similar to nohz.has_blocked and introduce a nohz.force_update.
> manipulating cpumask will even be more complex if we start to have a
> per node idle_cpus_mask like proposed here:
> https://lore.kernel.org/lkml/[email protected]/
>
> Also
>
>
> Something like below is simpler
>
> diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
> index 44e44c235f1f..91c314f58982 100644
> --- a/kernel/sched/fair.c
> +++ b/kernel/sched/fair.c
> @@ -10657,6 +10657,9 @@ static void nohz_newidle_balance(struct rq *this_rq)
> if (this_rq->avg_idle < sysctl_sched_migration_cost)
> return;
>
> + if (time_before(this_rq->next_balance, READ_ONCE(nohz.next_balance))
> + WRITE_ONCE(nohz.need_update, 1);
> +
I think we have to do this unconditionally, as we can observe the old
nohz.next_balance while a NOHZ balance is ongoing (which will update
nohz.next_balance without taking into account this newly idle CPU).
> /* Don't need to update blocked load of idle CPUs*/
> if (!READ_ONCE(nohz.has_blocked) ||
> time_before(jiffies, READ_ONCE(nohz.next_blocked)))
>
>
> Then we have to test nohz.need_update in nohz_balancer_kick()
>
But then, when can we safely clear this new nohz.need_update? We can't do
it unconditionally in nohz_idle_balance() as this could race with a new CPU
going NOHZ idle.
Perhaps instead we could have a single nohz.needs_update_mask, the CPU is
set in nohz_newidle_balance(), cleared when iterated over in
_nohz_idle_balance(), and nohz_balancer_kick() can trigger an
e.g. NOHZ_UPDATE_KICK if this new cpumask is non-empty.
>> + flags |= NOHZ_STATS_KICK;
>
> people complain that an update of blocked load is time consuming so we
> should not kick this update unnecessarily.
> We should introduce a new bit like NOHZ_NEXT_KICK that will only go
> through idle cpus and update nohz.next_balance
>
That sounds reasonable.
>> +
>> if (flags)
>> kick_ilb(flags);
>> }
>> @@ -10487,6 +10497,7 @@ static bool update_nohz_stats(struct rq *rq)
>> static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags,
>> enum cpu_idle_type idle)
>> {
>> + struct cpumask *cpus = this_cpu_cpumask_var_ptr(nohz_balance_mask);
>> /* Earliest time when we have to do rebalance again */
>> unsigned long now = jiffies;
>> unsigned long next_balance = now + 60*HZ;
>> @@ -10518,7 +10529,8 @@ static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags,
>> * Start with the next CPU after this_cpu so we will end with this_cpu and let a
>> * chance for other idle cpu to pull load.
>> */
>> - for_each_cpu_wrap(balance_cpu, nohz.idle_cpus_mask, this_cpu+1) {
>> + cpumask_copy(cpus, nohz.idle_cpus_mask);
>
> we are not sure to go through all idle cpus and ilb can abort
>
Right, this is missing something to re-kick an update, but I think we can
get rid of that entirely...
On Thu, 15 Jul 2021 at 13:56, Valentin Schneider
<[email protected]> wrote:
>
> Hi Vincent,
>
> Thanks for taking a look.
>
> On 15/07/21 09:42, Vincent Guittot wrote:
> > On Wed, 14 Jul 2021 at 13:39, Valentin Schneider
> > <[email protected]> wrote:
> >>
> >> Consider a system with some NOHZ-idle CPUs, such that
> >>
> >> nohz.idle_cpus_mask = S
> >> nohz.next_balance = T
> >>
> >> When a new CPU k goes NOHZ idle (nohz_balance_enter_idle()), we end up
> >> with:
> >>
> >> nohz.idle_cpus_mask = S \U {k}
> >> nohz.next_balance = T
> >>
> >> Note that the nohz.next_balance hasn't changed - it won't be updated until
> >> a NOHZ balance is triggered. This is problematic if the newly NOHZ idle CPU
> >> has an earlier rq.next_balance than the other NOHZ idle CPUs, IOW if:
> >>
> >> cpu_rq(k).next_balance < nohz.next_balance
> >>
> >> In such scenarios, the existing nohz.next_balance will prevent any NOHZ
> >> balance from happening, which itself will prevent nohz.next_balance from
> >> being updated to this new cpu_rq(k).next_balance. Unnecessary load balance
> >> delays of over 12ms caused by this were observed on an arm64 RB5 board.
> >
> > How many CPUs has the arm64 RB5 ?
>
> That's an 8 CPU DynamIQ system - 4 littles, 3 bigs and one "huge". That
> should give us a regular balance_interval of 8ms, but our tests have picked
> up CPUs staying idle for >20ms when they really have stuff to pull. In this
> case balance_interval increases are involved.
>
> >> @@ -10351,6 +10354,13 @@ static void nohz_balancer_kick(struct rq *rq)
> >> unlock:
> >> rcu_read_unlock();
> >> out:
> >> + /*
> >> + * Some CPUs have recently gone into NOHZ idle; kick a balance to
> >> + * collate the proper next balance interval.
> >> + */
> >> + if (!cpumask_subset(nohz.idle_cpus_mask, nohz.last_balance_mask))
> >
> > I don't really like having to manipulate a cpumask just to trigger an
> > ILB and force the update of nohz.next_balance. Could we use something
> > similar to nohz.has_blocked and introduce a nohz.force_update.
> > manipulating cpumask will even be more complex if we start to have a
> > per node idle_cpus_mask like proposed here:
> > https://lore.kernel.org/lkml/[email protected]/
> >
> > Also
> >
> >
> > Something like below is simpler
> >
> > diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
> > index 44e44c235f1f..91c314f58982 100644
> > --- a/kernel/sched/fair.c
> > +++ b/kernel/sched/fair.c
> > @@ -10657,6 +10657,9 @@ static void nohz_newidle_balance(struct rq *this_rq)
> > if (this_rq->avg_idle < sysctl_sched_migration_cost)
> > return;
> >
> > + if (time_before(this_rq->next_balance, READ_ONCE(nohz.next_balance))
> > + WRITE_ONCE(nohz.need_update, 1);
> > +
>
> I think we have to do this unconditionally, as we can observe the old
> nohz.next_balance while a NOHZ balance is ongoing (which will update
> nohz.next_balance without taking into account this newly idle CPU).
so maybe add this in nohz_balance_enter_idle() after the
smp_mb__after_atomic(). Ilb will see the cpu in the idle_cpus_mask so
even if nohz.next_balance is updated, it will take into account this
newly idle cpu
My goal was to use mechanism similar to what is used of nohz.has_blocked
>
>
> > /* Don't need to update blocked load of idle CPUs*/
> > if (!READ_ONCE(nohz.has_blocked) ||
> > time_before(jiffies, READ_ONCE(nohz.next_blocked)))
> >
> >
> > Then we have to test nohz.need_update in nohz_balancer_kick()
> >
>
> But then, when can we safely clear this new nohz.need_update? We can't do
> it unconditionally in nohz_idle_balance() as this could race with a new CPU
> going NOHZ idle.
not with the proposal above
>
> Perhaps instead we could have a single nohz.needs_update_mask, the CPU is
> set in nohz_newidle_balance(), cleared when iterated over in
> _nohz_idle_balance(), and nohz_balancer_kick() can trigger an
> e.g. NOHZ_UPDATE_KICK if this new cpumask is non-empty.
>
> >> + flags |= NOHZ_STATS_KICK;
> >
> > people complain that an update of blocked load is time consuming so we
> > should not kick this update unnecessarily.
> > We should introduce a new bit like NOHZ_NEXT_KICK that will only go
> > through idle cpus and update nohz.next_balance
> >
>
> That sounds reasonable.
>
> >> +
> >> if (flags)
> >> kick_ilb(flags);
> >> }
> >> @@ -10487,6 +10497,7 @@ static bool update_nohz_stats(struct rq *rq)
> >> static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags,
> >> enum cpu_idle_type idle)
> >> {
> >> + struct cpumask *cpus = this_cpu_cpumask_var_ptr(nohz_balance_mask);
> >> /* Earliest time when we have to do rebalance again */
> >> unsigned long now = jiffies;
> >> unsigned long next_balance = now + 60*HZ;
> >> @@ -10518,7 +10529,8 @@ static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags,
> >> * Start with the next CPU after this_cpu so we will end with this_cpu and let a
> >> * chance for other idle cpu to pull load.
> >> */
> >> - for_each_cpu_wrap(balance_cpu, nohz.idle_cpus_mask, this_cpu+1) {
> >> + cpumask_copy(cpus, nohz.idle_cpus_mask);
> >
> > we are not sure to go through all idle cpus and ilb can abort
> >
>
> Right, this is missing something to re-kick an update, but I think we can
> get rid of that entirely...
On 14/07/2021 13:39, Valentin Schneider wrote:
[...]
> @@ -8942,6 +8946,10 @@ void __init sched_init(void)
> cpumask_size(), GFP_KERNEL, cpu_to_node(i));
> per_cpu(select_idle_mask, i) = (cpumask_var_t)kzalloc_node(
> cpumask_size(), GFP_KERNEL, cpu_to_node(i));
> +#ifdef CONFIG_NOHZ_COMMON
> + per_cpu(nohz_balance_mask, i) = (cpumask_var_t)kzalloc_node(
> + cpumask_size(), GFP_KERNEL, cpu_to_node(i));
> +#endif /* CONFIG_NOHZ_COMMON */
> }
> #endif /* CONFIG_CPUMASK_OFFSTACK */
s/CONFIG_NOHZ_COMMON/CONFIG_NO_HZ_COMMON
Otherwise CONFIG_CPUMASK_OFFSTACK system will have issues.
[...]
On 15/07/21 15:01, Vincent Guittot wrote:
> On Thu, 15 Jul 2021 at 13:56, Valentin Schneider <[email protected]> wrote:
>> On 15/07/21 09:42, Vincent Guittot wrote:
>> > diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
>> > index 44e44c235f1f..91c314f58982 100644
>> > --- a/kernel/sched/fair.c
>> > +++ b/kernel/sched/fair.c
>> > @@ -10657,6 +10657,9 @@ static void nohz_newidle_balance(struct rq *this_rq)
>> > if (this_rq->avg_idle < sysctl_sched_migration_cost)
>> > return;
>> >
>> > + if (time_before(this_rq->next_balance, READ_ONCE(nohz.next_balance))
>> > + WRITE_ONCE(nohz.need_update, 1);
>> > +
>>
>> I think we have to do this unconditionally, as we can observe the old
>> nohz.next_balance while a NOHZ balance is ongoing (which will update
>> nohz.next_balance without taking into account this newly idle CPU).
>
> so maybe add this in nohz_balance_enter_idle() after the
> smp_mb__after_atomic(). Ilb will see the cpu in the idle_cpus_mask so
> even if nohz.next_balance is updated, it will take into account this
> newly idle cpu
>
> My goal was to use mechanism similar to what is used of nohz.has_blocked
>
OK, and then clearing it above the smp_mb() in _nohz_idle_balance() should
give us similar guarantees to nohz.has_blocked (i.e. if we don't observe
the cpumask write, then we'll observe the needs_update write).
Thanks for the suggestion, I'll go test this out.
Greeting,
FYI, we noticed the following commit (built with gcc-9):
commit: cbd87e97caf59c1a9d06d35e5a59404e4d7c8660 ("[PATCH] sched/fair: Update nohz.next_balance for newly NOHZ-idle CPUs")
url: https://github.com/0day-ci/linux/commits/Valentin-Schneider/sched-fair-Update-nohz-next_balance-for-newly-NOHZ-idle-CPUs/20210714-194021
base: https://git.kernel.org/cgit/linux/kernel/git/tip/tip.git 031e3bd8986fffe31e1ddbf5264cccfe30c9abd7
in testcase: trinity
version: trinity-x86_64-da65f0aa-1_20210719
with following parameters:
number: 99999
group: group-03
test-description: Trinity is a linux system call fuzz tester.
test-url: http://codemonkey.org.uk/projects/trinity/
on test machine: qemu-system-x86_64 -enable-kvm -cpu SandyBridge -smp 2 -m 16G
caused below changes (please refer to attached dmesg/kmsg for entire log/backtrace):
If you fix the issue, kindly add following tag
Reported-by: kernel test robot <[email protected]>
[ 11.102934][ C1] BUG: kernel NULL pointer dereference, address: 0000000000000000
[ 11.104253][ C1] #PF: supervisor write access in kernel mode
[ 11.105209][ C1] #PF: error_code(0x0002) - not-present page
[ 11.106215][ C1] PGD 0 P4D 0
[ 11.106848][ C1] Oops: 0002 [#1] SMP PTI
[ 11.106919][ C1] CPU: 1 PID: 0 Comm: swapper/1 Not tainted 5.13.0-rc6-00081-gcbd87e97caf5 #1
[ 11.106919][ C1] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.12.0-1 04/01/2014
[ 11.106919][ C1] RIP: 0010:__memcpy (arch/x86/lib/memcpy_64.S:39)
[ 11.106919][ C1] Code: 74 be 0f 1f 44 00 00 c7 05 97 29 d1 03 0f 00 00 00 eb ad cc cc cc cc cc 0f 1f 44 00 00 48 89 f8 48 89 d1 48 c1 e9 03 83 e2 07 <f3> 48 a5 89 d1 f3 a4 c3 66 0f 1f 44 00 00 48 89 f8 48 89 d1 f3 a4
All code
========
0: 74 be je 0xffffffffffffffc0
2: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1)
7: c7 05 97 29 d1 03 0f movl $0xf,0x3d12997(%rip) # 0x3d129a8
e: 00 00 00
11: eb ad jmp 0xffffffffffffffc0
13: cc int3
14: cc int3
15: cc int3
16: cc int3
17: cc int3
18: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1)
1d: 48 89 f8 mov %rdi,%rax
20: 48 89 d1 mov %rdx,%rcx
23: 48 c1 e9 03 shr $0x3,%rcx
27: 83 e2 07 and $0x7,%edx
2a:* f3 48 a5 rep movsq %ds:(%rsi),%es:(%rdi) <-- trapping instruction
2d: 89 d1 mov %edx,%ecx
2f: f3 a4 rep movsb %ds:(%rsi),%es:(%rdi)
31: c3 retq
32: 66 0f 1f 44 00 00 nopw 0x0(%rax,%rax,1)
38: 48 89 f8 mov %rdi,%rax
3b: 48 89 d1 mov %rdx,%rcx
3e: f3 a4 rep movsb %ds:(%rsi),%es:(%rdi)
Code starting with the faulting instruction
===========================================
0: f3 48 a5 rep movsq %ds:(%rsi),%es:(%rdi)
3: 89 d1 mov %edx,%ecx
5: f3 a4 rep movsb %ds:(%rsi),%es:(%rdi)
7: c3 retq
8: 66 0f 1f 44 00 00 nopw 0x0(%rax,%rax,1)
e: 48 89 f8 mov %rdi,%rax
11: 48 89 d1 mov %rdx,%rcx
14: f3 a4 rep movsb %ds:(%rsi),%es:(%rdi)
[ 11.106919][ C1] RSP: 0000:ffffa8f280120f18 EFLAGS: 00010246
[ 11.106919][ C1] RAX: 0000000000000000 RBX: 00000000ffff1d2f RCX: 0000000000000001
[ 11.106919][ C1] RDX: 0000000000000000 RSI: ffff8b4d002b6ed0 RDI: 0000000000000000
[ 11.106919][ C1] RBP: ffffa8f280120f80 R08: 0000000000000001 R09: 0000000000000001
[ 11.106919][ C1] R10: ffffffffacc07000 R11: ffff8b4d01589dc0 R12: 0000000000000000
[ 11.106919][ C1] R13: 0000000000000001 R14: 0000000000000002 R15: 0000000000000007
[ 11.106919][ C1] FS: 0000000000000000(0000) GS:ffff8b502fa00000(0000) knlGS:0000000000000000
[ 11.106919][ C1] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 11.106919][ C1] CR2: 0000000000000000 CR3: 00000002e58b6000 CR4: 00000000000406e0
[ 11.106919][ C1] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 11.106919][ C1] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 11.106919][ C1] Call Trace:
[ 11.106919][ C1] <IRQ>
[ 11.106919][ C1] _nohz_idle_balance+0x7a/0x400
[ 11.106919][ C1] ? lock_is_held_type (arch/x86/include/asm/irqflags.h:140 kernel/locking/lockdep.c:5557)
[ 11.106919][ C1] __do_softirq (arch/x86/include/asm/jump_label.h:19 include/linux/jump_label.h:200 include/trace/events/irq.h:142 kernel/softirq.c:559)
[ 11.106919][ C1] irq_exit_rcu (kernel/softirq.c:432 kernel/softirq.c:636 kernel/softirq.c:648)
[ 11.106919][ C1] sysvec_call_function_single (arch/x86/kernel/smp.c:243 (discriminator 14))
[ 11.106919][ C1] </IRQ>
[ 11.106919][ C1] asm_sysvec_call_function_single (arch/x86/include/asm/idtentry.h:655)
[ 11.106919][ C1] RIP: 0010:native_safe_halt (arch/x86/include/asm/irqflags.h:52)
[ 11.106919][ C1] Code: 00 0f 00 2d 16 e1 45 00 f4 c3 66 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 80 00 00 00 00 e9 07 00 00 00 0f 00 2d f6 e0 45 00 fb f4 <c3> cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc 0f 1f 44 00
All code
========
0: 00 0f add %cl,(%rdi)
2: 00 2d 16 e1 45 00 add %ch,0x45e116(%rip) # 0x45e11e
8: f4 hlt
9: c3 retq
a: 66 66 2e 0f 1f 84 00 data16 nopw %cs:0x0(%rax,%rax,1)
11: 00 00 00 00
15: 0f 1f 80 00 00 00 00 nopl 0x0(%rax)
1c: e9 07 00 00 00 jmpq 0x28
21: 0f 00 2d f6 e0 45 00 verw 0x45e0f6(%rip) # 0x45e11e
28: fb sti
29: f4 hlt
2a:* c3 retq <-- trapping instruction
2b: cc int3
2c: cc int3
2d: cc int3
2e: cc int3
2f: cc int3
30: cc int3
31: cc int3
32: cc int3
33: cc int3
34: cc int3
35: cc int3
36: cc int3
37: cc int3
38: cc int3
39: cc int3
3a: cc int3
3b: cc int3
3c: 0f .byte 0xf
3d: 1f (bad)
3e: 44 rex.R
...
Code starting with the faulting instruction
===========================================
0: c3 retq
1: cc int3
2: cc int3
3: cc int3
4: cc int3
5: cc int3
6: cc int3
7: cc int3
8: cc int3
9: cc int3
a: cc int3
b: cc int3
c: cc int3
d: cc int3
e: cc int3
f: cc int3
10: cc int3
11: cc int3
12: 0f .byte 0xf
13: 1f (bad)
14: 44 rex.R
To reproduce:
# build kernel
cd linux
cp config-5.13.0-rc6-00081-gcbd87e97caf5 .config
make HOSTCC=gcc-9 CC=gcc-9 ARCH=x86_64 olddefconfig prepare modules_prepare bzImage
git clone https://github.com/intel/lkp-tests.git
cd lkp-tests
bin/lkp qemu -k <bzImage> job-script # job-script is attached in this email
---
0DAY/LKP+ Test Infrastructure Open Source Technology Center
https://lists.01.org/hyperkitty/list/[email protected] Intel Corporation
Thanks,
Oliver Sang