Give every class a set_cpus_allowed() method, this enables some small
optimization in the rt,dl implementation by avoiding a double
cpumask_weight() call.
Signed-off-by: Peter Zijlstra (Intel) <[email protected]>
---
kernel/sched/core.c | 17 +++++++++++------
kernel/sched/deadline.c | 20 ++++++++++++--------
kernel/sched/fair.c | 1 +
kernel/sched/idle_task.c | 1 +
kernel/sched/rt.c | 12 ++++++++----
kernel/sched/sched.h | 2 ++
kernel/sched/stop_task.c | 1 +
7 files changed, 36 insertions(+), 18 deletions(-)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -4791,17 +4791,22 @@ static struct rq *move_queued_task(struc
return rq;
}
-void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
+/*
+ * sched_class::set_cpus_allowed must do the below, but is not required to
+ * actually call this function.
+ */
+void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask)
{
- lockdep_assert_held(&p->pi_lock);
-
- if (p->sched_class->set_cpus_allowed)
- p->sched_class->set_cpus_allowed(p, new_mask);
-
cpumask_copy(&p->cpus_allowed, new_mask);
p->nr_cpus_allowed = cpumask_weight(new_mask);
}
+void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
+{
+ lockdep_assert_held(&p->pi_lock);
+ p->sched_class->set_cpus_allowed(p, new_mask);
+}
+
/*
* This is how migration works:
*
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -1597,13 +1597,6 @@ static void set_cpus_allowed_dl(struct t
raw_spin_unlock(&src_dl_b->lock);
}
- /*
- * Update only if the task is actually running (i.e.,
- * it is on the rq AND it is not throttled).
- */
- if (!on_dl_rq(&p->dl))
- return;
-
weight = cpumask_weight(new_mask);
/*
@@ -1611,7 +1604,14 @@ static void set_cpus_allowed_dl(struct t
* can migrate or not.
*/
if ((p->nr_cpus_allowed > 1) == (weight > 1))
- return;
+ goto done;
+
+ /*
+ * Update only if the task is actually running (i.e.,
+ * it is on the rq AND it is not throttled).
+ */
+ if (!on_dl_rq(&p->dl))
+ goto done;
/*
* The process used to be able to migrate OR it can now migrate
@@ -1628,6 +1628,10 @@ static void set_cpus_allowed_dl(struct t
}
update_dl_migration(&rq->dl);
+
+done:
+ cpumask_copy(&p->cpus_allowed, new_mask);
+ p->nr_cpus_allowed = weight;
}
/* Assumes rq->lock is held */
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -8406,6 +8406,7 @@ const struct sched_class fair_sched_clas
.rq_offline = rq_offline_fair,
.task_waking = task_waking_fair,
+ .set_cpus_allowed = set_cpus_allowed_common,
#endif
.set_curr_task = set_curr_task_fair,
--- a/kernel/sched/idle_task.c
+++ b/kernel/sched/idle_task.c
@@ -96,6 +96,7 @@ const struct sched_class idle_sched_clas
#ifdef CONFIG_SMP
.select_task_rq = select_task_rq_idle,
+ .set_cpus_allowed = set_cpus_allowed_common,
#endif
.set_curr_task = set_curr_task_idle,
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -2065,9 +2065,6 @@ static void set_cpus_allowed_rt(struct t
BUG_ON(!rt_task(p));
- if (!task_on_rq_queued(p))
- return;
-
weight = cpumask_weight(new_mask);
/*
@@ -2075,7 +2072,10 @@ static void set_cpus_allowed_rt(struct t
* can migrate or not.
*/
if ((p->nr_cpus_allowed > 1) == (weight > 1))
- return;
+ goto done;
+
+ if (!task_on_rq_queued(p))
+ goto done;
rq = task_rq(p);
@@ -2094,6 +2094,10 @@ static void set_cpus_allowed_rt(struct t
}
update_rt_migration(&rq->rt);
+
+done:
+ cpumask_copy(&p->cpus_allowed, new_mask);
+ p->nr_cpus_allowed = weight;
}
/* Assumes rq->lock is held */
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1252,6 +1252,8 @@ extern void trigger_load_balance(struct
extern void idle_enter_fair(struct rq *this_rq);
extern void idle_exit_fair(struct rq *this_rq);
+extern void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask);
+
#else
static inline void idle_enter_fair(struct rq *rq) { }
--- a/kernel/sched/stop_task.c
+++ b/kernel/sched/stop_task.c
@@ -123,6 +123,7 @@ const struct sched_class stop_sched_clas
#ifdef CONFIG_SMP
.select_task_rq = select_task_rq_stop,
+ .set_cpus_allowed = set_cpus_allowed_common,
#endif
.set_curr_task = set_curr_task_stop,
Commit-ID: c5b2803840817115e9b568d5054e5007ae36176b
Gitweb: http://git.kernel.org/tip/c5b2803840817115e9b568d5054e5007ae36176b
Author: Peter Zijlstra <[email protected]>
AuthorDate: Fri, 15 May 2015 17:43:35 +0200
Committer: Ingo Molnar <[email protected]>
CommitDate: Wed, 12 Aug 2015 12:06:09 +0200
sched: Make sched_class::set_cpus_allowed() unconditional
Give every class a set_cpus_allowed() method, this enables some small
optimization in the RT,DL implementation by avoiding a double
cpumask_weight() call.
Signed-off-by: Peter Zijlstra (Intel) <[email protected]>
Cc: Linus Torvalds <[email protected]>
Cc: Mike Galbraith <[email protected]>
Cc: Peter Zijlstra <[email protected]>
Cc: Thomas Gleixner <[email protected]>
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Link: http://lkml.kernel.org/r/[email protected]
Signed-off-by: Ingo Molnar <[email protected]>
---
kernel/sched/core.c | 17 +++++++++++------
kernel/sched/deadline.c | 20 ++++++++++++--------
kernel/sched/fair.c | 1 +
kernel/sched/idle_task.c | 1 +
kernel/sched/rt.c | 12 ++++++++----
kernel/sched/sched.h | 2 ++
kernel/sched/stop_task.c | 1 +
7 files changed, 36 insertions(+), 18 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 2e3b983..740f90b 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -1151,17 +1151,22 @@ static int migration_cpu_stop(void *data)
return 0;
}
-void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
+/*
+ * sched_class::set_cpus_allowed must do the below, but is not required to
+ * actually call this function.
+ */
+void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask)
{
- lockdep_assert_held(&p->pi_lock);
-
- if (p->sched_class->set_cpus_allowed)
- p->sched_class->set_cpus_allowed(p, new_mask);
-
cpumask_copy(&p->cpus_allowed, new_mask);
p->nr_cpus_allowed = cpumask_weight(new_mask);
}
+void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
+{
+ lockdep_assert_held(&p->pi_lock);
+ p->sched_class->set_cpus_allowed(p, new_mask);
+}
+
/*
* Change a given task's CPU affinity. Migrate the thread to a
* proper CPU and schedule it away if the CPU it's executing on
diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index 20772ee..dc357fa 100644
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -1696,13 +1696,6 @@ static void set_cpus_allowed_dl(struct task_struct *p,
raw_spin_unlock(&src_dl_b->lock);
}
- /*
- * Update only if the task is actually running (i.e.,
- * it is on the rq AND it is not throttled).
- */
- if (!on_dl_rq(&p->dl))
- return;
-
weight = cpumask_weight(new_mask);
/*
@@ -1710,7 +1703,14 @@ static void set_cpus_allowed_dl(struct task_struct *p,
* can migrate or not.
*/
if ((p->nr_cpus_allowed > 1) == (weight > 1))
- return;
+ goto done;
+
+ /*
+ * Update only if the task is actually running (i.e.,
+ * it is on the rq AND it is not throttled).
+ */
+ if (!on_dl_rq(&p->dl))
+ goto done;
/*
* The process used to be able to migrate OR it can now migrate
@@ -1727,6 +1727,10 @@ static void set_cpus_allowed_dl(struct task_struct *p,
}
update_dl_migration(&rq->dl);
+
+done:
+ cpumask_copy(&p->cpus_allowed, new_mask);
+ p->nr_cpus_allowed = weight;
}
/* Assumes rq->lock is held */
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index f0950fd..6e2e348 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -8252,6 +8252,7 @@ const struct sched_class fair_sched_class = {
.task_waking = task_waking_fair,
.task_dead = task_dead_fair,
+ .set_cpus_allowed = set_cpus_allowed_common,
#endif
.set_curr_task = set_curr_task_fair,
diff --git a/kernel/sched/idle_task.c b/kernel/sched/idle_task.c
index c65dac8..c4ae0f1 100644
--- a/kernel/sched/idle_task.c
+++ b/kernel/sched/idle_task.c
@@ -96,6 +96,7 @@ const struct sched_class idle_sched_class = {
#ifdef CONFIG_SMP
.select_task_rq = select_task_rq_idle,
+ .set_cpus_allowed = set_cpus_allowed_common,
#endif
.set_curr_task = set_curr_task_idle,
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 00816ee..63692ef 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -2084,9 +2084,6 @@ static void set_cpus_allowed_rt(struct task_struct *p,
BUG_ON(!rt_task(p));
- if (!task_on_rq_queued(p))
- return;
-
weight = cpumask_weight(new_mask);
/*
@@ -2094,7 +2091,10 @@ static void set_cpus_allowed_rt(struct task_struct *p,
* can migrate or not.
*/
if ((p->nr_cpus_allowed > 1) == (weight > 1))
- return;
+ goto done;
+
+ if (!task_on_rq_queued(p))
+ goto done;
rq = task_rq(p);
@@ -2113,6 +2113,10 @@ static void set_cpus_allowed_rt(struct task_struct *p,
}
update_rt_migration(&rq->rt);
+
+done:
+ cpumask_copy(&p->cpus_allowed, new_mask);
+ p->nr_cpus_allowed = weight;
}
/* Assumes rq->lock is held */
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 22ccc55..68cda11 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1255,6 +1255,8 @@ extern void trigger_load_balance(struct rq *rq);
extern void idle_enter_fair(struct rq *this_rq);
extern void idle_exit_fair(struct rq *this_rq);
+extern void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask);
+
#else
static inline void idle_enter_fair(struct rq *rq) { }
diff --git a/kernel/sched/stop_task.c b/kernel/sched/stop_task.c
index 79ffec4..cbc67da 100644
--- a/kernel/sched/stop_task.c
+++ b/kernel/sched/stop_task.c
@@ -123,6 +123,7 @@ const struct sched_class stop_sched_class = {
#ifdef CONFIG_SMP
.select_task_rq = select_task_rq_stop,
+ .set_cpus_allowed = set_cpus_allowed_common,
#endif
.set_curr_task = set_curr_task_stop,
On 05/15/2015 11:43 AM, Peter Zijlstra wrote:
> Give every class a set_cpus_allowed() method, this enables some small
> optimization in the rt,dl implementation by avoiding a double
> cpumask_weight() call.
>
> Signed-off-by: Peter Zijlstra (Intel) <[email protected]>
Hey Peter,
Here's the splat I mentioned earlier. I'm not sure if it's caused by this commit
or not, it's just that git blame took me here.
[392896.406248] WARNING: CPU: 9 PID: 51 at kernel/sched/core.c:1178 do_set_cpus_allowed+0x1cf/0x480()
[392896.407070] Modules linked in:
[392896.407408] CPU: 9 PID: 51 Comm: migration/9 Not tainted 4.2.0-rc6-next-20150817-sasha-00041-g9b6b2ab-dirty #2460
[392896.408376] ffffffffab2c2ac0 ffff880477857948 ffffffffaafaf5a9 0000000000000000
[392896.409126] ffff880477857988 ffffffffa1323246 ffffffffa13ab19f ffff880477284000
[392896.409874] ffff8804781e15c0 ffff88006bc03d98 ffffffffae480d20 0000000000000001
[392896.410610] Call Trace:
[392896.410947] dump_stack (lib/dump_stack.c:52)
[392896.411460] warn_slowpath_common (kernel/panic.c:448)
[392896.412037] ? do_set_cpus_allowed (kernel/sched/sched.h:1050 (discriminator 9) kernel/sched/core.c:1180 (discriminator 9))
[392896.412629] warn_slowpath_null (kernel/panic.c:482)
[392896.413183] do_set_cpus_allowed (kernel/sched/sched.h:1050 (discriminator 9) kernel/sched/core.c:1180 (discriminator 9))
[392896.413778] cpuset_cpus_allowed_fallback (include/linux/rcupdate.h:911 kernel/cpuset.c:2381)
[392896.414415] ? cpuset_cpus_allowed_fallback (kernel/cpuset.c:2378)
[392896.415055] select_fallback_rq (kernel/sched/core.c:1589)
[392896.415636] ? put_prev_entity (kernel/sched/fair.c:3218)
[392896.416234] migration_call (kernel/sched/core.c:5230 kernel/sched/core.c:5492)
[392896.416786] notifier_call_chain (kernel/notifier.c:93)
[392896.417359] ? cpu_notify (kernel/cpu.c:334)
[392896.417863] __raw_notifier_call_chain (kernel/notifier.c:395)
[392896.419027] cpu_notify (include/linux/notifier.h:179 kernel/cpu.c:231 kernel/cpu.c:236)
[392896.419616] take_cpu_down (kernel/cpu.c:345)
[392896.420127] multi_cpu_stop (kernel/stop_machine.c:203)
[392896.420671] ? queue_stop_cpus_work (kernel/stop_machine.c:172)
[392896.421275] cpu_stopper_thread (kernel/stop_machine.c:440)
[392896.421849] ? cpu_stop_create (kernel/stop_machine.c:415)
[392896.422404] ? lock_release (kernel/locking/lockdep.c:3643)
[392896.422936] ? __raw_callee_save___pv_queued_spin_unlock (??:?)
[392896.423710] ? _raw_spin_unlock_irqrestore (./arch/x86/include/asm/paravirt.h:802 include/linux/spinlock_api_smp.h:162 kernel/locking/spinlock.c:191)
[392896.424354] ? trace_hardirqs_on_caller (kernel/locking/lockdep.c:2594 kernel/locking/lockdep.c:2636)
[392896.424978] ? _raw_spin_unlock_irqrestore (./arch/x86/include/asm/preempt.h:77 include/linux/spinlock_api_smp.h:163 kernel/locking/spinlock.c:191)
[392896.425610] ? cpu_stop_create (kernel/stop_machine.c:415)
[392896.426152] smpboot_thread_fn (kernel/smpboot.c:163 (discriminator 1))
[392896.426707] ? sort_range (kernel/smpboot.c:106)
[392896.427252] ? __kthread_parkme (kernel/kthread.c:165)
[392896.427817] ? sort_range (kernel/smpboot.c:106)
[392896.428332] kthread (kernel/kthread.c:209)
[392896.428862] ? kthread_stop (kernel/kthread.c:178)
[392896.429399] ? lock_release (kernel/locking/lockdep.c:3643)
[392896.429977] ? wait_for_completion (kernel/sched/completion.c:77 kernel/sched/completion.c:93 kernel/sched/completion.c:101 kernel/sched/completion.c:122)
Thanks,
Sasha