Signed-off-by: Denys Vlasenko <[email protected]>
Cc: Frederic Weisbecker <[email protected]>
Cc: Hidetoshi Seto <[email protected]>
Cc: Fernando Luis Vazquez Cao <[email protected]>
Cc: Tetsuo Handa <[email protected]>
Cc: Thomas Gleixner <[email protected]>
Cc: Ingo Molnar <[email protected]>
Cc: Peter Zijlstra <[email protected]>
Cc: Andrew Morton <[email protected]>
Cc: Arjan van de Ven <[email protected]>
Cc: Oleg Nesterov <[email protected]>
---
kernel/sched/core.c | 33 +++++++++++++++++----------------
1 file changed, 17 insertions(+), 16 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index ffea757..3137980 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -4208,6 +4208,21 @@ EXPORT_SYMBOL_GPL(yield_to);
* This task is about to go to sleep on IO. Increment rq->nr_iowait so
* that process accounting knows that this is a task in IO wait state.
*/
+#ifdef CONFIG_NO_HZ_COMMON
+static __sched void io_wait_end(struct rq *rq)
+{
+ if (atomic_dec_and_test(&rq->nr_iowait)) {
+ if (raw_smp_processor_id() != cpu_of(rq))
+ tick_nohz_iowait_to_idle(cpu_of(rq));
+ }
+}
+#else
+static inline void io_wait_end(struct rq *rq)
+{
+ atomic_dec(&rq->nr_iowait);
+}
+#endif
+
void __sched io_schedule(void)
{
struct rq *rq = raw_rq();
@@ -4218,14 +4233,7 @@ void __sched io_schedule(void)
current->in_iowait = 1;
schedule();
current->in_iowait = 0;
-#ifdef CONFIG_NO_HZ_COMMON
- if (atomic_dec_and_test(&rq->nr_iowait)) {
- if (raw_smp_processor_id() != cpu_of(rq))
- tick_nohz_iowait_to_idle(cpu_of(rq));
- }
-#else
- atomic_dec(&rq->nr_iowait);
-#endif
+ io_wait_end(rq);
delayacct_blkio_end();
}
EXPORT_SYMBOL(io_schedule);
@@ -4241,14 +4249,7 @@ long __sched io_schedule_timeout(long timeout)
current->in_iowait = 1;
ret = schedule_timeout(timeout);
current->in_iowait = 0;
-#ifdef CONFIG_NO_HZ_COMMON
- if (atomic_dec_and_test(&rq->nr_iowait)) {
- if (raw_smp_processor_id() != cpu_of(rq))
- tick_nohz_iowait_to_idle(cpu_of(rq));
- }
-#else
- atomic_dec(&rq->nr_iowait);
-#endif
+ io_wait_end(rq);
delayacct_blkio_end();
return ret;
}
--
1.8.1.4
In get_cpu_{idle,iowait}_time_us, we were fetching
per_cpu(tick_cpu_sched, cpu) upfront. But them we do
if (!tick_nohz_active)
return -1;
which may make that fetch pointless.
Moving it further down, closer to the usage point.
Signed-off-by: Denys Vlasenko <[email protected]>
Cc: Frederic Weisbecker <[email protected]>
Cc: Hidetoshi Seto <[email protected]>
Cc: Fernando Luis Vazquez Cao <[email protected]>
Cc: Tetsuo Handa <[email protected]>
Cc: Thomas Gleixner <[email protected]>
Cc: Ingo Molnar <[email protected]>
Cc: Peter Zijlstra <[email protected]>
Cc: Andrew Morton <[email protected]>
Cc: Arjan van de Ven <[email protected]>
Cc: Oleg Nesterov <[email protected]>
---
kernel/time/tick-sched.c | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index d78c942..4ffccd0 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -476,7 +476,7 @@ void tick_nohz_iowait_to_idle(int cpu)
*/
u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
{
- struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
+ struct tick_sched *ts;
ktime_t now, idle;
unsigned int seq;
@@ -487,6 +487,8 @@ u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
if (last_update_time)
*last_update_time = ktime_to_us(now);
+ ts = &per_cpu(tick_cpu_sched, cpu);
+
do {
ktime_t start, delta, iowait_exit;
@@ -531,7 +533,7 @@ EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
*/
u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
{
- struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
+ struct tick_sched *ts;
ktime_t now, iowait;
unsigned int seq;
@@ -542,6 +544,8 @@ u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
if (last_update_time)
*last_update_time = ktime_to_us(now);
+ ts = &per_cpu(tick_cpu_sched, cpu);
+
do {
ktime_t delta, end;
--
1.8.1.4
On Fri, Apr 25, 2014 at 08:57:29PM +0200, Denys Vlasenko wrote:
> Signed-off-by: Denys Vlasenko <[email protected]>
> Cc: Frederic Weisbecker <[email protected]>
> Cc: Hidetoshi Seto <[email protected]>
> Cc: Fernando Luis Vazquez Cao <[email protected]>
> Cc: Tetsuo Handa <[email protected]>
> Cc: Thomas Gleixner <[email protected]>
> Cc: Ingo Molnar <[email protected]>
> Cc: Peter Zijlstra <[email protected]>
> Cc: Andrew Morton <[email protected]>
> Cc: Arjan van de Ven <[email protected]>
> Cc: Oleg Nesterov <[email protected]>
> ---
> kernel/sched/core.c | 33 +++++++++++++++++----------------
> 1 file changed, 17 insertions(+), 16 deletions(-)
>
> diff --git a/kernel/sched/core.c b/kernel/sched/core.c
> index ffea757..3137980 100644
> --- a/kernel/sched/core.c
> +++ b/kernel/sched/core.c
> @@ -4208,6 +4208,21 @@ EXPORT_SYMBOL_GPL(yield_to);
> * This task is about to go to sleep on IO. Increment rq->nr_iowait so
> * that process accounting knows that this is a task in IO wait state.
> */
> +#ifdef CONFIG_NO_HZ_COMMON
> +static __sched void io_wait_end(struct rq *rq)
> +{
> + if (atomic_dec_and_test(&rq->nr_iowait)) {
> + if (raw_smp_processor_id() != cpu_of(rq))
> + tick_nohz_iowait_to_idle(cpu_of(rq));
> + }
> +}
> +#else
> +static inline void io_wait_end(struct rq *rq)
> +{
> + atomic_dec(&rq->nr_iowait);
> +}
> +#endif
> +
> void __sched io_schedule(void)
> {
> struct rq *rq = raw_rq();
> @@ -4218,14 +4233,7 @@ void __sched io_schedule(void)
> current->in_iowait = 1;
> schedule();
> current->in_iowait = 0;
> -#ifdef CONFIG_NO_HZ_COMMON
> - if (atomic_dec_and_test(&rq->nr_iowait)) {
> - if (raw_smp_processor_id() != cpu_of(rq))
> - tick_nohz_iowait_to_idle(cpu_of(rq));
> - }
> -#else
> - atomic_dec(&rq->nr_iowait);
> -#endif
> + io_wait_end(rq);
> delayacct_blkio_end();
There is much more to unify that the iowait accounting between all
the io_schedule() declensions.
Peterz I think you had a patch to unify that a few month ago?