When force idle, rq->curr == rq->idle but rq->nr_running > 0, results
that idle_cpu() returns 0. In function tick_irq_exit(), if idle_cpu()
is 0, tick_nohz_irq_exit() will not be called, and ts->idle_active will
not become 1, which became 0 in tick_nohz_irq_enter().
ts->idle_sleeptime won't update in function update_ts_time_stats(), if
ts->idle_active is 0, which should be 1. And this bug will result that
ts->idle_sleeptime is less than the actual value, and finally will
result that the idle time in /proc/stat is less than the actual value.
To fix this problem, we introduce sched_core_idle_cpu(), which returns
true when force idle, and we change the judgment criteria from
idle_cpu() to sched_core_idle_cpu() in function tick_irq_exit().
Signed-off-by: Cruz Zhao <[email protected]>
---
include/linux/sched.h | 2 ++
kernel/sched/core.c | 13 +++++++++++++
kernel/softirq.c | 2 +-
3 files changed, 16 insertions(+), 1 deletion(-)
diff --git a/include/linux/sched.h b/include/linux/sched.h
index b09a83bfad8b..73e61c0f10a7 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -2430,9 +2430,11 @@ extern void sched_core_free(struct task_struct *tsk);
extern void sched_core_fork(struct task_struct *p);
extern int sched_core_share_pid(unsigned int cmd, pid_t pid, enum pid_type type,
unsigned long uaddr);
+extern int sched_core_idle_cpu(int cpu);
#else
static inline void sched_core_free(struct task_struct *tsk) { }
static inline void sched_core_fork(struct task_struct *p) { }
+static inline int sched_core_idle_cpu(int cpu) { return idle_cpu(cpu); }
#endif
extern void sched_set_stop_task(int cpu, struct task_struct *stop);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 71c1a0f232b4..c80088956987 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -7286,6 +7286,19 @@ struct task_struct *idle_task(int cpu)
return cpu_rq(cpu)->idle;
}
+#ifdef CONFIG_SCHED_CORE
+int sched_core_idle_cpu(int cpu)
+{
+ struct rq *rq = cpu_rq(cpu);
+
+ if (sched_core_enabled(rq) && rq->curr == rq->idle)
+ return 1;
+
+ return idle_cpu(cpu);
+}
+
+#endif
+
#ifdef CONFIG_SMP
/*
* This function computes an effective utilization for the given CPU, to be
diff --git a/kernel/softirq.c b/kernel/softirq.c
index c8a6913c067d..98b98991ce45 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -630,7 +630,7 @@ static inline void tick_irq_exit(void)
int cpu = smp_processor_id();
/* Make sure that timer wheel updates are propagated */
- if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
+ if ((sched_core_idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
if (!in_hardirq())
tick_nohz_irq_exit();
}
--
2.27.0
On Mon, Jun 19, 2023 at 01:02:40AM +0800, Cruz Zhao wrote:
> When force idle, rq->curr == rq->idle but rq->nr_running > 0, results
> that idle_cpu() returns 0. In function tick_irq_exit(), if idle_cpu()
> is 0, tick_nohz_irq_exit() will not be called, and ts->idle_active will
> not become 1, which became 0 in tick_nohz_irq_enter().
> ts->idle_sleeptime won't update in function update_ts_time_stats(), if
> ts->idle_active is 0, which should be 1. And this bug will result that
> ts->idle_sleeptime is less than the actual value, and finally will
> result that the idle time in /proc/stat is less than the actual value.
>
> To fix this problem, we introduce sched_core_idle_cpu(), which returns
> true when force idle, and we change the judgment criteria from
> idle_cpu() to sched_core_idle_cpu() in function tick_irq_exit().
>
> Signed-off-by: Cruz Zhao <[email protected]>
> ---
> include/linux/sched.h | 2 ++
> kernel/sched/core.c | 13 +++++++++++++
> kernel/softirq.c | 2 +-
> 3 files changed, 16 insertions(+), 1 deletion(-)
>
> diff --git a/include/linux/sched.h b/include/linux/sched.h
> index b09a83bfad8b..73e61c0f10a7 100644
> --- a/include/linux/sched.h
> +++ b/include/linux/sched.h
> @@ -2430,9 +2430,11 @@ extern void sched_core_free(struct task_struct *tsk);
> extern void sched_core_fork(struct task_struct *p);
> extern int sched_core_share_pid(unsigned int cmd, pid_t pid, enum pid_type type,
> unsigned long uaddr);
> +extern int sched_core_idle_cpu(int cpu);
> #else
> static inline void sched_core_free(struct task_struct *tsk) { }
> static inline void sched_core_fork(struct task_struct *p) { }
> +static inline int sched_core_idle_cpu(int cpu) { return idle_cpu(cpu); }
> #endif
>
> extern void sched_set_stop_task(int cpu, struct task_struct *stop);
> diff --git a/kernel/sched/core.c b/kernel/sched/core.c
> index 71c1a0f232b4..c80088956987 100644
> --- a/kernel/sched/core.c
> +++ b/kernel/sched/core.c
> @@ -7286,6 +7286,19 @@ struct task_struct *idle_task(int cpu)
> return cpu_rq(cpu)->idle;
> }
>
> +#ifdef CONFIG_SCHED_CORE
> +int sched_core_idle_cpu(int cpu)
> +{
> + struct rq *rq = cpu_rq(cpu);
> +
> + if (sched_core_enabled(rq) && rq->curr == rq->idle)
> + return 1;
> +
> + return idle_cpu(cpu);
> +}
> +
> +#endif
> +
> #ifdef CONFIG_SMP
> /*
> * This function computes an effective utilization for the given CPU, to be
> diff --git a/kernel/softirq.c b/kernel/softirq.c
> index c8a6913c067d..98b98991ce45 100644
> --- a/kernel/softirq.c
> +++ b/kernel/softirq.c
> @@ -630,7 +630,7 @@ static inline void tick_irq_exit(void)
> int cpu = smp_processor_id();
>
> /* Make sure that timer wheel updates are propagated */
> - if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
> + if ((sched_core_idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
> if (!in_hardirq())
> tick_nohz_irq_exit();
> }
Urgh...
did you audit all idle_cpu() users?
Or so we want something like so instead?
---
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index a68d1276bab0..b0a1cd184b1b 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -7303,6 +7303,11 @@ int idle_cpu(int cpu)
if (rq->curr != rq->idle)
return 0;
+#ifdef CONFIG_SCHED_CORE
+ if (sched_core_enabled() && rq->core->core_forceidle_count)
+ return 1;
+#endif
+
if (rq->nr_running)
return 0;