When sched_core_enabled(), we sometimes need to call update_rq_clock()
to update the rq clock of sibling CPUs on the same core, before that we
need to clear RQCF_UPDATED of rq->clock_update_flags to avoid the
WARN_DOUBLE_CLOCK warning. Because at this time the rq->clock_update_flags
of sibling CPUs may be RQCF_UPDATED. If sched_core_enabled(), we will get
a core-wide rq->lock, so at this point we can safely clear RQCF_UPDATED of
rq->clock_update_flags of all CPUs on this core to avoid the
WARN_DOUBLE_CLOCK warning.
We sometimes use rq_pin_lock() and raw_spin_rq_lock() separately,
For example newidle_balance() and _double_lock_balance(). We will
temporarily give up core-wide rq->lock, and then use raw_spin_rq_lock()
to reacquire core-wide rq->lock without rq_pin_lock(), so We can not
clear RQCF_UPDATED of rq->clock_update_flags of other cpus on the
same core in rq_pin_lock().
Steps to reproduce:
1. Enable CONFIG_SCHED_DEBUG and CONFIG_SCHED_CORE when compiling
the kernel
2. echo 1 > /sys/kernel/debug/clear_warn_once
echo "WARN_DOUBLE_CLOCK" > /sys/kernel/debug/sched/features
3. Run the linux/tools/testing/selftests/sched/cs_prctl_test test
Signed-off-by: Hao Jia <[email protected]>
Reviewed-by: Phil Auld <[email protected]>
---
v3->v4:
- Replace "core wide" with "core-wide" everywhere.
- Add "Reviewed-by: Phil Auld <[email protected]>".
[v3] https://lore.kernel.org/all/[email protected]
v2->v3:
- Modify the function name to sched_core_clear_rqcf_updated,
and add function comments.
- Modify commit information.
[v2] https://lore.kernel.org/all/[email protected]
v1->v2:
- Adapt WARN_DOUBLE_CLOCK machinery for core-sched instead of clearing
WARN_DOUBLE_CLOCK warning one by one.
- Modify commit information
[v1] https://lore.kernel.org/all/[email protected]
kernel/sched/core.c | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 0d18c3969f90..c6e2c79152ef 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -429,11 +429,32 @@ void sched_core_put(void)
schedule_work(&_work);
}
+/*
+ * Now, we have obtained a core-wide rq->lock, then we need to clear
+ * RQCF_UPDATED of rq->clock_update_flags of the sibiling CPU
+ * on this core to avoid the WARN_DOUBLE_CLOCK warning.
+ */
+static inline void sched_core_clear_rqcf_updated(struct rq *rq)
+{
+#ifdef CONFIG_SCHED_DEBUG
+ const struct cpumask *smt_mask;
+ int i;
+
+ if (rq->core_enabled) {
+ smt_mask = cpu_smt_mask(rq->cpu);
+ for_each_cpu(i, smt_mask) {
+ if (rq->cpu != i)
+ cpu_rq(i)->clock_update_flags &= (RQCF_REQ_SKIP|RQCF_ACT_SKIP);
+ }
+ }
+#endif
+}
#else /* !CONFIG_SCHED_CORE */
static inline void sched_core_enqueue(struct rq *rq, struct task_struct *p) { }
static inline void
sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) { }
+static inline void sched_core_clear_rqcf_updated(struct rq *rq) { }
#endif /* CONFIG_SCHED_CORE */
@@ -548,6 +569,7 @@ void raw_spin_rq_lock_nested(struct rq *rq, int subclass)
if (likely(lock == __rq_lockp(rq))) {
/* preempt_count *MUST* be > 1 */
preempt_enable_no_resched();
+ sched_core_clear_rqcf_updated(rq);
return;
}
raw_spin_unlock(lock);
--
2.37.0
On Thu, Apr 06, 2023 at 02:44:15PM +0800, Hao Jia wrote:
> diff --git a/kernel/sched/core.c b/kernel/sched/core.c
> index 0d18c3969f90..c6e2c79152ef 100644
> --- a/kernel/sched/core.c
> +++ b/kernel/sched/core.c
> @@ -429,11 +429,32 @@ void sched_core_put(void)
> schedule_work(&_work);
> }
>
> +/*
> + * Now, we have obtained a core-wide rq->lock, then we need to clear
> + * RQCF_UPDATED of rq->clock_update_flags of the sibiling CPU
> + * on this core to avoid the WARN_DOUBLE_CLOCK warning.
> + */
> +static inline void sched_core_clear_rqcf_updated(struct rq *rq)
> +{
> +#ifdef CONFIG_SCHED_DEBUG
> + const struct cpumask *smt_mask;
> + int i;
> +
> + if (rq->core_enabled) {
> + smt_mask = cpu_smt_mask(rq->cpu);
> + for_each_cpu(i, smt_mask) {
> + if (rq->cpu != i)
> + cpu_rq(i)->clock_update_flags &= (RQCF_REQ_SKIP|RQCF_ACT_SKIP);
> + }
> + }
> +#endif
> +}
> #else /* !CONFIG_SCHED_CORE */
>
> static inline void sched_core_enqueue(struct rq *rq, struct task_struct *p) { }
> static inline void
> sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) { }
> +static inline void sched_core_clear_rqcf_updated(struct rq *rq) { }
>
> #endif /* CONFIG_SCHED_CORE */
>
> @@ -548,6 +569,7 @@ void raw_spin_rq_lock_nested(struct rq *rq, int subclass)
> if (likely(lock == __rq_lockp(rq))) {
> /* preempt_count *MUST* be > 1 */
> preempt_enable_no_resched();
> + sched_core_clear_rqcf_updated(rq);
> return;
> }
> raw_spin_unlock(lock);
This still looks absolutely wrong. The whole RQCF thing is a pin action.
On 2023/5/4 Peter Zijlstra wrote:
> On Thu, Apr 06, 2023 at 02:44:15PM +0800, Hao Jia wrote:
>> diff --git a/kernel/sched/core.c b/kernel/sched/core.c
>> index 0d18c3969f90..c6e2c79152ef 100644
>> --- a/kernel/sched/core.c
>> +++ b/kernel/sched/core.c
>> @@ -429,11 +429,32 @@ void sched_core_put(void)
>> schedule_work(&_work);
>> }
>>
>> +/*
>> + * Now, we have obtained a core-wide rq->lock, then we need to clear
>> + * RQCF_UPDATED of rq->clock_update_flags of the sibiling CPU
>> + * on this core to avoid the WARN_DOUBLE_CLOCK warning.
>> + */
>> +static inline void sched_core_clear_rqcf_updated(struct rq *rq)
>> +{
>> +#ifdef CONFIG_SCHED_DEBUG
>> + const struct cpumask *smt_mask;
>> + int i;
>> +
>> + if (rq->core_enabled) {
>> + smt_mask = cpu_smt_mask(rq->cpu);
>> + for_each_cpu(i, smt_mask) {
>> + if (rq->cpu != i)
>> + cpu_rq(i)->clock_update_flags &= (RQCF_REQ_SKIP|RQCF_ACT_SKIP);
>> + }
>> + }
>> +#endif
>> +}
>> #else /* !CONFIG_SCHED_CORE */
>>
>> static inline void sched_core_enqueue(struct rq *rq, struct task_struct *p) { }
>> static inline void
>> sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) { }
>> +static inline void sched_core_clear_rqcf_updated(struct rq *rq) { }
>>
>> #endif /* CONFIG_SCHED_CORE */
>>
>> @@ -548,6 +569,7 @@ void raw_spin_rq_lock_nested(struct rq *rq, int subclass)
>> if (likely(lock == __rq_lockp(rq))) {
>> /* preempt_count *MUST* be > 1 */
>> preempt_enable_no_resched();
>> + sched_core_clear_rqcf_updated(rq);
>> return;
>> }
>> raw_spin_unlock(lock);
>
> This still looks absolutely wrong. The whole RQCF thing is a pin action.
Do you think it is better for us to extend rq_pin_lock() to clean RQCF
updated than to do it in raw_spin_rq_lock_nested()?
Before doing this, we need to solve the situation where rq_pin_lock()
and raw_spin_rq_lock() are used separately.
Any suggestion will be very helpful for me.
Thanks,
Hao