Received-SPF: pass (google.com: best guess record for domain of linux-kernel-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) client-ip=209.132.180.67;
Date:   Tue, 12 Nov 2019 11:51:30 +0100
From:   Peter Zijlstra <peterz@infradead.org>
To:     Juri Lelli <juri.lelli@redhat.com>
Cc:     mingo@redhat.com, glenn@aurora.tech, linux-kernel@vger.kernel.org,
        rostedt@goodmis.org, vincent.guittot@linaro.org,
        dietmar.eggemann@arm.com, tglx@linutronix.de,
        luca.abeni@santannapisa.it, c.scordino@evidence.eu.com,
        tommaso.cucinotta@santannapisa.it, bristot@redhat.com
Subject: Re: [PATCH 2/2] sched/deadline: Temporary copy static parameters to
 boosted non-DEADLINE entities
Message-ID: <20191112105130.GZ4131@hirez.programming.kicks-ass.net>
References: <20191112075056.19971-1-juri.lelli@redhat.com>
 <20191112075056.19971-3-juri.lelli@redhat.com>
MIME-Version: 1.0
Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline
In-Reply-To: <20191112075056.19971-3-juri.lelli@redhat.com>
User-Agent: Mutt/1.10.1 (2018-07-13)
Sender: linux-kernel-owner@vger.kernel.org
Precedence: bulk

On Tue, Nov 12, 2019 at 08:50:56AM +0100, Juri Lelli wrote:
> Boosted entities (Priority Inheritance) use static DEADLINE parameters
> of the top priority waiter. However, there might be cases where top
> waiter could be a non-DEADLINE entity that is currently boosted by a
> DEADLINE entity from a different lock chain (i.e., nested priority
> chains involving entities of non-DEADLINE classes). In this case, top
> waiter static DEADLINE parameters could null (initialized to 0 at
> fork()) and replenish_dl_entity() would hit a BUG().

Argh!

> Fix this by temporarily copying static DEADLINE parameters of top
> DEADLINE waiter (there must be at least one in the chain(s) for the
> problem above to happen) into boosted entities. Parameters are reset
> during deboost.

Also, yuck!

> --- a/kernel/sched/core.c
> +++ b/kernel/sched/core.c
> @@ -4441,19 +4441,21 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task)
>  		if (!dl_prio(p->normal_prio) ||
>  		    (pi_task && dl_entity_preempt(&pi_task->dl, &p->dl))) {
>  			p->dl.dl_boosted = 1;
> +			if (!dl_prio(p->normal_prio))
> +				__dl_copy_static(p, pi_task);
>  			queue_flag |= ENQUEUE_REPLENISH;
>  		} else
>  			p->dl.dl_boosted = 0;
>  		p->sched_class = &dl_sched_class;

So I thought our basic approach was deadline inheritance and screw
runtime accounting.

Given that, I don't quite understand the REPLENISH hack there. Should we
not simply copy dl->deadline around (and restore on unboost)?

That is, should we not do something 'simple' like this:


diff --git a/include/linux/sched.h b/include/linux/sched.h
index 84b26d38c929..1579c571cb83 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -522,6 +522,7 @@ struct sched_dl_entity {
 	 */
 	s64				runtime;	/* Remaining runtime for this instance	*/
 	u64				deadline;	/* Absolute deadline for this instance	*/
+	u64				normal_deadline;
 	unsigned int			flags;		/* Specifying the scheduler behaviour	*/
 
 	/*
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 26e4ffa01e7a..16164b0ba80b 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -4452,9 +4452,11 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task)
 		if (!dl_prio(p->normal_prio) ||
 		    (pi_task && dl_entity_preempt(&pi_task->dl, &p->dl))) {
 			p->dl.dl_boosted = 1;
-			queue_flag |= ENQUEUE_REPLENISH;
-		} else
+			p->dl.deadline = pi_task->dl.deadline;
+		} else {
 			p->dl.dl_boosted = 0;
+			p->dl.deadline = p->dl.normal_deadline;
+		}
 		p->sched_class = &dl_sched_class;
 	} else if (rt_prio(prio)) {
 		if (dl_prio(oldprio))
diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index 43323f875cb9..0ad7c2797f11 100644
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -674,6 +674,7 @@ static inline void setup_new_dl_entity(struct sched_dl_entity *dl_se)
 	 * spent on hardirq context, etc.).
 	 */
 	dl_se->deadline = rq_clock(rq) + dl_se->dl_deadline;
+	dl_se->normal_deadline = dl_se->deadline;
 	dl_se->runtime = dl_se->dl_runtime;
 }
 
@@ -709,6 +710,7 @@ static void replenish_dl_entity(struct sched_dl_entity *dl_se,
 	 */
 	if (dl_se->dl_deadline == 0) {
 		dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline;
+		dl_se->normal_deadline = dl_se->deadline;
 		dl_se->runtime = pi_se->dl_runtime;
 	}
 
@@ -723,6 +725,7 @@ static void replenish_dl_entity(struct sched_dl_entity *dl_se,
 	 */
 	while (dl_se->runtime <= 0) {
 		dl_se->deadline += pi_se->dl_period;
+		dl_se->normal_deadline = dl_se->normal;
 		dl_se->runtime += pi_se->dl_runtime;
 	}
 
@@ -738,6 +741,7 @@ static void replenish_dl_entity(struct sched_dl_entity *dl_se,
 	if (dl_time_before(dl_se->deadline, rq_clock(rq))) {
 		printk_deferred_once("sched: DL replenish lagged too much\n");
 		dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline;
+		dl_se->normal_deadline = dl_se->deadline;
 		dl_se->runtime = pi_se->dl_runtime;
 	}
 
@@ -898,6 +902,7 @@ static void update_dl_entity(struct sched_dl_entity *dl_se,
 		}
 
 		dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline;
+		dl_se->normal_deadline = dl_se->deadline;
 		dl_se->runtime = pi_se->dl_runtime;
 	}
 }