Received-SPF: pass (google.com: best guess record for domain of linux-kernel-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) client-ip=209.132.180.67;
From:   =?UTF-8?q?Jan=20H=2E=20Sch=C3=B6nherr?= <jschoenh@amazon.de>
To:     Ingo Molnar <mingo@redhat.com>,
        Peter Zijlstra <peterz@infradead.org>
Cc:     =?UTF-8?q?Jan=20H=2E=20Sch=C3=B6nherr?= <jschoenh@amazon.de>,
        linux-kernel@vger.kernel.org
Subject: [RFC 26/60] cosched: Construct runqueue hierarchy
Date:   Fri,  7 Sep 2018 23:40:13 +0200
Message-Id: <20180907214047.26914-27-jschoenh@amazon.de>
In-Reply-To: <20180907214047.26914-1-jschoenh@amazon.de>
References: <20180907214047.26914-1-jschoenh@amazon.de>
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Sender: linux-kernel-owner@vger.kernel.org
Precedence: bulk

With scheduling domains sufficiently prepared, we can now initialize
the full hierarchy of runqueues and link it with the already existing
bottom level, which we set up earlier.

Signed-off-by: Jan H. Schönherr <jschoenh@amazon.de>
---
 kernel/sched/core.c    |  1 +
 kernel/sched/cosched.c | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++
 kernel/sched/sched.h   |  2 ++
 3 files changed, 79 insertions(+)

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index cc801f84bf97..5350cab7ac4a 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5876,6 +5876,7 @@ void __init sched_init_smp(void)
 	 */
 	mutex_lock(&sched_domains_mutex);
 	sched_init_domains(cpu_active_mask);
+	cosched_init_hierarchy();
 	mutex_unlock(&sched_domains_mutex);
 
 	/* Move init over to a non-isolated CPU */
diff --git a/kernel/sched/cosched.c b/kernel/sched/cosched.c
index 7a793aa93114..48394050ec34 100644
--- a/kernel/sched/cosched.c
+++ b/kernel/sched/cosched.c
@@ -351,3 +351,79 @@ void cosched_init_topology(void)
 	/* Make permanent */
 	set_sched_topology(tl);
 }
+
+/*
+ * Build the SD-RQ hierarchy according to the scheduling domains.
+ *
+ * Note, that the scheduler is already live at this point, but the scheduling
+ * domains only just have become available. That means, we only setup everything
+ * above the bottom level of the SD-RQ hierarchy and link it with the already
+ * active bottom level.
+ *
+ * We can do this without any locks, as nothing will automatically traverse into
+ * these data structures. This requires an update of the sdrq.is_root property,
+ * which will happen only later.
+ */
+void cosched_init_hierarchy(void)
+{
+	struct sched_domain *sd;
+	struct sdrq *sdrq;
+	int cpu, level = 1;
+
+	/* Only one CPU in the system, we are finished here */
+	if (cpumask_weight(cpu_possible_mask) == 1)
+		return;
+
+	/* Determine and initialize top */
+	for_each_domain(0, sd) {
+		if (!sd->parent)
+			break;
+		level++;
+	}
+
+	init_sdrq_data(&sd->shared->rq.sdrq_data, NULL, sched_domain_span(sd),
+		       level);
+	init_cfs_rq(&sd->shared->rq.cfs);
+	init_tg_cfs_entry(&root_task_group, &sd->shared->rq.cfs, NULL,
+			  &sd->shared->rq, NULL);
+	init_sdrq(&root_task_group, &sd->shared->rq.cfs.sdrq, NULL, NULL,
+		  &sd->shared->rq.sdrq_data);
+
+	root_task_group.top_cfsrq = &sd->shared->rq.cfs;
+
+	/* Initialize others top-down, per CPU */
+	for_each_possible_cpu(cpu) {
+		/* Find highest not-yet initialized position for this CPU */
+		for_each_domain(cpu, sd) {
+			if (sd->shared->rq.sdrq_data.span_weight)
+				break;
+		}
+		if (WARN(!sd, "SD hierarchy seems to have multiple roots"))
+			continue;
+		sd = sd->child;
+
+		/* Initialize from there downwards */
+		for_each_lower_domain(sd) {
+			init_sdrq_data(&sd->shared->rq.sdrq_data,
+				       &sd->parent->shared->rq.sdrq_data,
+				       sched_domain_span(sd), -1);
+			init_cfs_rq(&sd->shared->rq.cfs);
+			init_tg_cfs_entry(&root_task_group, &sd->shared->rq.cfs,
+					  NULL, &sd->shared->rq, NULL);
+			init_sdrq(&root_task_group, &sd->shared->rq.cfs.sdrq,
+				  &sd->parent->shared->rq.cfs.sdrq, NULL,
+				  &sd->shared->rq.sdrq_data);
+		}
+
+		/* Link up with local data structures */
+		sdrq = &cpu_rq(cpu)->cfs.sdrq;
+		sd = cpu_rq(cpu)->sd;
+
+		/* sdrq_data */
+		sdrq->data->parent = &sd->shared->rq.sdrq_data;
+
+		/* sdrq */
+		sdrq->sd_parent = &sd->shared->rq.cfs.sdrq;
+		list_add_tail(&sdrq->siblings, &sdrq->sd_parent->children);
+	}
+}
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index ed9c526b74ee..d65c98c34c13 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1132,9 +1132,11 @@ static inline struct cfs_rq *taskgroup_next_cfsrq(struct task_group *tg,
 #ifdef CONFIG_COSCHEDULING
 void cosched_init_bottom(void);
 void cosched_init_topology(void);
+void cosched_init_hierarchy(void);
 #else /* !CONFIG_COSCHEDULING */
 static inline void cosched_init_bottom(void) { }
 static inline void cosched_init_topology(void) { }
+static inline void cosched_init_hierarchy(void) { }
 #endif /* !CONFIG_COSCHEDULING */
 
 #ifdef CONFIG_SCHED_SMT
-- 
2.9.3.1.gcba166c.dirty