Received-SPF: pass (google.com: domain of linux-kernel+bounces-30740-linux.lists.archive=gmail.com@vger.kernel.org designates 147.75.80.249 as permitted sender) client-ip=147.75.80.249;
From: Huang Shijie <shijie@os.amperecomputing.com>
To: gregkh@linuxfoundation.org
Cc: patches@amperecomputing.com,
	rafael@kernel.org,
	paul.walmsley@sifive.com,
	palmer@dabbelt.com,
	aou@eecs.berkeley.edu,
	yury.norov@gmail.com,
	kuba@kernel.org,
	vschneid@redhat.com,
	mingo@kernel.org,
	akpm@linux-foundation.org,
	vbabka@suse.cz,
	rppt@kernel.org,
	tglx@linutronix.de,
	jpoimboe@kernel.org,
	ndesaulniers@google.com,
	mikelley@microsoft.com,
	mhiramat@kernel.org,
	arnd@arndb.de,
	linux-kernel@vger.kernel.org,
	linux-riscv@lists.infradead.org,
	linux-arm-kernel@lists.infradead.org,
	catalin.marinas@arm.com,
	will@kernel.org,
	mark.rutland@arm.com,
	mpe@ellerman.id.au,
	linuxppc-dev@lists.ozlabs.org,
	chenhuacai@kernel.org,
	jiaxun.yang@flygoat.com,
	linux-mips@vger.kernel.org,
	cl@os.amperecomputing.com,
	Huang Shijie <shijie@os.amperecomputing.com>
Subject: [PATCH] NUMA: Early use of cpu_to_node() returns 0 instead of the correct node id
Date: Fri, 19 Jan 2024 11:32:27 +0800
Message-Id: <20240119033227.14113-1-shijie@os.amperecomputing.com>
Content-Transfer-Encoding: 8bit
Content-Type: text/plain
Precedence: bulk
MIME-Version: 1.0
X-MS-Exchange-AntiSpam-MessageData-ChunkCount: 1
X-MS-Exchange-AntiSpam-MessageData-0:
	=?us-ascii?Q?af8Q+WQwi3uK7cpNq0kHacZPa1xnibsp/Chy/WSQx5XWEk9BtTaGQeSVh0s3?=
 =?us-ascii?Q?FFsiwqi4ubHaxnTAKn+VXe3dnbWC/sFZEE/mWEp0KAWTNRbtwUdJroMvgdYP?=
 =?us-ascii?Q?nYChSKzURwli1SqtAPxiQGKDFY82k0fyrLwEX/em2M7MVrikOmcnrfFxiHGB?=
 =?us-ascii?Q?yAKWJCTClkor6RrKIykv/xX7uH6U2T1I7oesGg8VjFFqanR8rebd6E074cCI?=
 =?us-ascii?Q?9SrJQJprW7bzCa1RGz62dFkZ5e9WmP0SrJL3yxu1vxLSzZC5AcA1q9EXanYx?=
 =?us-ascii?Q?VJNV+NlohBzcWUTxf3vNZiLwRA4TFKW4nE3slbEYLv3wZiSQ3MKKoviLB7kl?=
 =?us-ascii?Q?zFIjjXk6VpdNx3daAuV7yKs/xQBdt58FHVmFTYvKPtu5ZhQdgwhuB6Zi2Kiv?=
 =?us-ascii?Q?DsnIKzcfgghFB9k2kBbqBwrvezwNFUE3Cxzr1ARknsGLYFbANslR1P9+WcOT?=
 =?us-ascii?Q?86KZxDlIV3G2HLLUipIlqSLpNQuhALccoC7HN7pA8iiURWc8md0m8nho/aoP?=
 =?us-ascii?Q?d6wcOQYpqc6OsW1imVQ3YDa3xlZoQSu+cklN+kTjGoFuhbi3vIuSZnp1wJJI?=
 =?us-ascii?Q?XBcrYzNXx66DJDXUDKVZIyrmAM3QJUyciCow+24//KDAXiARCvxH5WANPBzc?=
 =?us-ascii?Q?PTkJvEObZ+isceAgzqF1S+M+8WY69741kIA7tyX+YdpJ1glBipuG5PQHYl03?=
 =?us-ascii?Q?2dONFGRwr1QfXYEWtsDSz3IuLMdWvQExhHLWEXqxCiO49kqz0PJkVrFq8hxM?=
 =?us-ascii?Q?heNQ18dYJSftB16GgsvMBrcDpjCYUJoJ4BWdBST95T+28J41PsNIkokUWAZf?=
 =?us-ascii?Q?FnFiQNGf3sE1jTFDY6c2tRgqUIrh4qKg3AKPjzBjTlU5uXXSPuDVBt6MzUGv?=
 =?us-ascii?Q?Oaory/EyzDxjPB3PCY1Cs2RCrBtPZ5cWjmEqiMYD1xAupjlqmQ6853vAIIkX?=
 =?us-ascii?Q?EHRa7XExnAV+P00HcYv0mhS6Y9ZVQJS8W+kvdX8t4V/DXHIsmufvZXAw/KHL?=
 =?us-ascii?Q?6Zm0+DFnaQKT1ZeSPsKlFezn1aLwqgH8taWbeMBpst98Itud7w/devZ0Q4v5?=
 =?us-ascii?Q?sq18ZfT61xPFWhdKtZMkTorU9psipjxLRBSQAPyxb9yw98EHK/nCCgo6avcK?=
 =?us-ascii?Q?Oe3ifrruc3kh4wH6O3Cs3ArDnAGVbT63KO8sQCJHGx+y2jIVdeWNxELkAfqw?=
 =?us-ascii?Q?El7xkqyZJqVepN9cTT4kUylioVHm36fyiLNKncPfETTEoBIvDD1ITh//AYRM?=
 =?us-ascii?Q?aAPK/tLWSJAelovZOQB2i+JBcfhbUHzw1aDMSI9at3p+fvq3cuVH653Gk2op?=
 =?us-ascii?Q?bPOSHnR7sGr07O5N4Cej6C5KlPIIlmCnQl8XY5S+MpkIOO8mrGwWnxTkFztj?=
 =?us-ascii?Q?TAmCVz70tHCuJ0y5Qr2IHkm3FFwlAfp7mfCfTEvGrxH6aX6IuSzzIQmSrChl?=
 =?us-ascii?Q?UsrDpKBp6/H9w2Vyg/YmkN8DOAJEjGYarxAsl9QYsgb45DYEUuLAslOWZ0t9?=
 =?us-ascii?Q?mquG8Qrxh2s876u1qirCwzqrgQEsfwlDOAlBqvdMMuL9mjtAL5vHrWI/FiwG?=
 =?us-ascii?Q?EIIez2WcpkaeJfHPxzzTyabmUsATAtNMdgplWSyGYGThcUxIlUJBpvOUdUe5?=
 =?us-ascii?Q?DL2m7a9ck0QCzj7pUNCMf/I=3D?=
X-OriginatorOrg: os.amperecomputing.com
X-MS-Exchange-CrossTenant-Network-Message-Id: 7173bb2c-947c-4966-58a0-08dc189f5997
X-MS-Exchange-CrossTenant-AuthSource: PH0PR01MB7975.prod.exchangelabs.com
X-MS-Exchange-CrossTenant-AuthAs: Internal
X-MS-Exchange-CrossTenant-OriginalArrivalTime: 19 Jan 2024 03:33:06.4071
 (UTC)
X-MS-Exchange-CrossTenant-FromEntityHeader: Hosted
X-MS-Exchange-CrossTenant-Id: 3bc2b170-fd94-476d-b0ce-4229bdc904a7
X-MS-Exchange-CrossTenant-MailboxType: HOSTED
X-MS-Exchange-CrossTenant-UserPrincipalName: aakv5cxf2piirah6hPpVtdKCF3b3gxrbVnv6icYLbFAuRAdJX03JDjNmwXb8AC6f0V9pQRG84jFIDmfpKaAPpY2wHhP5wSYY2rjVw4W+hZ7bkEvc+Z19RHkS/HVG3KMg
X-MS-Exchange-Transport-CrossTenantHeadersStamped: DM8PR01MB7144

During the kernel booting, the generic cpu_to_node() is called too early in
arm64, powerpc and riscv when CONFIG_NUMA is enabled.

There are at least four places in the common code where
the generic cpu_to_node() is called before it is initialized:
	   1.) early_trace_init()         in kernel/trace/trace.c
	   2.) sched_init()               in kernel/sched/core.c
	   3.) init_sched_fair_class()    in kernel/sched/fair.c
	   4.) workqueue_init_early()     in kernel/workqueue.c

In order to fix the bug, the patch changes generic cpu_to_node to
function pointer, and export it for kernel modules.
Introduce smp_prepare_boot_cpu_start() to wrap the original
smp_prepare_boot_cpu(), and set cpu_to_node with early_cpu_to_node.
Introduce smp_prepare_cpus_done() to wrap the original smp_prepare_cpus(),
and set the cpu_to_node to formal _cpu_to_node().

Signed-off-by: Huang Shijie <shijie@os.amperecomputing.com>
---
 drivers/base/arch_numa.c | 11 +++++++++++
 include/linux/topology.h |  6 ++----
 init/main.c              | 29 +++++++++++++++++++++++++++--
 3 files changed, 40 insertions(+), 6 deletions(-)

diff --git a/drivers/base/arch_numa.c b/drivers/base/arch_numa.c
index 5b59d133b6af..867a477fa975 100644
--- a/drivers/base/arch_numa.c
+++ b/drivers/base/arch_numa.c
@@ -61,6 +61,17 @@ EXPORT_SYMBOL(cpumask_of_node);
 
 #endif
 
+#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
+#ifndef cpu_to_node
+int _cpu_to_node(int cpu)
+{
+	return per_cpu(numa_node, cpu);
+}
+int (*cpu_to_node)(int cpu);
+EXPORT_SYMBOL(cpu_to_node);
+#endif
+#endif
+
 static void numa_update_cpu(unsigned int cpu, bool remove)
 {
 	int nid = cpu_to_node(cpu);
diff --git a/include/linux/topology.h b/include/linux/topology.h
index 52f5850730b3..e7ce2bae11dd 100644
--- a/include/linux/topology.h
+++ b/include/linux/topology.h
@@ -91,10 +91,8 @@ static inline int numa_node_id(void)
 #endif
 
 #ifndef cpu_to_node
-static inline int cpu_to_node(int cpu)
-{
-	return per_cpu(numa_node, cpu);
-}
+extern int (*cpu_to_node)(int cpu);
+extern int _cpu_to_node(int cpu);
 #endif
 
 #ifndef set_numa_node
diff --git a/init/main.c b/init/main.c
index e24b0780fdff..b142e9c51161 100644
--- a/init/main.c
+++ b/init/main.c
@@ -870,6 +870,18 @@ static void __init print_unknown_bootoptions(void)
 	memblock_free(unknown_options, len);
 }
 
+static void __init smp_prepare_boot_cpu_start(void)
+{
+	smp_prepare_boot_cpu();	/* arch-specific boot-cpu hooks */
+
+#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
+#ifndef cpu_to_node
+	/* The early_cpu_to_node should be ready now. */
+	cpu_to_node = early_cpu_to_node;
+#endif
+#endif
+}
+
 asmlinkage __visible __init __no_sanitize_address __noreturn __no_stack_protector
 void start_kernel(void)
 {
@@ -899,7 +911,7 @@ void start_kernel(void)
 	setup_command_line(command_line);
 	setup_nr_cpu_ids();
 	setup_per_cpu_areas();
-	smp_prepare_boot_cpu();	/* arch-specific boot-cpu hooks */
+	smp_prepare_boot_cpu_start();
 	boot_cpu_hotplug_init();
 
 	pr_notice("Kernel command line: %s\n", saved_command_line);
@@ -1519,6 +1531,19 @@ void __init console_on_rootfs(void)
 	fput(file);
 }
 
+static void __init smp_prepare_cpus_done(unsigned int setup_max_cpus)
+{
+	/* Different ARCHs may override smp_prepare_cpus() */
+	smp_prepare_cpus(setup_max_cpus);
+
+#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
+#ifndef cpu_to_node
+	/* Change to the formal function. */
+	cpu_to_node = _cpu_to_node;
+#endif
+#endif
+}
+
 static noinline void __init kernel_init_freeable(void)
 {
 	/* Now the scheduler is fully set up and can do blocking allocations */
@@ -1531,7 +1556,7 @@ static noinline void __init kernel_init_freeable(void)
 
 	cad_pid = get_pid(task_pid(current));
 
-	smp_prepare_cpus(setup_max_cpus);
+	smp_prepare_cpus_done(setup_max_cpus);
 
 	workqueue_init();
 
-- 
2.40.1