This patch adds multi-processor (SMP) support for LoongArch.
Signed-off-by: Huacai Chen <[email protected]>
---
arch/loongarch/Kconfig | 43 +-
arch/loongarch/include/asm/atomic.h | 4 +
arch/loongarch/include/asm/barrier.h | 108 +++
arch/loongarch/include/asm/cmpxchg.h | 1 +
arch/loongarch/include/asm/futex.h | 1 +
arch/loongarch/include/asm/hardirq.h | 2 +
arch/loongarch/include/asm/irq.h | 3 +
arch/loongarch/include/asm/percpu.h | 202 ++++++
arch/loongarch/include/asm/pgtable.h | 21 +
arch/loongarch/include/asm/smp.h | 134 ++++
arch/loongarch/include/asm/stackframe.h | 20 +
arch/loongarch/include/asm/tlbflush.h | 22 +-
arch/loongarch/include/asm/topology.h | 7 +-
arch/loongarch/kernel/Makefile | 2 +
arch/loongarch/kernel/acpi.c | 70 +-
arch/loongarch/kernel/head.S | 23 +
arch/loongarch/kernel/irq.c | 46 +-
arch/loongarch/kernel/proc.c | 5 +
arch/loongarch/kernel/process.c | 7 +
arch/loongarch/kernel/reset.c | 12 +
arch/loongarch/kernel/setup.c | 26 +
arch/loongarch/kernel/signal.c | 20 +
arch/loongarch/kernel/smp.c | 894 ++++++++++++++++++++++++
arch/loongarch/kernel/topology.c | 43 +-
arch/loongarch/kernel/vmlinux.lds.S | 3 +
arch/loongarch/mm/tlbex.S | 69 ++
include/linux/cpuhotplug.h | 1 +
27 files changed, 1777 insertions(+), 12 deletions(-)
create mode 100644 arch/loongarch/include/asm/smp.h
create mode 100644 arch/loongarch/kernel/smp.c
diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig
index da9fe71b0f5f..8627d2520924 100644
--- a/arch/loongarch/Kconfig
+++ b/arch/loongarch/Kconfig
@@ -61,6 +61,7 @@ config LOONGARCH
select GENERIC_LIB_LSHRDI3
select GENERIC_LIB_UCMPDI2
select GENERIC_PCI_IOMAP
+ select GENERIC_SMP_IDLE_THREAD
select GENERIC_TIME_VSYSCALL
select GPIOLIB
select HANDLE_DOMAIN_IRQ
@@ -93,7 +94,7 @@ config LOONGARCH
select HAVE_RSEQ
select HAVE_SYSCALL_TRACEPOINTS
select HAVE_TIF_NOHZ
- select HAVE_VIRT_CPU_ACCOUNTING_GEN
+ select HAVE_VIRT_CPU_ACCOUNTING_GEN if !SMP
select IRQ_FORCED_THREADING
select IRQ_LOONGARCH_CPU
select MODULES_USE_ELF_REL if MODULES
@@ -262,6 +263,46 @@ config EFI
resultant kernel should continue to boot on existing non-EFI
platforms.
+config SMP
+ bool "Multi-Processing support"
+ help
+ This enables support for systems with more than one CPU. If you have
+ a system with only one CPU, say N. If you have a system with more
+ than one CPU, say Y.
+
+ If you say N here, the kernel will run on uni- and multiprocessor
+ machines, but will use only one CPU of a multiprocessor machine. If
+ you say Y here, the kernel will run on many, but not all,
+ uniprocessor machines. On a uniprocessor machine, the kernel
+ will run faster if you say N here.
+
+ People using multiprocessor machines who say Y here should also say
+ Y to "Enhanced Real Time Clock Support", below.
+
+ See also the SMP-HOWTO available at
+ <http://www.tldp.org/docs.html#howto>.
+
+ If you don't know what to do here, say N.
+
+config HOTPLUG_CPU
+ bool "Support for hot-pluggable CPUs"
+ depends on SMP
+ help
+ Say Y here to allow turning CPUs off and on. CPUs can be
+ controlled through /sys/devices/system/cpu.
+ (Note: power management support will enable this option
+ automatically on SMP systems. )
+ Say N if you want to disable CPU hotplug.
+
+config NR_CPUS
+ int "Maximum number of CPUs (2-256)"
+ range 2 256
+ depends on SMP
+ default "16"
+ help
+ This allows you to specify the maximum number of CPUs which this
+ kernel will support.
+
config FORCE_MAX_ZONEORDER
int "Maximum zone order"
range 14 64 if PAGE_SIZE_64KB
diff --git a/arch/loongarch/include/asm/atomic.h b/arch/loongarch/include/asm/atomic.h
index 091552d84aa0..afb5256e5d11 100644
--- a/arch/loongarch/include/asm/atomic.h
+++ b/arch/loongarch/include/asm/atomic.h
@@ -125,6 +125,7 @@ static inline int arch_atomic_sub_if_positive(int i, atomic_t *v)
" sc.w %1, %2 \n"
" beq $zero, %1, 1b \n"
"2: \n"
+ __WEAK_LLSC_MB
: "=&r" (result), "=&r" (temp),
"+" GCC_OFF_SMALL_ASM() (v->counter)
: "I" (-i));
@@ -137,6 +138,7 @@ static inline int arch_atomic_sub_if_positive(int i, atomic_t *v)
" sc.w %1, %2 \n"
" beq $zero, %1, 1b \n"
"2: \n"
+ __WEAK_LLSC_MB
: "=&r" (result), "=&r" (temp),
"+" GCC_OFF_SMALL_ASM() (v->counter)
: "r" (i));
@@ -263,6 +265,7 @@ static inline long arch_atomic64_sub_if_positive(long i, atomic64_t *v)
" sc.d %1, %2 \n"
" beq %1, $zero, 1b \n"
"2: \n"
+ __WEAK_LLSC_MB
: "=&r" (result), "=&r" (temp),
"+" GCC_OFF_SMALL_ASM() (v->counter)
: "I" (-i));
@@ -275,6 +278,7 @@ static inline long arch_atomic64_sub_if_positive(long i, atomic64_t *v)
" sc.d %1, %2 \n"
" beq %1, $zero, 1b \n"
"2: \n"
+ __WEAK_LLSC_MB
: "=&r" (result), "=&r" (temp),
"+" GCC_OFF_SMALL_ASM() (v->counter)
: "r" (i));
diff --git a/arch/loongarch/include/asm/barrier.h b/arch/loongarch/include/asm/barrier.h
index 8ab8d8f15b88..55b7fd325cf3 100644
--- a/arch/loongarch/include/asm/barrier.h
+++ b/arch/loongarch/include/asm/barrier.h
@@ -20,6 +20,19 @@
#define mb() fast_mb()
#define iob() fast_iob()
+#define __smp_mb() __asm__ __volatile__("dbar 0" : : : "memory")
+#define __smp_rmb() __asm__ __volatile__("dbar 0" : : : "memory")
+#define __smp_wmb() __asm__ __volatile__("dbar 0" : : : "memory")
+
+#ifdef CONFIG_SMP
+#define __WEAK_LLSC_MB " dbar 0 \n"
+#else
+#define __WEAK_LLSC_MB " \n"
+#endif
+
+#define __smp_mb__before_atomic() barrier()
+#define __smp_mb__after_atomic() barrier()
+
/**
* array_index_mask_nospec() - generate a ~0 mask when index < size, 0 otherwise
* @index: array element index
@@ -48,6 +61,101 @@ static inline unsigned long array_index_mask_nospec(unsigned long index,
return mask;
}
+#define __smp_load_acquire(p) \
+({ \
+ union { typeof(*p) __val; char __c[1]; } __u; \
+ unsigned long __tmp = 0; \
+ compiletime_assert_atomic_type(*p); \
+ switch (sizeof(*p)) { \
+ case 1: \
+ *(__u8 *)__u.__c = *(volatile __u8 *)p; \
+ __smp_mb(); \
+ break; \
+ case 2: \
+ *(__u16 *)__u.__c = *(volatile __u16 *)p; \
+ __smp_mb(); \
+ break; \
+ case 4: \
+ __asm__ __volatile__( \
+ "amor.w %[val], %[tmp], %[mem] \n" \
+ : [val] "=&r" (*(__u32 *)__u.__c) \
+ : [mem] "ZB" (*(u32 *) p), [tmp] "r" (__tmp) \
+ : "memory"); \
+ break; \
+ case 8: \
+ __asm__ __volatile__( \
+ "amor.d %[val], %[tmp], %[mem] \n" \
+ : [val] "=&r" (*(__u64 *)__u.__c) \
+ : [mem] "ZB" (*(u64 *) p), [tmp] "r" (__tmp) \
+ : "memory"); \
+ break; \
+ } \
+ (typeof(*p))__u.__val; \
+})
+
+#define __smp_store_release(p, v) \
+do { \
+ union { typeof(*p) __val; char __c[1]; } __u = \
+ { .__val = (__force typeof(*p)) (v) }; \
+ unsigned long __tmp; \
+ compiletime_assert_atomic_type(*p); \
+ switch (sizeof(*p)) { \
+ case 1: \
+ __smp_mb(); \
+ *(volatile __u8 *)p = *(__u8 *)__u.__c; \
+ break; \
+ case 2: \
+ __smp_mb(); \
+ *(volatile __u16 *)p = *(__u16 *)__u.__c; \
+ break; \
+ case 4: \
+ __asm__ __volatile__( \
+ "amswap.w %[tmp], %[val], %[mem] \n" \
+ : [mem] "+ZB" (*(u32 *)p), [tmp] "=&r" (__tmp) \
+ : [val] "r" (*(__u32 *)__u.__c) \
+ : ); \
+ break; \
+ case 8: \
+ __asm__ __volatile__( \
+ "amswap.d %[tmp], %[val], %[mem] \n" \
+ : [mem] "+ZB" (*(u64 *)p), [tmp] "=&r" (__tmp) \
+ : [val] "r" (*(__u64 *)__u.__c) \
+ : ); \
+ break; \
+ } \
+} while (0)
+
+#define __smp_store_mb(p, v) \
+do { \
+ union { typeof(p) __val; char __c[1]; } __u = \
+ { .__val = (__force typeof(p)) (v) }; \
+ unsigned long __tmp; \
+ switch (sizeof(p)) { \
+ case 1: \
+ *(volatile __u8 *)&p = *(__u8 *)__u.__c; \
+ __smp_mb(); \
+ break; \
+ case 2: \
+ *(volatile __u16 *)&p = *(__u16 *)__u.__c; \
+ __smp_mb(); \
+ break; \
+ case 4: \
+ __asm__ __volatile__( \
+ "amswap.w %[tmp], %[val], %[mem] \n" \
+ : [mem] "+ZB" (*(u32 *)&p), [tmp] "=&r" (__tmp) \
+ : [val] "r" (*(__u32 *)__u.__c) \
+ : ); \
+ break; \
+ case 8: \
+ __asm__ __volatile__( \
+ "amswap.d %[tmp], %[val], %[mem] \n" \
+ : [mem] "+ZB" (*(u64 *)&p), [tmp] "=&r" (__tmp) \
+ : [val] "r" (*(__u64 *)__u.__c) \
+ : ); \
+ break; \
+ } \
+} while (0)
+
#include <asm-generic/barrier.h>
#endif /* __ASM_BARRIER_H */
diff --git a/arch/loongarch/include/asm/cmpxchg.h b/arch/loongarch/include/asm/cmpxchg.h
index 13ee1b62dc12..76d601091c1d 100644
--- a/arch/loongarch/include/asm/cmpxchg.h
+++ b/arch/loongarch/include/asm/cmpxchg.h
@@ -67,6 +67,7 @@ static inline unsigned long __xchg(volatile void *ptr, unsigned long x,
" " st " $t0, %1 \n" \
" beq $zero, $t0, 1b \n" \
"2: \n" \
+ __WEAK_LLSC_MB \
: "=&r" (__ret), "=ZB"(*m) \
: "ZB"(*m), "Jr" (old), "Jr" (new) \
: "t0", "memory"); \
diff --git a/arch/loongarch/include/asm/futex.h b/arch/loongarch/include/asm/futex.h
index 6512feb041d1..22ddd3f7114b 100644
--- a/arch/loongarch/include/asm/futex.h
+++ b/arch/loongarch/include/asm/futex.h
@@ -86,6 +86,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, u32 oldval, u32 newv
"2: sc.w $t0, %2 \n"
" beq $zero, $t0, 1b \n"
"3: \n"
+ __WEAK_LLSC_MB
" .section .fixup,\"ax\" \n"
"4: li.d %0, %6 \n"
" b 3b \n"
diff --git a/arch/loongarch/include/asm/hardirq.h b/arch/loongarch/include/asm/hardirq.h
index ccde14a45f67..d50298147a02 100644
--- a/arch/loongarch/include/asm/hardirq.h
+++ b/arch/loongarch/include/asm/hardirq.h
@@ -21,4 +21,6 @@ typedef struct {
DECLARE_PER_CPU_ALIGNED(irq_cpustat_t, irq_stat);
+#define __ARCH_IRQ_STAT
+
#endif /* _ASM_HARDIRQ_H */
diff --git a/arch/loongarch/include/asm/irq.h b/arch/loongarch/include/asm/irq.h
index 4c0843952ee2..061c56857fbf 100644
--- a/arch/loongarch/include/asm/irq.h
+++ b/arch/loongarch/include/asm/irq.h
@@ -120,6 +120,9 @@ extern struct irq_domain *liointc_domain;
extern struct irq_domain *pch_msi_domain[MAX_IO_PICS];
extern struct irq_domain *pch_pic_domain[MAX_IO_PICS];
+extern void fixup_irqs(void);
+extern irqreturn_t loongson3_ipi_interrupt(int irq, void *dev);
+
#include <asm-generic/irq.h>
#endif /* _ASM_IRQ_H */
diff --git a/arch/loongarch/include/asm/percpu.h b/arch/loongarch/include/asm/percpu.h
index ea5979872485..572f468fbb8a 100644
--- a/arch/loongarch/include/asm/percpu.h
+++ b/arch/loongarch/include/asm/percpu.h
@@ -5,6 +5,8 @@
#ifndef __ASM_PERCPU_H
#define __ASM_PERCPU_H
+#include <asm/cmpxchg.h>
+
/* Use r21 for fast access */
register unsigned long __my_cpu_offset __asm__("$r21");
@@ -15,6 +17,206 @@ static inline void set_my_cpu_offset(unsigned long off)
}
#define __my_cpu_offset __my_cpu_offset
+#define PERCPU_OP(op, asm_op, c_op) \
+static inline unsigned long __percpu_##op(void *ptr, \
+ unsigned long val, int size) \
+{ \
+ unsigned long ret; \
+ \
+ switch (size) { \
+ case 4: \
+ __asm__ __volatile__( \
+ "am"#asm_op".w" " %[ret], %[val], %[ptr] \n" \
+ : [ret] "=&r" (ret), [ptr] "+ZB"(*(u32 *)ptr) \
+ : [val] "r" (val)); \
+ break; \
+ case 8: \
+ __asm__ __volatile__( \
+ "am"#asm_op".d" " %[ret], %[val], %[ptr] \n" \
+ : [ret] "=&r" (ret), [ptr] "+ZB"(*(u64 *)ptr) \
+ : [val] "r" (val)); \
+ break; \
+ default: \
+ ret = 0; \
+ BUILD_BUG(); \
+ } \
+ \
+ return ret c_op val; \
+}
+
+PERCPU_OP(add, add, +)
+PERCPU_OP(and, and, &)
+PERCPU_OP(or, or, |)
+#undef PERCPU_OP
+
+static inline unsigned long __percpu_read(void *ptr, int size)
+{
+ unsigned long ret;
+
+ switch (size) {
+ case 1:
+ __asm__ __volatile__ ("ldx.b %[ret], $r21, %[ptr] \n"
+ : [ret] "=&r"(ret)
+ : [ptr] "r"(ptr)
+ :);
+ break;
+ case 2:
+ __asm__ __volatile__ ("ldx.h %[ret], $r21, %[ptr] \n"
+ : [ret] "=&r"(ret)
+ : [ptr] "r"(ptr)
+ :);
+ break;
+ case 4:
+ __asm__ __volatile__ ("ldx.w %[ret], $r21, %[ptr] \n"
+ : [ret] "=&r"(ret)
+ : [ptr] "r"(ptr)
+ :);
+ break;
+ case 8:
+ __asm__ __volatile__ ("ldx.d %[ret], $r21, %[ptr] \n"
+ : [ret] "=&r"(ret)
+ : [ptr] "r"(ptr)
+ :);
+ break;
+ default:
+ ret = 0;
+ BUILD_BUG();
+ }
+
+ return ret;
+}
+
+static inline void __percpu_write(void *ptr, unsigned long val, int size)
+{
+ switch (size) {
+ case 1:
+ __asm__ __volatile__("stx.b %[val], $r21, %[ptr] \n"
+ :
+ : [val] "r" (val), [ptr] "r" (ptr)
+ : "memory");
+ break;
+ case 2:
+ __asm__ __volatile__("stx.h %[val], $r21, %[ptr] \n"
+ :
+ : [val] "r" (val), [ptr] "r" (ptr)
+ : "memory");
+ break;
+ case 4:
+ __asm__ __volatile__("stx.w %[val], $r21, %[ptr] \n"
+ :
+ : [val] "r" (val), [ptr] "r" (ptr)
+ : "memory");
+ break;
+ case 8:
+ __asm__ __volatile__("stx.d %[val], $r21, %[ptr] \n"
+ :
+ : [val] "r" (val), [ptr] "r" (ptr)
+ : "memory");
+ break;
+ default:
+ BUILD_BUG();
+ }
+}
+
+static inline unsigned long __percpu_xchg(void *ptr, unsigned long val,
+ int size)
+{
+ switch (size) {
+ case 1:
+ case 2:
+ return __xchg_small((volatile void *)ptr, val, size);
+
+ case 4:
+ return __xchg_asm("amswap.w", (volatile u32 *)ptr, (u32)val);
+
+ case 8:
+ return __xchg_asm("amswap.d", (volatile u64 *)ptr, (u64)val);
+
+ default:
+ BUILD_BUG();
+ }
+
+ return 0;
+}
+
+/* this_cpu_cmpxchg */
+#define _protect_cmpxchg_local(pcp, o, n) \
+({ \
+ typeof(*raw_cpu_ptr(&(pcp))) __ret; \
+ preempt_disable(); \
+ __ret = cmpxchg_local(raw_cpu_ptr(&(pcp)), o, n); \
+ preempt_enable(); \
+ __ret; \
+})
+
+#define _percpu_read(pcp) \
+({ \
+ typeof(pcp) __retval; \
+ __retval = (typeof(pcp))__percpu_read(&(pcp), sizeof(pcp)); \
+ __retval; \
+})
+
+#define _percpu_write(pcp, val) \
+do { \
+ __percpu_write(&(pcp), (unsigned long)(val), sizeof(pcp)); \
+} while (0) \
+
+#define _pcp_protect(operation, pcp, val) \
+({ \
+ typeof(pcp) __retval; \
+ preempt_disable(); \
+ __retval = (typeof(pcp))operation(raw_cpu_ptr(&(pcp)), \
+ (val), sizeof(pcp)); \
+ preempt_enable(); \
+ __retval; \
+})
+
+#define _percpu_add(pcp, val) \
+ _pcp_protect(__percpu_add, pcp, val)
+
+#define _percpu_add_return(pcp, val) _percpu_add(pcp, val)
+
+#define _percpu_and(pcp, val) \
+ _pcp_protect(__percpu_and, pcp, val)
+
+#define _percpu_or(pcp, val) \
+ _pcp_protect(__percpu_or, pcp, val)
+
+#define _percpu_xchg(pcp, val) ((typeof(pcp)) \
+ _pcp_protect(__percpu_xchg, pcp, (unsigned long)(val)))
+
+#define this_cpu_add_4(pcp, val) _percpu_add(pcp, val)
+#define this_cpu_add_8(pcp, val) _percpu_add(pcp, val)
+
+#define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val)
+#define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val)
+
+#define this_cpu_and_4(pcp, val) _percpu_and(pcp, val)
+#define this_cpu_and_8(pcp, val) _percpu_and(pcp, val)
+
+#define this_cpu_or_4(pcp, val) _percpu_or(pcp, val)
+#define this_cpu_or_8(pcp, val) _percpu_or(pcp, val)
+
+#define this_cpu_read_1(pcp) _percpu_read(pcp)
+#define this_cpu_read_2(pcp) _percpu_read(pcp)
+#define this_cpu_read_4(pcp) _percpu_read(pcp)
+#define this_cpu_read_8(pcp) _percpu_read(pcp)
+
+#define this_cpu_write_1(pcp, val) _percpu_write(pcp, val)
+#define this_cpu_write_2(pcp, val) _percpu_write(pcp, val)
+#define this_cpu_write_4(pcp, val) _percpu_write(pcp, val)
+#define this_cpu_write_8(pcp, val) _percpu_write(pcp, val)
+
+#define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val)
+#define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val)
+#define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val)
+#define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val)
+
+#define this_cpu_cmpxchg_1(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
+#define this_cpu_cmpxchg_2(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
+#define this_cpu_cmpxchg_4(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
+#define this_cpu_cmpxchg_8(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
+
#include <asm-generic/percpu.h>
#endif /* __ASM_PERCPU_H */
diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h
index 0fb709e751e9..9c1c431f788b 100644
--- a/arch/loongarch/include/asm/pgtable.h
+++ b/arch/loongarch/include/asm/pgtable.h
@@ -299,8 +299,29 @@ static inline void set_pte(pte_t *ptep, pte_t pteval)
* Make sure the buddy is global too (if it's !none,
* it better already be global)
*/
+#ifdef CONFIG_SMP
+ /*
+ * For SMP, multiple CPUs can race, so we need to do
+ * this atomically.
+ */
+ unsigned long page_global = _PAGE_GLOBAL;
+ unsigned long tmp;
+
+ __asm__ __volatile__ (
+ "1:" __LL "%[tmp], %[buddy] \n"
+ " bnez %[tmp], 2f \n"
+ " or %[tmp], %[tmp], %[global] \n"
+ __SC "%[tmp], %[buddy] \n"
+ " beqz %[tmp], 1b \n"
+ " nop \n"
+ "2: \n"
+ __WEAK_LLSC_MB
+ : [buddy] "+m" (buddy->pte), [tmp] "=&r" (tmp)
+ : [global] "r" (page_global));
+#else /* !CONFIG_SMP */
if (pte_none(*buddy))
pte_val(*buddy) = pte_val(*buddy) | _PAGE_GLOBAL;
+#endif /* CONFIG_SMP */
}
}
diff --git a/arch/loongarch/include/asm/smp.h b/arch/loongarch/include/asm/smp.h
new file mode 100644
index 000000000000..446e2b8f482f
--- /dev/null
+++ b/arch/loongarch/include/asm/smp.h
@@ -0,0 +1,134 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Author: Huacai Chen <[email protected]>
+ * Copyright (C) 2020-2021 Loongson Technology Corporation Limited
+ */
+#ifndef __ASM_SMP_H
+#define __ASM_SMP_H
+
+#include <linux/atomic.h>
+#include <linux/bitops.h>
+#include <linux/linkage.h>
+#include <linux/smp.h>
+#include <linux/threads.h>
+#include <linux/cpumask.h>
+
+struct task_struct;
+
+struct plat_smp_ops {
+ void (*send_ipi_single)(int cpu, unsigned int action);
+ void (*send_ipi_mask)(const struct cpumask *mask, unsigned int action);
+ void (*smp_setup)(void);
+ void (*prepare_cpus)(unsigned int max_cpus);
+ int (*boot_secondary)(int cpu, struct task_struct *idle);
+ void (*init_secondary)(void);
+ void (*smp_finish)(void);
+#ifdef CONFIG_HOTPLUG_CPU
+ int (*cpu_disable)(void);
+ void (*cpu_die)(unsigned int cpu);
+#endif
+};
+
+#ifdef CONFIG_SMP
+
+static inline void plat_smp_setup(void)
+{
+ extern const struct plat_smp_ops *mp_ops; /* private */
+
+ mp_ops->smp_setup();
+}
+
+#else /* !CONFIG_SMP */
+
+static inline void plat_smp_setup(void) { }
+
+#endif /* !CONFIG_SMP */
+
+extern int smp_num_siblings;
+extern int num_processors;
+extern int disabled_cpus;
+extern cpumask_t cpu_sibling_map[];
+extern cpumask_t cpu_core_map[];
+extern cpumask_t cpu_foreign_map[];
+
+static inline int raw_smp_processor_id(void)
+{
+#if defined(__VDSO__)
+ extern int vdso_smp_processor_id(void)
+ __compiletime_error("VDSO should not call smp_processor_id()");
+ return vdso_smp_processor_id();
+#else
+ return current_thread_info()->cpu;
+#endif
+}
+#define raw_smp_processor_id raw_smp_processor_id
+
+/* Map from cpu id to sequential logical cpu number. This will only
+ * not be idempotent when cpus failed to come on-line. */
+extern int __cpu_number_map[NR_CPUS];
+#define cpu_number_map(cpu) __cpu_number_map[cpu]
+
+/* The reverse map from sequential logical cpu number to cpu id. */
+extern int __cpu_logical_map[NR_CPUS];
+#define cpu_logical_map(cpu) __cpu_logical_map[cpu]
+
+#define cpu_physical_id(cpu) cpu_logical_map(cpu)
+
+#define SMP_BOOT_CPU 0x1
+#define SMP_RESCHEDULE 0x2
+#define SMP_CALL_FUNCTION 0x4
+
+extern asmlinkage void smpboot_entry(void);
+
+extern void calculate_cpu_foreign_map(void);
+
+/*
+ * Generate IPI list text
+ */
+extern void show_ipi_list(struct seq_file *p, int prec);
+
+/*
+ * this function sends a 'reschedule' IPI to another CPU.
+ * it goes straight through and wastes no time serializing
+ * anything. Worst case is that we lose a reschedule ...
+ */
+static inline void smp_send_reschedule(int cpu)
+{
+ extern const struct plat_smp_ops *mp_ops; /* private */
+
+ mp_ops->send_ipi_single(cpu, SMP_RESCHEDULE);
+}
+
+#ifdef CONFIG_HOTPLUG_CPU
+static inline int __cpu_disable(void)
+{
+ extern const struct plat_smp_ops *mp_ops; /* private */
+
+ return mp_ops->cpu_disable();
+}
+
+static inline void __cpu_die(unsigned int cpu)
+{
+ extern const struct plat_smp_ops *mp_ops; /* private */
+
+ mp_ops->cpu_die(cpu);
+}
+
+extern void play_dead(void);
+#endif
+
+static inline void arch_send_call_function_single_ipi(int cpu)
+{
+ extern const struct plat_smp_ops *mp_ops; /* private */
+
+ mp_ops->send_ipi_single(cpu, SMP_CALL_FUNCTION);
+}
+
+static inline void arch_send_call_function_ipi_mask(const struct cpumask *mask)
+{
+ extern const struct plat_smp_ops *mp_ops; /* private */
+
+ mp_ops->send_ipi_mask(mask, SMP_CALL_FUNCTION);
+}
+
+#endif /* __ASM_SMP_H */
diff --git a/arch/loongarch/include/asm/stackframe.h b/arch/loongarch/include/asm/stackframe.h
index 01059d75e1bb..e4190c64feef 100644
--- a/arch/loongarch/include/asm/stackframe.h
+++ b/arch/loongarch/include/asm/stackframe.h
@@ -79,6 +79,25 @@
* t0 and loads the new value in sp. If not, it clobbers t0 and
* stores the new value in t1, leaving sp unaffected.
*/
+#ifdef CONFIG_SMP
+ /* SMP variation */
+ .macro get_saved_sp docfi=0 tosp=0
+ csrrd t0, PERCPU_BASE_KS
+ la.abs t1, kernelsp
+ LONG_ADDU t1, t1, t0
+ .if \tosp
+ or t0, sp, zero
+ LONG_L sp, t1, 0
+ .endif
+ .endm
+
+ .macro set_saved_sp stackp temp temp2
+ la.abs \temp, kernelsp
+ LONG_ADDU \temp, \temp, x0
+ LONG_S \stackp, \temp, 0
+ .endm
+#else /* !CONFIG_SMP */
+ /* Uniprocessor variation */
.macro get_saved_sp docfi=0 tosp=0
la.abs t1, kernelsp
.if \tosp
@@ -96,6 +115,7 @@
la.abs \temp, kernelsp
LONG_S \stackp, \temp, 0
.endm
+#endif
.macro SAVE_SOME docfi=0
csrrd t1, LOONGARCH_CSR_PRMD
diff --git a/arch/loongarch/include/asm/tlbflush.h b/arch/loongarch/include/asm/tlbflush.h
index a5672367d966..32862e53b845 100644
--- a/arch/loongarch/include/asm/tlbflush.h
+++ b/arch/loongarch/include/asm/tlbflush.h
@@ -18,14 +18,22 @@
*/
extern void local_flush_tlb_all(void);
extern void local_flush_tlb_mm(struct mm_struct *mm);
-extern void local_flush_tlb_range(struct vm_area_struct *vma,
- unsigned long start, unsigned long end);
-extern void local_flush_tlb_kernel_range(unsigned long start,
- unsigned long end);
-extern void local_flush_tlb_page(struct vm_area_struct *vma,
- unsigned long page);
+extern void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
+extern void local_flush_tlb_kernel_range(unsigned long start, unsigned long end);
+extern void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
extern void local_flush_tlb_one(unsigned long vaddr);
+#ifdef CONFIG_SMP
+
+extern void flush_tlb_all(void);
+extern void flush_tlb_mm(struct mm_struct *);
+extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long, unsigned long);
+extern void flush_tlb_kernel_range(unsigned long, unsigned long);
+extern void flush_tlb_page(struct vm_area_struct *, unsigned long);
+extern void flush_tlb_one(unsigned long vaddr);
+
+#else /* CONFIG_SMP */
+
#define flush_tlb_all() local_flush_tlb_all()
#define flush_tlb_mm(mm) local_flush_tlb_mm(mm)
#define flush_tlb_range(vma, vmaddr, end) local_flush_tlb_range(vma, vmaddr, end)
@@ -33,4 +41,6 @@ extern void local_flush_tlb_one(unsigned long vaddr);
#define flush_tlb_page(vma, page) local_flush_tlb_page(vma, page)
#define flush_tlb_one(vaddr) local_flush_tlb_one(vaddr)
+#endif /* CONFIG_SMP */
+
#endif /* __ASM_TLBFLUSH_H */
diff --git a/arch/loongarch/include/asm/topology.h b/arch/loongarch/include/asm/topology.h
index bf39f3911424..bc541a723f8a 100644
--- a/arch/loongarch/include/asm/topology.h
+++ b/arch/loongarch/include/asm/topology.h
@@ -7,7 +7,12 @@
#include <linux/smp.h>
-#define cpu_logical_map(cpu) 0
+#ifdef CONFIG_SMP
+#define topology_physical_package_id(cpu) (cpu_data[cpu].package)
+#define topology_core_id(cpu) (cpu_core(&cpu_data[cpu]))
+#define topology_core_cpumask(cpu) (&cpu_core_map[cpu])
+#define topology_sibling_cpumask(cpu) (&cpu_sibling_map[cpu])
+#endif
#include <asm-generic/topology.h>
diff --git a/arch/loongarch/kernel/Makefile b/arch/loongarch/kernel/Makefile
index 9af33727664d..9d5cb652bfc0 100644
--- a/arch/loongarch/kernel/Makefile
+++ b/arch/loongarch/kernel/Makefile
@@ -19,4 +19,6 @@ obj-$(CONFIG_MODULES) += module.o
obj-$(CONFIG_PROC_FS) += proc.o
+obj-$(CONFIG_SMP) += smp.o
+
CPPFLAGS_vmlinux.lds := $(KBUILD_CFLAGS)
diff --git a/arch/loongarch/kernel/acpi.c b/arch/loongarch/kernel/acpi.c
index 3f77c937e9ad..5ca7ffe9adcc 100644
--- a/arch/loongarch/kernel/acpi.c
+++ b/arch/loongarch/kernel/acpi.c
@@ -125,6 +125,35 @@ void __init acpi_boot_table_init(void)
}
}
+static int set_processor_mask(u32 id, u32 flags)
+{
+
+ int cpu, cpuid = id;
+
+ if (num_processors >= nr_cpu_ids) {
+ pr_warn("acpi: nr_cpus/possible_cpus limit of %i reached."
+ " processor 0x%x ignored.\n", nr_cpu_ids, cpuid);
+
+ return -ENODEV;
+
+ }
+ if (cpuid == loongson_sysconf.boot_cpu_id)
+ cpu = 0;
+ else
+ cpu = cpumask_next_zero(-1, cpu_present_mask);
+
+ if (flags & ACPI_MADT_ENABLED) {
+ num_processors++;
+ set_cpu_possible(cpu, true);
+ set_cpu_present(cpu, true);
+ __cpu_number_map[cpuid] = cpu;
+ __cpu_logical_map[cpu] = cpuid;
+ } else
+ disabled_cpus++;
+
+ return cpu;
+}
+
static int __init
acpi_parse_cpuintc(union acpi_subtable_headers *header, const unsigned long end)
{
@@ -135,6 +164,7 @@ acpi_parse_cpuintc(union acpi_subtable_headers *header, const unsigned long end)
return -EINVAL;
acpi_table_print_madt_entry(&header->common);
+ set_processor_mask(processor->core_id, processor->flags);
return 0;
}
@@ -236,7 +266,12 @@ acpi_parse_pch_lpc(union acpi_subtable_headers *header, const unsigned long end)
static void __init acpi_process_madt(void)
{
- int error;
+ int i, error;
+
+ for (i = 0; i < NR_CPUS; i++) {
+ __cpu_number_map[i] = -1;
+ __cpu_logical_map[i] = -1;
+ }
/* Parse MADT CPUINTC entries */
error = acpi_table_parse_madt(ACPI_MADT_TYPE_CORE_PIC, acpi_parse_cpuintc, MAX_CORE_PIC);
@@ -322,3 +357,36 @@ void __init arch_reserve_mem_area(acpi_physical_address addr, size_t size)
{
memblock_mark_nomap(addr, size);
}
+
+#ifdef CONFIG_ACPI_HOTPLUG_CPU
+
+#include <acpi/processor.h>
+
+int acpi_map_cpu(acpi_handle handle, phys_cpuid_t physid, u32 acpi_id, int *pcpu)
+{
+ int cpu;
+
+ cpu = set_processor_mask(physid, ACPI_MADT_ENABLED);
+ if (cpu < 0) {
+ pr_info(PREFIX "Unable to map lapic to logical cpu number\n");
+ return cpu;
+ }
+
+ *pcpu = cpu;
+
+ return 0;
+}
+EXPORT_SYMBOL(acpi_map_cpu);
+
+int acpi_unmap_cpu(int cpu)
+{
+ set_cpu_present(cpu, false);
+ num_processors--;
+
+ pr_info("cpu%d hot remove!\n", cpu);
+
+ return 0;
+}
+EXPORT_SYMBOL(acpi_unmap_cpu);
+
+#endif /* CONFIG_ACPI_HOTPLUG_CPU */
diff --git a/arch/loongarch/kernel/head.S b/arch/loongarch/kernel/head.S
index 53e02fb0559d..56e929778267 100644
--- a/arch/loongarch/kernel/head.S
+++ b/arch/loongarch/kernel/head.S
@@ -69,4 +69,27 @@ SYM_CODE_START(kernel_entry) # kernel entry point
SYM_CODE_END(kernel_entry)
+#ifdef CONFIG_SMP
+
+/*
+ * SMP slave cpus entry point. Board specific code for bootstrap calls this
+ * function after setting up the stack and tp registers.
+ */
+SYM_CODE_START(smpboot_entry)
+ li.d t0, CSR_DMW0_INIT # UC, PLV0
+ csrwr t0, LOONGARCH_CSR_DMWIN0
+ li.d t0, CSR_DMW1_INIT # CA, PLV0
+ csrwr t0, LOONGARCH_CSR_DMWIN1
+ li.w t0, 0xb0 # PLV=0, IE=0, PG=1
+ csrwr t0, LOONGARCH_CSR_CRMD
+ li.w t0, 0x04 # PLV=0, PIE=1, PWE=0
+ csrwr t0, LOONGARCH_CSR_PRMD
+ li.w t0, 0x00 # FPE=0, SXE=0, ASXE=0, BTE=0
+ csrwr t0, LOONGARCH_CSR_EUEN
+
+ bl start_secondary
+SYM_CODE_END(smpboot_entry)
+
+#endif /* CONFIG_SMP */
+
SYM_ENTRY(kernel_entry_end, SYM_L_GLOBAL, SYM_A_NONE)
diff --git a/arch/loongarch/kernel/irq.c b/arch/loongarch/kernel/irq.c
index c5de109957b1..cfe21c128450 100644
--- a/arch/loongarch/kernel/irq.c
+++ b/arch/loongarch/kernel/irq.c
@@ -54,6 +54,41 @@ int find_pch_pic(u32 gsi)
return -1;
}
+#ifdef CONFIG_HOTPLUG_CPU
+static void handle_irq_affinity(void)
+{
+ struct irq_desc *desc;
+ struct irq_chip *chip;
+ unsigned int irq;
+ unsigned long flags;
+ struct cpumask *affinity;
+
+ for_each_active_irq(irq) {
+ desc = irq_to_desc(irq);
+ if (!desc)
+ continue;
+
+ raw_spin_lock_irqsave(&desc->lock, flags);
+
+ affinity = desc->irq_data.common->affinity;
+ if (!cpumask_intersects(affinity, cpu_online_mask))
+ cpumask_copy(affinity, cpu_online_mask);
+
+ chip = irq_data_get_irq_chip(&desc->irq_data);
+ if (chip && chip->irq_set_affinity)
+ chip->irq_set_affinity(&desc->irq_data, desc->irq_data.common->affinity, true);
+ raw_spin_unlock_irqrestore(&desc->lock, flags);
+ }
+}
+
+void fixup_irqs(void)
+{
+ handle_irq_affinity();
+ irq_cpu_offline();
+ clear_csr_ecfg(ECFG0_IM);
+}
+#endif
+
/*
* 'what should we do if we get a hw irq event on an illegal vector'.
* each architecture has to answer this themselves.
@@ -72,6 +107,7 @@ asmlinkage void spurious_interrupt(void)
int arch_show_interrupts(struct seq_file *p, int prec)
{
+ show_ipi_list(p, prec);
seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count));
return 0;
}
@@ -107,13 +143,21 @@ void __init setup_IRQ(void)
void __init init_IRQ(void)
{
- int i;
+ int i, r, ipi_irq;
+ static int ipi_dummy_dev;
unsigned int order = get_order(IRQ_STACK_SIZE);
clear_csr_ecfg(ECFG0_IM);
clear_csr_estat(ESTATF_IP);
setup_IRQ();
+#ifdef CONFIG_SMP
+ ipi_irq = get_ipi_irq();
+ irq_set_percpu_devid(ipi_irq);
+ r = request_percpu_irq(ipi_irq, loongson3_ipi_interrupt, "IPI", &ipi_dummy_dev);
+ if (r < 0)
+ panic("IPI IRQ request failed\n");
+#endif
for (i = 0; i < NR_IRQS; i++)
irq_set_noprobe(i);
diff --git a/arch/loongarch/kernel/proc.c b/arch/loongarch/kernel/proc.c
index 503fc6e4b96e..5bab1cc83fa3 100644
--- a/arch/loongarch/kernel/proc.c
+++ b/arch/loongarch/kernel/proc.c
@@ -35,6 +35,11 @@ static int show_cpuinfo(struct seq_file *m, void *v)
unsigned int fp_version = cpu_data[n].fpu_vers;
struct proc_cpuinfo_notifier_args proc_cpuinfo_notifier_args;
+#ifdef CONFIG_SMP
+ if (!cpu_online(n))
+ return 0;
+#endif
+
/*
* For the first processor also print the system type
*/
diff --git a/arch/loongarch/kernel/process.c b/arch/loongarch/kernel/process.c
index 3b647631bbb6..2ef232be0d67 100644
--- a/arch/loongarch/kernel/process.c
+++ b/arch/loongarch/kernel/process.c
@@ -54,6 +54,13 @@
unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE;
EXPORT_SYMBOL(boot_option_idle_override);
+#ifdef CONFIG_HOTPLUG_CPU
+void arch_cpu_idle_dead(void)
+{
+ play_dead();
+}
+#endif
+
asmlinkage void ret_from_fork(void);
asmlinkage void ret_from_kernel_thread(void);
diff --git a/arch/loongarch/kernel/reset.c b/arch/loongarch/kernel/reset.c
index b140c2aa2c5a..e26e322a83c5 100644
--- a/arch/loongarch/kernel/reset.c
+++ b/arch/loongarch/kernel/reset.c
@@ -65,16 +65,28 @@ EXPORT_SYMBOL(pm_power_off);
void machine_halt(void)
{
+#ifdef CONFIG_SMP
+ preempt_disable();
+ smp_send_stop();
+#endif
default_halt();
}
void machine_power_off(void)
{
+#ifdef CONFIG_SMP
+ preempt_disable();
+ smp_send_stop();
+#endif
pm_power_off();
}
void machine_restart(char *command)
{
+#ifdef CONFIG_SMP
+ preempt_disable();
+ smp_send_stop();
+#endif
do_kernel_restart(command);
pm_restart();
}
diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c
index 7ec32959b36c..3ad6dc1dc565 100644
--- a/arch/loongarch/kernel/setup.c
+++ b/arch/loongarch/kernel/setup.c
@@ -39,6 +39,7 @@
#include <asm/pgalloc.h>
#include <asm/sections.h>
#include <asm/setup.h>
+#include <asm/smp.h>
#include <asm/time.h>
#define SMBIOS_BIOSSIZE_OFFSET 0x09
@@ -436,6 +437,29 @@ void __init platform_init(void)
efi_runtime_init();
}
+#ifdef CONFIG_SMP
+static void __init prefill_possible_map(void)
+{
+ int i, possible;
+
+ possible = num_processors + disabled_cpus;
+ if (possible > nr_cpu_ids)
+ possible = nr_cpu_ids;
+
+ pr_info("SMP: Allowing %d CPUs, %d hotplug CPUs\n",
+ possible, max((possible - num_processors), 0));
+
+ for (i = 0; i < possible; i++)
+ set_cpu_possible(i, true);
+ for (; i < NR_CPUS; i++)
+ set_cpu_possible(i, false);
+
+ nr_cpu_ids = possible;
+}
+#else
+static inline void prefill_possible_map(void) {}
+#endif
+
void __init setup_arch(char **cmdline_p)
{
cpu_probe();
@@ -452,6 +476,8 @@ void __init setup_arch(char **cmdline_p)
arch_mem_init(cmdline_p);
resource_init();
+ plat_smp_setup();
+ prefill_possible_map();
paging_init();
}
diff --git a/arch/loongarch/kernel/signal.c b/arch/loongarch/kernel/signal.c
index 78319d2575c6..a513c463d67d 100644
--- a/arch/loongarch/kernel/signal.c
+++ b/arch/loongarch/kernel/signal.c
@@ -451,8 +451,27 @@ void arch_do_signal_or_restart(struct pt_regs *regs, bool has_signal)
restore_saved_sigmask();
}
+#ifdef CONFIG_SMP
+static int smp_save_fp_context(void __user *sc)
+{
+ return save_hw_fp_context(sc);
+}
+
+static int smp_restore_fp_context(void __user *sc)
+{
+ return cpu_has_fpu
+ ? restore_hw_fp_context(sc)
+ : copy_fp_from_sigcontext(sc);
+}
+#endif
+
static int signal_setup(void)
{
+#ifdef CONFIG_SMP
+ /* For now just do the cpu_has_fpu check when the functions are invoked */
+ save_fp_context = smp_save_fp_context;
+ restore_fp_context = smp_restore_fp_context;
+#else
if (cpu_has_fpu) {
save_fp_context = save_hw_fp_context;
restore_fp_context = restore_hw_fp_context;
@@ -460,6 +479,7 @@ static int signal_setup(void)
save_fp_context = copy_fp_to_sigcontext;
restore_fp_context = copy_fp_from_sigcontext;
}
+#endif /* CONFIG_SMP */
return 0;
}
diff --git a/arch/loongarch/kernel/smp.c b/arch/loongarch/kernel/smp.c
new file mode 100644
index 000000000000..75b86053dc7c
--- /dev/null
+++ b/arch/loongarch/kernel/smp.c
@@ -0,0 +1,894 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2020-2021 Loongson Technology Corporation Limited
+ */
+#include <linux/cache.h>
+#include <linux/cpu.h>
+#include <linux/cpumask.h>
+#include <linux/delay.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/smp.h>
+#include <linux/spinlock.h>
+#include <linux/threads.h>
+#include <linux/export.h>
+#include <linux/time.h>
+#include <linux/timex.h>
+#include <linux/err.h>
+#include <linux/seq_file.h>
+#include <linux/tracepoint.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/hotplug.h>
+#include <linux/sched/task_stack.h>
+
+#include <asm/cpu.h>
+#include <asm/idle.h>
+#include <asm/loongson.h>
+#include <asm/mmu_context.h>
+#include <asm/processor.h>
+#include <asm/setup.h>
+#include <asm/time.h>
+
+int __cpu_number_map[NR_CPUS]; /* Map physical to logical */
+EXPORT_SYMBOL(__cpu_number_map);
+
+int __cpu_logical_map[NR_CPUS]; /* Map logical to physical */
+EXPORT_SYMBOL(__cpu_logical_map);
+
+/* Number of threads (siblings) per CPU core */
+int smp_num_siblings = 1;
+EXPORT_SYMBOL(smp_num_siblings);
+
+/* Representing the threads (siblings) of each logical CPU */
+cpumask_t cpu_sibling_map[NR_CPUS] __read_mostly;
+EXPORT_SYMBOL(cpu_sibling_map);
+
+/* Representing the core map of multi-core chips of each logical CPU */
+cpumask_t cpu_core_map[NR_CPUS] __read_mostly;
+EXPORT_SYMBOL(cpu_core_map);
+
+static DECLARE_COMPLETION(cpu_starting);
+static DECLARE_COMPLETION(cpu_running);
+
+/*
+ * A logcal cpu mask containing only one VPE per core to
+ * reduce the number of IPIs on large MT systems.
+ */
+cpumask_t cpu_foreign_map[NR_CPUS] __read_mostly;
+EXPORT_SYMBOL(cpu_foreign_map);
+
+/* representing cpus for which sibling maps can be computed */
+static cpumask_t cpu_sibling_setup_map;
+
+/* representing cpus for which core maps can be computed */
+static cpumask_t cpu_core_setup_map;
+
+static DEFINE_PER_CPU(int, cpu_state);
+DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
+EXPORT_PER_CPU_SYMBOL(irq_stat);
+
+#define MAX_CPUS 64
+
+#define STATUS 0x00
+#define EN 0x04
+#define SET 0x08
+#define CLEAR 0x0c
+#define MBUF 0x20
+
+extern unsigned long long smp_group[MAX_PACKAGES];
+static u32 core_offsets[4] = {0x000, 0x100, 0x200, 0x300};
+
+static volatile void *ipi_set_regs[MAX_CPUS];
+static volatile void *ipi_clear_regs[MAX_CPUS];
+static volatile void *ipi_status_regs[MAX_CPUS];
+static volatile void *ipi_en_regs[MAX_CPUS];
+static volatile void *ipi_mailbox_buf[MAX_CPUS];
+
+static u32 (*ipi_read_clear)(int cpu);
+static void (*ipi_write_action)(int cpu, u32 action);
+
+enum ipi_msg_type {
+ IPI_RESCHEDULE,
+ IPI_CALL_FUNCTION,
+};
+
+static const char *ipi_types[NR_IPI] __tracepoint_string = {
+ [IPI_RESCHEDULE] = "Rescheduling interrupts",
+ [IPI_CALL_FUNCTION] = "Call Function interrupts",
+};
+
+void show_ipi_list(struct seq_file *p, int prec)
+{
+ unsigned int cpu, i;
+
+ for (i = 0; i < NR_IPI; i++) {
+ seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i, prec >= 4 ? " " : "");
+ for_each_online_cpu(cpu)
+ seq_printf(p, "%10u ", per_cpu(irq_stat, cpu).ipi_irqs[i]);
+ seq_printf(p, " LoongArch %d %s\n", i + 1, ipi_types[i]);
+ }
+}
+
+/* Send mail buffer via Mail_Send */
+static void csr_mail_send(uint64_t data, int cpu, int mailbox)
+{
+ uint64_t val;
+
+ /* Send high 32 bits */
+ val = IOCSR_MBUF_SEND_BLOCKING;
+ val |= (IOCSR_MBUF_SEND_BOX_HI(mailbox) << IOCSR_MBUF_SEND_BOX_SHIFT);
+ val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT);
+ val |= (data & IOCSR_MBUF_SEND_H32_MASK);
+ iocsr_writeq(val, LOONGARCH_IOCSR_MBUF_SEND);
+
+ /* Send low 32 bits */
+ val = IOCSR_MBUF_SEND_BLOCKING;
+ val |= (IOCSR_MBUF_SEND_BOX_LO(mailbox) << IOCSR_MBUF_SEND_BOX_SHIFT);
+ val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT);
+ val |= (data << IOCSR_MBUF_SEND_BUF_SHIFT);
+ iocsr_writeq(val, LOONGARCH_IOCSR_MBUF_SEND);
+};
+
+static u32 csr_ipi_read_clear(int cpu)
+{
+ u32 action;
+
+ /* Load the ipi register to figure out what we're supposed to do */
+ action = iocsr_readl(LOONGARCH_IOCSR_IPI_STATUS);
+ /* Clear the ipi register to clear the interrupt */
+ iocsr_writel(action, LOONGARCH_IOCSR_IPI_CLEAR);
+
+ return action;
+}
+
+static void csr_ipi_write_action(int cpu, u32 action)
+{
+ unsigned int irq = 0;
+
+ while ((irq = ffs(action))) {
+ uint32_t val = IOCSR_IPI_SEND_BLOCKING;
+
+ val |= (irq - 1);
+ val |= (cpu << IOCSR_IPI_SEND_CPU_SHIFT);
+ iocsr_writel(val, LOONGARCH_IOCSR_IPI_SEND);
+ action &= ~BIT(irq - 1);
+ }
+}
+
+static u32 legacy_ipi_read_clear(int cpu)
+{
+ u32 action;
+
+ /* Load the ipi register to figure out what we're supposed to do */
+ action = xconf_readl(ipi_status_regs[cpu]);
+ /* Clear the ipi register to clear the interrupt */
+ xconf_writel(action, ipi_clear_regs[cpu]);
+
+ return action;
+}
+
+static void legacy_ipi_write_action(int cpu, u32 action)
+{
+ xconf_writel((u32)action, ipi_set_regs[cpu]);
+}
+
+static void ipi_method_init(void)
+{
+ if (cpu_has_csripi) {
+ ipi_read_clear = csr_ipi_read_clear;
+ ipi_write_action = csr_ipi_write_action;
+ } else {
+ ipi_read_clear = legacy_ipi_read_clear;
+ ipi_write_action = legacy_ipi_write_action;
+ }
+}
+
+static void ipi_regaddrs_init(void)
+{
+ int i, node, core;
+
+ for (i = 0; i < MAX_CPUS; i++) {
+ node = i / 4;
+ core = i % 4;
+ ipi_set_regs[i] = (void *)
+ (smp_group[node] + core_offsets[core] + SET);
+ ipi_clear_regs[i] = (void *)
+ (smp_group[node] + core_offsets[core] + CLEAR);
+ ipi_status_regs[i] = (void *)
+ (smp_group[node] + core_offsets[core] + STATUS);
+ ipi_en_regs[i] = (void *)
+ (smp_group[node] + core_offsets[core] + EN);
+ ipi_mailbox_buf[i] = (void *)
+ (smp_group[node] + core_offsets[core] + MBUF);
+ }
+}
+
+/*
+ * Simple enough, just poke the appropriate ipi register
+ */
+static void loongson3_send_ipi_single(int cpu, unsigned int action)
+{
+ ipi_write_action(cpu_logical_map(cpu), (u32)action);
+}
+
+static void
+loongson3_send_ipi_mask(const struct cpumask *mask, unsigned int action)
+{
+ unsigned int i;
+
+ for_each_cpu(i, mask)
+ ipi_write_action(cpu_logical_map(i), (u32)action);
+}
+
+irqreturn_t loongson3_ipi_interrupt(int irq, void *dev)
+{
+ unsigned int action;
+ unsigned int cpu = smp_processor_id();
+
+ action = ipi_read_clear(cpu_logical_map(cpu));
+
+ smp_mb();
+
+ if (action & SMP_RESCHEDULE) {
+ scheduler_ipi();
+ per_cpu(irq_stat, cpu).ipi_irqs[IPI_RESCHEDULE]++;
+ }
+
+ if (action & SMP_CALL_FUNCTION) {
+ generic_smp_call_function_interrupt();
+ per_cpu(irq_stat, cpu).ipi_irqs[IPI_CALL_FUNCTION]++;
+ }
+
+ return IRQ_HANDLED;
+}
+
+/*
+ * SMP init and finish on secondary CPUs
+ */
+static void loongson3_init_secondary(void)
+{
+ unsigned int cpu = smp_processor_id();
+ unsigned int imask = ECFGF_IP0 | ECFGF_IP1 | ECFGF_IP2 |
+ ECFGF_IPI | ECFGF_PMC | ECFGF_TIMER;
+
+ change_csr_ecfg(ECFG0_IM, imask);
+
+ if (cpu_has_csripi)
+ iocsr_writel(0xffffffff, LOONGARCH_IOCSR_IPI_EN);
+ else
+ xconf_writel(0xffffffff, ipi_en_regs[cpu_logical_map(cpu)]);
+
+ per_cpu(cpu_state, cpu) = CPU_ONLINE;
+ cpu_set_core(&cpu_data[cpu],
+ cpu_logical_map(cpu) % loongson_sysconf.cores_per_package);
+ cpu_set_cluster(&cpu_data[cpu],
+ cpu_logical_map(cpu) / loongson_sysconf.cores_per_package);
+ cpu_data[cpu].package =
+ cpu_logical_map(cpu) / loongson_sysconf.cores_per_package;
+}
+
+static void loongson3_smp_finish(void)
+{
+ int cpu = smp_processor_id();
+
+ local_irq_enable();
+
+ if (cpu_has_csripi)
+ iocsr_writeq(0, LOONGARCH_IOCSR_MBUF0);
+ else
+ xconf_writeq(0, ipi_mailbox_buf[cpu_logical_map(cpu)]+0x0);
+
+ pr_info("CPU#%d finished\n", smp_processor_id());
+}
+
+static void __init loongson3_smp_setup(void)
+{
+ ipi_method_init();
+ ipi_regaddrs_init();
+
+ if (cpu_has_csripi)
+ iocsr_writel(0xffffffff, LOONGARCH_IOCSR_IPI_EN);
+ else
+ xconf_writel(0xffffffff, ipi_en_regs[cpu_logical_map(0)]);
+
+ pr_info("Detected %i available CPU(s)\n", loongson_sysconf.nr_cpus);
+
+ cpu_set_core(&cpu_data[0],
+ cpu_logical_map(0) % loongson_sysconf.cores_per_package);
+ cpu_set_cluster(&cpu_data[0],
+ cpu_logical_map(0) / loongson_sysconf.cores_per_package);
+ cpu_data[0].package = cpu_logical_map(0) / loongson_sysconf.cores_per_package;
+}
+
+static void __init loongson3_prepare_cpus(unsigned int max_cpus)
+{
+ int i = 0;
+
+ for (i = 0; i < loongson_sysconf.nr_cpus; i++) {
+ set_cpu_present(i, true);
+
+ if (cpu_has_csripi)
+ csr_mail_send(0, __cpu_logical_map[i], 0);
+ else
+ xconf_writeq(0, ipi_mailbox_buf[__cpu_logical_map[i]]+0x0);
+ }
+
+ per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
+}
+
+/*
+ * Setup the PC, SP, and TP of a secondary processor and start it running!
+ */
+static int loongson3_boot_secondary(int cpu, struct task_struct *idle)
+{
+ unsigned long startargs[4];
+
+ pr_info("Booting CPU#%d...\n", cpu);
+
+ /* startargs[] are initial PC, SP and TP for secondary CPU */
+ startargs[0] = (unsigned long)&smpboot_entry;
+ startargs[1] = (unsigned long)__KSTK_TOS(idle);
+ startargs[2] = (unsigned long)task_thread_info(idle);
+ startargs[3] = 0;
+
+ pr_debug("CPU#%d, func_pc=%lx, sp=%lx, tp=%lx\n",
+ cpu, startargs[0], startargs[1], startargs[2]);
+
+ if (cpu_has_csripi) {
+ csr_mail_send(startargs[3], cpu_logical_map(cpu), 3);
+ csr_mail_send(startargs[2], cpu_logical_map(cpu), 2);
+ csr_mail_send(startargs[1], cpu_logical_map(cpu), 1);
+ csr_mail_send(startargs[0], cpu_logical_map(cpu), 0);
+ } else {
+ xconf_writeq(startargs[3], ipi_mailbox_buf[cpu_logical_map(cpu)]+0x18);
+ xconf_writeq(startargs[2], ipi_mailbox_buf[cpu_logical_map(cpu)]+0x10);
+ xconf_writeq(startargs[1], ipi_mailbox_buf[cpu_logical_map(cpu)]+0x8);
+ xconf_writeq(startargs[0], ipi_mailbox_buf[cpu_logical_map(cpu)]+0x0);
+ }
+ loongson3_send_ipi_single(cpu, SMP_BOOT_CPU);
+
+ return 0;
+}
+
+#ifdef CONFIG_HOTPLUG_CPU
+
+static bool io_master(int cpu)
+{
+ int i, node, master;
+
+ if (cpu == 0)
+ return true;
+
+ for (i = 1; i < loongson_sysconf.nr_io_pics; i++) {
+ node = eiointc_get_node(i);
+ master = cpumask_first(cpumask_of_node(node));
+ if (cpu == master)
+ return true;
+ }
+
+ return false;
+}
+
+static int loongson3_cpu_disable(void)
+{
+ unsigned long flags;
+ unsigned int cpu = smp_processor_id();
+
+ if (io_master(cpu))
+ return -EBUSY;
+
+ set_cpu_online(cpu, false);
+ calculate_cpu_foreign_map();
+ local_irq_save(flags);
+ fixup_irqs();
+ local_irq_restore(flags);
+ local_flush_tlb_all();
+
+ return 0;
+}
+
+static void loongson3_cpu_die(unsigned int cpu)
+{
+ while (per_cpu(cpu_state, cpu) != CPU_DEAD)
+ cpu_relax();
+
+ mb();
+}
+
+/* To shutdown a core in Loongson 3, the target core should go to XKPRANGE
+ * and flush all L1 entries at first. Then, another core (usually Core 0)
+ * can safely disable the clock of the target core. loongson3_play_dead()
+ * is called via XKPRANGE (uncached and unmmaped) */
+static void loongson3_play_dead(int *state_addr)
+{
+ register int val;
+ register long cpuid, core, node, count;
+ register void *addr, *base, *initfunc;
+
+ __asm__ __volatile__(
+ " li.d %[addr], 0x8000000000000000\n"
+ "1: cacop 0x8, %[addr], 0 \n" /* flush L1 ICache */
+ " cacop 0x8, %[addr], 1 \n"
+ " cacop 0x8, %[addr], 2 \n"
+ " cacop 0x8, %[addr], 3 \n"
+ " cacop 0x9, %[addr], 0 \n" /* flush L1 DCache */
+ " cacop 0x9, %[addr], 1 \n"
+ " cacop 0x9, %[addr], 2 \n"
+ " cacop 0x9, %[addr], 3 \n"
+ " addi.w %[sets], %[sets], -1 \n"
+ " addi.d %[addr], %[addr], 0x40 \n"
+ " bnez %[sets], 1b \n"
+ " li.d %[addr], 0x8000000000000000\n"
+ "2: cacop 0xa, %[addr], 0 \n" /* flush L1 VCache */
+ " cacop 0xa, %[addr], 1 \n"
+ " cacop 0xa, %[addr], 2 \n"
+ " cacop 0xa, %[addr], 3 \n"
+ " cacop 0xa, %[addr], 4 \n"
+ " cacop 0xa, %[addr], 5 \n"
+ " cacop 0xa, %[addr], 6 \n"
+ " cacop 0xa, %[addr], 7 \n"
+ " cacop 0xa, %[addr], 8 \n"
+ " cacop 0xa, %[addr], 9 \n"
+ " cacop 0xa, %[addr], 10 \n"
+ " cacop 0xa, %[addr], 11 \n"
+ " cacop 0xa, %[addr], 12 \n"
+ " cacop 0xa, %[addr], 13 \n"
+ " cacop 0xa, %[addr], 14 \n"
+ " cacop 0xa, %[addr], 15 \n"
+ " addi.w %[vsets], %[vsets], -1 \n"
+ " addi.d %[addr], %[addr], 0x40 \n"
+ " bnez %[vsets], 2b \n"
+ " li.w %[val], 0x7 \n" /* *state_addr = CPU_DEAD; */
+ " st.w %[val], %[state_addr], 0 \n"
+ " dbar 0 \n"
+ " cacop 0x11, %[state_addr], 0 \n" /* flush entry of *state_addr */
+ : [addr] "=&r" (addr), [val] "=&r" (val)
+ : [state_addr] "r" (state_addr),
+ [sets] "r" (cpu_data[smp_processor_id()].dcache.sets),
+ [vsets] "r" (cpu_data[smp_processor_id()].vcache.sets));
+
+ local_irq_enable();
+ change_csr_ecfg(ECFG0_IM, ECFGF_IPI);
+
+ __asm__ __volatile__(
+ " idle 0 \n"
+ " csrrd %[cpuid], 0x20 \n"
+ " andi %[cpuid], %[cpuid], 0x1ff \n"
+ " li.d %[base], 0x800000001fe01000 \n"
+ " andi %[core], %[cpuid], 0x3 \n"
+ " slli.w %[core], %[core], 8 \n" /* get core id */
+ " or %[base], %[base], %[core] \n"
+ " andi %[node], %[cpuid], 0x3c \n"
+ " slli.d %[node], %[node], 42 \n" /* get node id */
+ " or %[base], %[base], %[node] \n"
+ "1: li.w %[count], 0x100 \n" /* wait for init loop */
+ "2: addi.w %[count], %[count], -1 \n"
+ " bnez %[count], 2b \n" /* limit mailbox access */
+ " ld.w %[initfunc], %[base], 0x20 \n" /* check PC */
+ " beqz %[initfunc], 1b \n"
+ " ld.d $a1, %[base], 0x38 \n"
+ " ld.d $tp, %[base], 0x30 \n" /* get TP via mailbox */
+ " ld.d $sp, %[base], 0x28 \n" /* get SP via mailbox */
+ " ld.d %[initfunc], %[base], 0x20 \n" /* get PC via mailbox */
+ " jirl $zero, %[initfunc], 0 \n" /* jump to initial PC */
+ " nop \n"
+ : [core] "=&r" (core), [node] "=&r" (node),
+ [base] "=&r" (base), [cpuid] "=&r" (cpuid),
+ [count] "=&r" (count), [initfunc] "=&r" (initfunc)
+ : /* No Input */
+ : "a1");
+
+ unreachable();
+}
+
+void play_dead(void)
+{
+ int *state_addr;
+ unsigned int cpu = smp_processor_id();
+ void (*play_dead_uncached)(int *s);
+
+ idle_task_exit();
+ play_dead_uncached = (void *)TO_UNCAC(__pa((unsigned long)loongson3_play_dead));
+ state_addr = &per_cpu(cpu_state, cpu);
+ mb();
+ play_dead_uncached(state_addr);
+}
+
+static int loongson3_enable_clock(unsigned int cpu)
+{
+ uint64_t core_id = cpu_core(&cpu_data[cpu]);
+ uint64_t package_id = cpu_data[cpu].package;
+
+ LOONGSON_FREQCTRL(package_id) |= 1 << (core_id * 4 + 3);
+
+ return 0;
+}
+
+static int loongson3_disable_clock(unsigned int cpu)
+{
+ uint64_t core_id = cpu_core(&cpu_data[cpu]);
+ uint64_t package_id = cpu_data[cpu].package;
+
+ LOONGSON_FREQCTRL(package_id) &= ~(1 << (core_id * 4 + 3));
+
+ return 0;
+}
+
+static int register_loongson3_notifier(void)
+{
+ return cpuhp_setup_state_nocalls(CPUHP_LOONGARCH_SOC_PREPARE,
+ "loongarch/loongson:prepare",
+ loongson3_enable_clock,
+ loongson3_disable_clock);
+}
+early_initcall(register_loongson3_notifier);
+
+#endif
+
+const struct plat_smp_ops loongson3_smp_ops = {
+ .send_ipi_single = loongson3_send_ipi_single,
+ .send_ipi_mask = loongson3_send_ipi_mask,
+ .smp_setup = loongson3_smp_setup,
+ .prepare_cpus = loongson3_prepare_cpus,
+ .boot_secondary = loongson3_boot_secondary,
+ .init_secondary = loongson3_init_secondary,
+ .smp_finish = loongson3_smp_finish,
+#ifdef CONFIG_HOTPLUG_CPU
+ .cpu_disable = loongson3_cpu_disable,
+ .cpu_die = loongson3_cpu_die,
+#endif
+};
+
+const struct plat_smp_ops *mp_ops = &loongson3_smp_ops;
+EXPORT_SYMBOL(mp_ops);
+
+/*
+ * Power management
+ */
+#ifdef CONFIG_PM
+
+static int loongson3_ipi_suspend(void)
+{
+ return 0;
+}
+
+static void loongson3_ipi_resume(void)
+{
+ if (cpu_has_csripi)
+ iocsr_writel(0xffffffff, LOONGARCH_IOCSR_IPI_EN);
+ else
+ xconf_writel(0xffffffff, ipi_en_regs[cpu_logical_map(0)]);
+}
+
+static struct syscore_ops loongson3_ipi_syscore_ops = {
+ .resume = loongson3_ipi_resume,
+ .suspend = loongson3_ipi_suspend,
+};
+
+/*
+ * Enable boot cpu ipi before enabling nonboot cpus
+ * during syscore_resume.
+ */
+static int __init ipi_pm_init(void)
+{
+ register_syscore_ops(&loongson3_ipi_syscore_ops);
+ return 0;
+}
+
+core_initcall(ipi_pm_init);
+#endif
+
+static inline void set_cpu_sibling_map(int cpu)
+{
+ int i;
+
+ cpumask_set_cpu(cpu, &cpu_sibling_setup_map);
+
+ if (smp_num_siblings > 1) {
+ for_each_cpu(i, &cpu_sibling_setup_map) {
+ if (cpus_are_siblings(cpu, i)) {
+ cpumask_set_cpu(i, &cpu_sibling_map[cpu]);
+ cpumask_set_cpu(cpu, &cpu_sibling_map[i]);
+ }
+ }
+ } else
+ cpumask_set_cpu(cpu, &cpu_sibling_map[cpu]);
+}
+
+static inline void set_cpu_core_map(int cpu)
+{
+ int i;
+
+ cpumask_set_cpu(cpu, &cpu_core_setup_map);
+
+ for_each_cpu(i, &cpu_core_setup_map) {
+ if (cpu_data[cpu].package == cpu_data[i].package) {
+ cpumask_set_cpu(i, &cpu_core_map[cpu]);
+ cpumask_set_cpu(cpu, &cpu_core_map[i]);
+ }
+ }
+}
+
+/*
+ * Calculate a new cpu_foreign_map mask whenever a
+ * new cpu appears or disappears.
+ */
+void calculate_cpu_foreign_map(void)
+{
+ int i, k, core_present;
+ cpumask_t temp_foreign_map;
+
+ /* Re-calculate the mask */
+ cpumask_clear(&temp_foreign_map);
+ for_each_online_cpu(i) {
+ core_present = 0;
+ for_each_cpu(k, &temp_foreign_map)
+ if (cpus_are_siblings(i, k))
+ core_present = 1;
+ if (!core_present)
+ cpumask_set_cpu(i, &temp_foreign_map);
+ }
+
+ for_each_online_cpu(i)
+ cpumask_andnot(&cpu_foreign_map[i],
+ &temp_foreign_map, &cpu_sibling_map[i]);
+}
+
+/*
+ * First C code run on the secondary CPUs after being started up by
+ * the master.
+ */
+asmlinkage void start_secondary(void)
+{
+ unsigned int cpu;
+
+ sync_counter();
+ cpu = smp_processor_id();
+ set_my_cpu_offset(per_cpu_offset(cpu));
+
+ cpu_probe();
+ constant_clockevent_init();
+ mp_ops->init_secondary();
+
+ notify_cpu_starting(cpu);
+
+ /* Notify boot CPU that we're starting */
+ complete(&cpu_starting);
+
+ /* The CPU is running, now mark it online */
+ set_cpu_online(cpu, true);
+
+ set_cpu_sibling_map(cpu);
+ set_cpu_core_map(cpu);
+
+ calculate_cpu_foreign_map();
+
+ /*
+ * Notify boot CPU that we're up & online and it can safely return
+ * from __cpu_up
+ */
+ complete(&cpu_running);
+
+ /*
+ * irq will be enabled in ->smp_finish(), enabling it too early
+ * is dangerous.
+ */
+ WARN_ON_ONCE(!irqs_disabled());
+ mp_ops->smp_finish();
+
+ cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
+}
+
+static void stop_this_cpu(void *dummy)
+{
+ set_cpu_online(smp_processor_id(), false);
+ calculate_cpu_foreign_map();
+ local_irq_disable();
+ while (true);
+}
+
+void smp_send_stop(void)
+{
+ smp_call_function(stop_this_cpu, NULL, 0);
+}
+
+void __init smp_cpus_done(unsigned int max_cpus)
+{
+}
+
+/* called from main before smp_init() */
+void __init smp_prepare_cpus(unsigned int max_cpus)
+{
+ init_new_context(current, &init_mm);
+ current_thread_info()->cpu = 0;
+ mp_ops->prepare_cpus(max_cpus);
+ set_cpu_sibling_map(0);
+ set_cpu_core_map(0);
+ calculate_cpu_foreign_map();
+#ifndef CONFIG_HOTPLUG_CPU
+ init_cpu_present(cpu_possible_mask);
+#endif
+}
+
+/* Preload SMP state for boot cpu */
+void smp_prepare_boot_cpu(void)
+{
+ unsigned int cpu;
+
+ set_cpu_possible(0, true);
+ set_cpu_online(0, true);
+ set_my_cpu_offset(per_cpu_offset(0));
+
+ for_each_possible_cpu(cpu)
+ set_cpu_numa_node(cpu, 0);
+}
+
+int __cpu_up(unsigned int cpu, struct task_struct *tidle)
+{
+ int err;
+
+ err = mp_ops->boot_secondary(cpu, tidle);
+ if (err)
+ return err;
+
+ /* Wait for CPU to start and be ready to sync counters */
+ if (!wait_for_completion_timeout(&cpu_starting,
+ msecs_to_jiffies(1000))) {
+ pr_crit("CPU%u: failed to start\n", cpu);
+ return -EIO;
+ }
+
+ /* Wait for CPU to finish startup & mark itself online before return */
+ wait_for_completion(&cpu_running);
+ return 0;
+}
+
+int setup_profiling_timer(unsigned int multiplier)
+{
+ return 0;
+}
+
+static void flush_tlb_all_ipi(void *info)
+{
+ local_flush_tlb_all();
+}
+
+void flush_tlb_all(void)
+{
+ on_each_cpu(flush_tlb_all_ipi, NULL, 1);
+}
+
+static void flush_tlb_mm_ipi(void *mm)
+{
+ local_flush_tlb_mm((struct mm_struct *)mm);
+}
+
+void flush_tlb_mm(struct mm_struct *mm)
+{
+ preempt_disable();
+
+ if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
+ on_each_cpu_mask(mm_cpumask(mm), flush_tlb_mm_ipi, mm, 1);
+ } else {
+ unsigned int cpu;
+
+ for_each_online_cpu(cpu) {
+ if (cpu != smp_processor_id() && cpu_context(cpu, mm))
+ cpu_context(cpu, mm) = 0;
+ }
+ local_flush_tlb_mm(mm);
+ }
+
+ preempt_enable();
+}
+
+struct flush_tlb_data {
+ struct vm_area_struct *vma;
+ unsigned long addr1;
+ unsigned long addr2;
+};
+
+static void flush_tlb_range_ipi(void *info)
+{
+ struct flush_tlb_data *fd = info;
+
+ local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2);
+}
+
+void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
+{
+ struct mm_struct *mm = vma->vm_mm;
+
+ preempt_disable();
+ if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
+ struct flush_tlb_data fd = {
+ .vma = vma,
+ .addr1 = start,
+ .addr2 = end,
+ };
+
+ on_each_cpu_mask(mm_cpumask(mm), flush_tlb_range_ipi, &fd, 1);
+ } else {
+ unsigned int cpu;
+ int exec = vma->vm_flags & VM_EXEC;
+
+ for_each_online_cpu(cpu) {
+ /*
+ * flush_cache_range() will only fully flush icache if
+ * the VMA is executable, otherwise we must invalidate
+ * ASID without it appearing to has_valid_asid() as if
+ * mm has been completely unused by that CPU.
+ */
+ if (cpu != smp_processor_id() && cpu_context(cpu, mm))
+ cpu_context(cpu, mm) = !exec;
+ }
+ local_flush_tlb_range(vma, start, end);
+ }
+ preempt_enable();
+}
+
+static void flush_tlb_kernel_range_ipi(void *info)
+{
+ struct flush_tlb_data *fd = info;
+
+ local_flush_tlb_kernel_range(fd->addr1, fd->addr2);
+}
+
+void flush_tlb_kernel_range(unsigned long start, unsigned long end)
+{
+ struct flush_tlb_data fd = {
+ .addr1 = start,
+ .addr2 = end,
+ };
+
+ on_each_cpu(flush_tlb_kernel_range_ipi, &fd, 1);
+}
+
+static void flush_tlb_page_ipi(void *info)
+{
+ struct flush_tlb_data *fd = info;
+
+ local_flush_tlb_page(fd->vma, fd->addr1);
+}
+
+void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
+{
+ preempt_disable();
+ if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) {
+ struct flush_tlb_data fd = {
+ .vma = vma,
+ .addr1 = page,
+ };
+
+ on_each_cpu_mask(mm_cpumask(vma->vm_mm), flush_tlb_page_ipi, &fd, 1);
+ } else {
+ unsigned int cpu;
+
+ for_each_online_cpu(cpu) {
+ /*
+ * flush_cache_page() only does partial flushes, so
+ * invalidate ASID without it appearing to
+ * has_valid_asid() as if mm has been completely unused
+ * by that CPU.
+ */
+ if (cpu != smp_processor_id() && cpu_context(cpu, vma->vm_mm))
+ cpu_context(cpu, vma->vm_mm) = 1;
+ }
+ local_flush_tlb_page(vma, page);
+ }
+ preempt_enable();
+}
+EXPORT_SYMBOL(flush_tlb_page);
+
+static void flush_tlb_one_ipi(void *info)
+{
+ unsigned long vaddr = (unsigned long) info;
+
+ local_flush_tlb_one(vaddr);
+}
+
+void flush_tlb_one(unsigned long vaddr)
+{
+ on_each_cpu(flush_tlb_one_ipi, (void *)vaddr, 1);
+}
+EXPORT_SYMBOL(flush_tlb_one);
diff --git a/arch/loongarch/kernel/topology.c b/arch/loongarch/kernel/topology.c
index 3b2cbb95875b..ab1a75c0b5a6 100644
--- a/arch/loongarch/kernel/topology.c
+++ b/arch/loongarch/kernel/topology.c
@@ -1,13 +1,52 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/cpu.h>
+#include <linux/cpumask.h>
#include <linux/init.h>
+#include <linux/node.h>
+#include <linux/nodemask.h>
#include <linux/percpu.h>
-static struct cpu cpu_device;
+static DEFINE_PER_CPU(struct cpu, cpu_devices);
+
+#ifdef CONFIG_HOTPLUG_CPU
+int arch_register_cpu(int cpu)
+{
+ int ret;
+ struct cpu *c = &per_cpu(cpu_devices, cpu);
+
+ c->hotpluggable = 1;
+ ret = register_cpu(c, cpu);
+ if (ret < 0)
+ pr_warn("register_cpu %d failed (%d)\n", cpu, ret);
+
+ return ret;
+}
+EXPORT_SYMBOL(arch_register_cpu);
+
+void arch_unregister_cpu(int cpu)
+{
+ struct cpu *c = &per_cpu(cpu_devices, cpu);
+
+ c->hotpluggable = 0;
+ unregister_cpu(c);
+}
+EXPORT_SYMBOL(arch_unregister_cpu);
+#endif
static int __init topology_init(void)
{
- return register_cpu(&cpu_device, 0);
+ int i, ret;
+
+ for_each_present_cpu(i) {
+ struct cpu *c = &per_cpu(cpu_devices, i);
+
+ c->hotpluggable = !!i;
+ ret = register_cpu(c, i);
+ if (ret < 0)
+ pr_warn("topology_init: register_cpu %d failed (%d)\n", i, ret);
+ }
+
+ return 0;
}
subsys_initcall(topology_init);
diff --git a/arch/loongarch/kernel/vmlinux.lds.S b/arch/loongarch/kernel/vmlinux.lds.S
index 3ae212100de5..9a919f512d06 100644
--- a/arch/loongarch/kernel/vmlinux.lds.S
+++ b/arch/loongarch/kernel/vmlinux.lds.S
@@ -70,6 +70,9 @@ SECTIONS
.exit.data : {
EXIT_DATA
}
+#ifdef CONFIG_SMP
+ PERCPU_SECTION(1 << CONFIG_L1_CACHE_SHIFT)
+#endif
/*
* Align to 64K in attempt to eliminate holes before the
diff --git a/arch/loongarch/mm/tlbex.S b/arch/loongarch/mm/tlbex.S
index 6ff03662811a..e05df448ddbb 100644
--- a/arch/loongarch/mm/tlbex.S
+++ b/arch/loongarch/mm/tlbex.S
@@ -82,7 +82,14 @@ vmalloc_done_load:
slli.d t0, t0, _PTE_T_LOG2
add.d t1, ra, t0
+#ifdef CONFIG_SMP
+smp_pgtable_change_load:
+#endif
+#ifdef CONFIG_SMP
+ ll.d t0, t1, 0
+#else
ld.d t0, t1, 0
+#endif
tlbsrch
srli.d ra, t0, _PAGE_PRESENT_SHIFT
@@ -90,7 +97,12 @@ vmalloc_done_load:
beq ra, $r0, nopage_tlb_load
ori t0, t0, _PAGE_VALID
+#ifdef CONFIG_SMP
+ sc.d t0, t1, 0
+ beq t0, $r0, smp_pgtable_change_load
+#else
st.d t0, t1, 0
+#endif
ori t1, t1, 8
xori t1, t1, 8
ld.d t0, t1, 0
@@ -114,14 +126,24 @@ vmalloc_load:
* spots a huge page.
*/
tlb_huge_update_load:
+#ifdef CONFIG_SMP
+ ll.d t0, t1, 0
+#else
ld.d t0, t1, 0
+#endif
srli.d ra, t0, _PAGE_PRESENT_SHIFT
andi ra, ra, 1
beq ra, $r0, nopage_tlb_load
tlbsrch
ori t0, t0, _PAGE_VALID
+#ifdef CONFIG_SMP
+ sc.d t0, t1, 0
+ beq t0, $r0, tlb_huge_update_load
+ ld.d t0, t1, 0
+#else
st.d t0, t1, 0
+#endif
addu16i.d t1, $r0, -(CSR_TLBIDX_EHINV >> 16)
addi.d ra, t1, 0
csrxchg ra, t1, LOONGARCH_CSR_TLBIDX
@@ -167,6 +189,7 @@ tlb_huge_update_load:
csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
nopage_tlb_load:
+ dbar 0
csrrd ra, EXCEPTION_KS2
la.abs t0, tlb_do_page_fault_0
jirl $r0, t0, 0
@@ -223,7 +246,14 @@ vmalloc_done_store:
slli.d t0, t0, _PTE_T_LOG2
add.d t1, ra, t0
+#ifdef CONFIG_SMP
+smp_pgtable_change_store:
+#endif
+#ifdef CONFIG_SMP
+ ll.d t0, t1, 0
+#else
ld.d t0, t1, 0
+#endif
tlbsrch
srli.d ra, t0, _PAGE_PRESENT_SHIFT
@@ -232,7 +262,12 @@ vmalloc_done_store:
bne ra, $r0, nopage_tlb_store
ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
+#ifdef CONFIG_SMP
+ sc.d t0, t1, 0
+ beq t0, $r0, smp_pgtable_change_store
+#else
st.d t0, t1, 0
+#endif
ori t1, t1, 8
xori t1, t1, 8
@@ -257,7 +292,11 @@ vmalloc_store:
* spots a huge page.
*/
tlb_huge_update_store:
+#ifdef CONFIG_SMP
+ ll.d t0, t1, 0
+#else
ld.d t0, t1, 0
+#endif
srli.d ra, t0, _PAGE_PRESENT_SHIFT
andi ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT)
xori ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT)
@@ -266,7 +305,13 @@ tlb_huge_update_store:
tlbsrch
ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
+#ifdef CONFIG_SMP
+ sc.d t0, t1, 0
+ beq t0, $r0, tlb_huge_update_store
+ ld.d t0, t1, 0
+#else
st.d t0, t1, 0
+#endif
addu16i.d t1, $r0, -(CSR_TLBIDX_EHINV >> 16)
addi.d ra, t1, 0
csrxchg ra, t1, LOONGARCH_CSR_TLBIDX
@@ -312,6 +357,7 @@ tlb_huge_update_store:
csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
nopage_tlb_store:
+ dbar 0
csrrd ra, EXCEPTION_KS2
la.abs t0, tlb_do_page_fault_1
jirl $r0, t0, 0
@@ -367,7 +413,14 @@ vmalloc_done_modify:
slli.d t0, t0, _PTE_T_LOG2
add.d t1, ra, t0
+#ifdef CONFIG_SMP
+smp_pgtable_change_modify:
+#endif
+#ifdef CONFIG_SMP
+ ll.d t0, t1, 0
+#else
ld.d t0, t1, 0
+#endif
tlbsrch
srli.d ra, t0, _PAGE_WRITE_SHIFT
@@ -375,7 +428,12 @@ vmalloc_done_modify:
beq ra, $r0, nopage_tlb_modify
ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
+#ifdef CONFIG_SMP
+ sc.d t0, t1, 0
+ beq t0, $r0, smp_pgtable_change_modify
+#else
st.d t0, t1, 0
+#endif
ori t1, t1, 8
xori t1, t1, 8
ld.d t0, t1, 0
@@ -399,7 +457,11 @@ vmalloc_modify:
* build_tlbchange_handler_head spots a huge page.
*/
tlb_huge_update_modify:
+#ifdef CONFIG_SMP
+ ll.d t0, t1, 0
+#else
ld.d t0, t1, 0
+#endif
srli.d ra, t0, _PAGE_WRITE_SHIFT
andi ra, ra, 1
@@ -408,7 +470,13 @@ tlb_huge_update_modify:
tlbsrch
ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
+#ifdef CONFIG_SMP
+ sc.d t0, t1, 0
+ beq t0, $r0, tlb_huge_update_modify
+ ld.d t0, t1, 0
+#else
st.d t0, t1, 0
+#endif
/*
* A huge PTE describes an area the size of the
* configured huge page size. This is twice the
@@ -448,6 +516,7 @@ tlb_huge_update_modify:
csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
nopage_tlb_modify:
+ dbar 0
csrrd ra, EXCEPTION_KS2
la.abs t0, tlb_do_page_fault_1
jirl $r0, t0, 0
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 541d02f05352..0624995cba9d 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -126,6 +126,7 @@ enum cpuhp_state {
CPUHP_ZCOMP_PREPARE,
CPUHP_TIMERS_PREPARE,
CPUHP_MIPS_SOC_PREPARE,
+ CPUHP_LOONGARCH_SOC_PREPARE,
CPUHP_BP_PREPARE_DYN,
CPUHP_BP_PREPARE_DYN_END = CPUHP_BP_PREPARE_DYN + 20,
CPUHP_BRINGUP_CPU,
--
2.27.0
On Fri, Sep 17, 2021 at 5:57 AM Huacai Chen <[email protected]> wrote:
> +
> +struct task_struct;
> +
> +struct plat_smp_ops {
> + void (*send_ipi_single)(int cpu, unsigned int action);
> + void (*send_ipi_mask)(const struct cpumask *mask, unsigned int action);
> + void (*smp_setup)(void);
> + void (*prepare_cpus)(unsigned int max_cpus);
> + int (*boot_secondary)(int cpu, struct task_struct *idle);
> + void (*init_secondary)(void);
> + void (*smp_finish)(void);
> +#ifdef CONFIG_HOTPLUG_CPU
> + int (*cpu_disable)(void);
> + void (*cpu_die)(unsigned int cpu);
> +#endif
> +};
Do you foresee having more than one implementation of these in the
near future? If not, I would suggest leaving out the extra indirection
and just using direct function calls.
> +#ifdef CONFIG_SMP
> +
> +static inline void plat_smp_setup(void)
> +{
> + extern const struct plat_smp_ops *mp_ops; /* private */
> +
> + mp_ops->smp_setup();
> +}
> +
> +#else /* !CONFIG_SMP */
> +
> +static inline void plat_smp_setup(void) { }
> +
> +#endif /* !CONFIG_SMP */
You could even go further and do what arch/arm64 has, making
SMP support unconditional. This obviously depends on hardware
roadmaps, but if all harfdware you support has multiple cores,
then non-SMP mode just adds complexity.
> +
> +#define MAX_CPUS 64
You CONFIG_NR_CPUS allows up to 256. I think you need to
adjust one of the numbers to match the other, or remove this
definition and just use CONFIG_NR_CPUS directly.
> +
> +static volatile void *ipi_set_regs[MAX_CPUS];
> +static volatile void *ipi_clear_regs[MAX_CPUS];
> +static volatile void *ipi_status_regs[MAX_CPUS];
> +static volatile void *ipi_en_regs[MAX_CPUS];
> +static volatile void *ipi_mailbox_buf[MAX_CPUS];
Why are these 'volatile'? If they are MMIO registers, they
should be __iomem, and accessed using readl()/writel()
etc
Arnd
Hi, Arnd,
On Fri, Sep 17, 2021 at 5:57 PM Arnd Bergmann <[email protected]> wrote:
>
> On Fri, Sep 17, 2021 at 5:57 AM Huacai Chen <[email protected]> wrote:
>
> > +
> > +struct task_struct;
> > +
> > +struct plat_smp_ops {
> > + void (*send_ipi_single)(int cpu, unsigned int action);
> > + void (*send_ipi_mask)(const struct cpumask *mask, unsigned int action);
> > + void (*smp_setup)(void);
> > + void (*prepare_cpus)(unsigned int max_cpus);
> > + int (*boot_secondary)(int cpu, struct task_struct *idle);
> > + void (*init_secondary)(void);
> > + void (*smp_finish)(void);
> > +#ifdef CONFIG_HOTPLUG_CPU
> > + int (*cpu_disable)(void);
> > + void (*cpu_die)(unsigned int cpu);
> > +#endif
> > +};
>
>
> Do you foresee having more than one implementation of these in the
> near future? If not, I would suggest leaving out the extra indirection
> and just using direct function calls.
OK, let me rethink this, if it is still needed, I will tell you why.
>
> > +#ifdef CONFIG_SMP
> > +
> > +static inline void plat_smp_setup(void)
> > +{
> > + extern const struct plat_smp_ops *mp_ops; /* private */
> > +
> > + mp_ops->smp_setup();
> > +}
> > +
> > +#else /* !CONFIG_SMP */
> > +
> > +static inline void plat_smp_setup(void) { }
> > +
> > +#endif /* !CONFIG_SMP */
>
> You could even go further and do what arch/arm64 has, making
> SMP support unconditional. This obviously depends on hardware
> roadmaps, but if all harfdware you support has multiple cores,
> then non-SMP mode just adds complexity.
As mentioned in another patch, we do have some MCU hardware (no FP, no
SMP, and even no MMU).
>
> > +
> > +#define MAX_CPUS 64
>
> You CONFIG_NR_CPUS allows up to 256. I think you need to
> adjust one of the numbers to match the other, or remove this
> definition and just use CONFIG_NR_CPUS directly.
Legacy IPI method only supports at most 64 CPUs (limited by the MMIO
register space). Maybe we can remove the whole legacy method support,
then we can remove MAX_CPUS, too.
>
> > +
> > +static volatile void *ipi_set_regs[MAX_CPUS];
> > +static volatile void *ipi_clear_regs[MAX_CPUS];
> > +static volatile void *ipi_status_regs[MAX_CPUS];
> > +static volatile void *ipi_en_regs[MAX_CPUS];
> > +static volatile void *ipi_mailbox_buf[MAX_CPUS];
>
> Why are these 'volatile'? If they are MMIO registers, they
> should be __iomem, and accessed using readl()/writel()
> etc
Yes, they are MMIO registers and __iomem is needed.
Huacai
>
> Arnd