Hi,
This series of patches does a first attempt at smp integration (I call it first,
although I don't believe it's too far away from a real one). The series comprises 52 patches,
and the full diffstat says in the end:
34 files changed, 1858 insertions(+), 2344 deletions(-)
smp_32.c and smp_64.c are completely gone. smp.h and smpboot_{32,64}.c are not, although
substantially reduced. I know that ideally, no leftovers are wanted, but the series were
already big enough (I tried to split the patches as much as I could), so I considered this
a good cutting point. If this is all agreed upon and merged, I'll start looking at the other
bits that are left.
All patches are tested is various random x86 subarchitectures, and all of them works.
The result also boots fine in mine x86_64 and i386 boxes.
Comments are welcome.
arch/x86/kernel/Makefile | 11
arch/x86/kernel/apic_32.c | 7
arch/x86/kernel/smp_32.c | 730 ---------------------------------
arch/x86/kernel/smp_64.c | 674 +++---------------------------
arch/x86/kernel/smpboot.c | 372 ++++++++++++++++
arch/x86/kernel/smpboot_32.c | 363 ----------------
arch/x86/kernel/smpboot_64.c | 432 -------------------
arch/x86/mach-voyager/voyager_smp.c | 3
b/arch/x86/kernel/Makefile | 2
b/arch/x86/kernel/apic_32.c | 74 +--
b/arch/x86/kernel/ipi.c | 178 ++++++++
b/arch/x86/kernel/mpparse_32.c | 7
b/arch/x86/kernel/mpparse_64.c | 7
b/arch/x86/kernel/smp.c | 253 +++++++++++
b/arch/x86/kernel/smp_32.c | 2
b/arch/x86/kernel/smp_64.c | 2
b/arch/x86/kernel/smpboot.c | 53 ++
b/arch/x86/kernel/smpboot_32.c | 6
b/arch/x86/kernel/smpboot_64.c | 2
b/arch/x86/kernel/smpcommon.c | 83 +++
b/arch/x86/kernel/smpcommon_32.c | 82 ---
b/arch/x86/kernel/tlb_32.c | 243 ++++++++++
b/arch/x86/kernel/tlb_64.c | 274 ++++++++++++
b/arch/x86/mach-voyager/voyager_smp.c | 9
b/include/asm-x86/processor.h | 10
b/include/asm-x86/smp.h | 8
b/include/asm-x86/smp_32.h | 4
b/include/asm-x86/smp_64.h | 5
include/asm-x86/processor.h | 5
include/asm-x86/smp.h | 99 ++++
include/asm-x86/smp_32.h | 86 ---
include/asm-x86/smp_64.h | 21
linux-2.6-x86/arch/x86/kernel/Makefile | 4
linux-2.6-x86/arch/x86/kernel/smp.c | 91 ++++
34 files changed, 1858 insertions(+), 2344 deletions(-)
move extern function definitions that are the same between smp_{32,64}.h
to smp.h
Signed-off-by: Glauber Costa <[email protected]>
---
include/asm-x86/smp.h | 3 +++
include/asm-x86/smp_32.h | 4 ----
include/asm-x86/smp_64.h | 4 ----
3 files changed, 3 insertions(+), 8 deletions(-)
diff --git a/include/asm-x86/smp.h b/include/asm-x86/smp.h
index f250d1c..ad7b99d 100644
--- a/include/asm-x86/smp.h
+++ b/include/asm-x86/smp.h
@@ -8,5 +8,8 @@
# include "smp_64.h"
#endif
+extern void smp_alloc_memory(void);
+extern void lock_ipi_call_lock(void);
+extern void unlock_ipi_call_lock(void);
#endif /* __ASSEMBLY__ */
#endif
diff --git a/include/asm-x86/smp_32.h b/include/asm-x86/smp_32.h
index 56152e3..2781225 100644
--- a/include/asm-x86/smp_32.h
+++ b/include/asm-x86/smp_32.h
@@ -22,10 +22,6 @@ extern cpumask_t cpu_callin_map;
extern int smp_num_siblings;
extern unsigned int num_processors;
-extern void smp_alloc_memory(void);
-extern void lock_ipi_call_lock(void);
-extern void unlock_ipi_call_lock(void);
-
extern void (*mtrr_hook) (void);
extern void zap_low_mappings (void);
diff --git a/include/asm-x86/smp_64.h b/include/asm-x86/smp_64.h
index e0a7551..2c21df2 100644
--- a/include/asm-x86/smp_64.h
+++ b/include/asm-x86/smp_64.h
@@ -19,10 +19,6 @@ extern cpumask_t cpu_initialized;
extern int smp_num_siblings;
extern unsigned int num_processors;
-extern void smp_alloc_memory(void);
-extern void lock_ipi_call_lock(void);
-extern void unlock_ipi_call_lock(void);
-
extern int smp_call_function_mask(cpumask_t mask, void (*func)(void *),
void *info, int wait);
--
1.5.0.6
this is the first step of integrating smp.h between x86_64
and i386
Signed-off-by: Glauber Costa <[email protected]>
---
include/asm-x86/smp.h | 7 +++++++
1 files changed, 7 insertions(+), 0 deletions(-)
diff --git a/include/asm-x86/smp.h b/include/asm-x86/smp.h
index f2e8319..f250d1c 100644
--- a/include/asm-x86/smp.h
+++ b/include/asm-x86/smp.h
@@ -1,5 +1,12 @@
+#ifndef _ASM_X86_SMP_H_
+#define _ASM_X86_SMP_H_
+#ifndef __ASSEMBLY__
+
#ifdef CONFIG_X86_32
# include "smp_32.h"
#else
# include "smp_64.h"
#endif
+
+#endif /* __ASSEMBLY__ */
+#endif
--
1.5.0.6
x86_64 will benefit from it
Signed-off-by: Glauber Costa <[email protected]>
---
include/asm-x86/smp.h | 14 ++++++++++++++
include/asm-x86/smp_32.h | 14 --------------
2 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/include/asm-x86/smp.h b/include/asm-x86/smp.h
index c130a87..d11b92b 100644
--- a/include/asm-x86/smp.h
+++ b/include/asm-x86/smp.h
@@ -8,6 +8,20 @@ extern cpumask_t cpu_callout_map;
extern int smp_num_siblings;
extern unsigned int num_processors;
+struct smp_ops {
+ void (*smp_prepare_boot_cpu)(void);
+ void (*smp_prepare_cpus)(unsigned max_cpus);
+ int (*cpu_up)(unsigned cpu);
+ void (*smp_cpus_done)(unsigned max_cpus);
+
+ void (*smp_send_stop)(void);
+ void (*smp_send_reschedule)(int cpu);
+ int (*smp_call_function_mask)(cpumask_t mask,
+ void (*func)(void *info), void *info,
+ int wait);
+};
+
+
#ifdef CONFIG_X86_32
# include "smp_32.h"
#else
diff --git a/include/asm-x86/smp_32.h b/include/asm-x86/smp_32.h
index 9a4057d..72faad6 100644
--- a/include/asm-x86/smp_32.h
+++ b/include/asm-x86/smp_32.h
@@ -38,20 +38,6 @@ extern void remove_siblinginfo(int cpu);
/* Globals due to paravirt */
extern void set_cpu_sibling_map(int cpu);
-struct smp_ops
-{
- void (*smp_prepare_boot_cpu)(void);
- void (*smp_prepare_cpus)(unsigned max_cpus);
- int (*cpu_up)(unsigned cpu);
- void (*smp_cpus_done)(unsigned max_cpus);
-
- void (*smp_send_stop)(void);
- void (*smp_send_reschedule)(int cpu);
- int (*smp_call_function_mask)(cpumask_t mask,
- void (*func)(void *info), void *info,
- int wait);
-};
-
#ifdef CONFIG_SMP
extern struct smp_ops smp_ops;
--
1.5.0.6
the smp_ops symbol is temporarily defined in smp_64.c, but it will soon
be unified
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smp_64.c | 2 ++
include/asm-x86/smp.h | 3 +++
include/asm-x86/smp_32.h | 2 --
3 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/arch/x86/kernel/smp_64.c b/arch/x86/kernel/smp_64.c
index 2fd74b0..80dba12 100644
--- a/arch/x86/kernel/smp_64.c
+++ b/arch/x86/kernel/smp_64.c
@@ -528,3 +528,5 @@ asmlinkage void smp_call_function_interrupt(void)
}
}
+struct smp_ops smp_ops;
+EXPORT_SYMBOL_GPL(smp_ops);
diff --git a/include/asm-x86/smp.h b/include/asm-x86/smp.h
index d11b92b..ee98bee 100644
--- a/include/asm-x86/smp.h
+++ b/include/asm-x86/smp.h
@@ -21,6 +21,9 @@ struct smp_ops {
int wait);
};
+#ifdef CONFIG_SMP
+extern struct smp_ops smp_ops;
+#endif
#ifdef CONFIG_X86_32
# include "smp_32.h"
diff --git a/include/asm-x86/smp_32.h b/include/asm-x86/smp_32.h
index 72faad6..74755e8 100644
--- a/include/asm-x86/smp_32.h
+++ b/include/asm-x86/smp_32.h
@@ -39,8 +39,6 @@ extern void remove_siblinginfo(int cpu);
extern void set_cpu_sibling_map(int cpu);
#ifdef CONFIG_SMP
-extern struct smp_ops smp_ops;
-
static inline void smp_prepare_boot_cpu(void)
{
smp_ops.smp_prepare_boot_cpu();
--
1.5.0.6
definition is moved to common header, x86_64 function name
now is native_smp_call_function_mask
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smp_64.c | 7 ++++---
include/asm-x86/smp.h | 7 +++++++
include/asm-x86/smp_32.h | 6 ------
3 files changed, 11 insertions(+), 9 deletions(-)
diff --git a/arch/x86/kernel/smp_64.c b/arch/x86/kernel/smp_64.c
index fd18161..225b765 100644
--- a/arch/x86/kernel/smp_64.c
+++ b/arch/x86/kernel/smp_64.c
@@ -386,9 +386,9 @@ static int __smp_call_function_mask(cpumask_t mask,
* You must not call this function with disabled interrupts or from a
* hardware interrupt handler or from a bottom half handler.
*/
-int smp_call_function_mask(cpumask_t mask,
- void (*func)(void *), void *info,
- int wait)
+int native_smp_call_function_mask(cpumask_t mask,
+ void (*func)(void *), void *info,
+ int wait)
{
int ret;
@@ -531,5 +531,6 @@ asmlinkage void smp_call_function_interrupt(void)
struct smp_ops smp_ops = {
.smp_send_reschedule = native_smp_send_reschedule,
+ .smp_call_function_mask = native_smp_call_function_mask,
};
EXPORT_SYMBOL_GPL(smp_ops);
diff --git a/include/asm-x86/smp.h b/include/asm-x86/smp.h
index 28f33c0..d9782f4 100644
--- a/include/asm-x86/smp.h
+++ b/include/asm-x86/smp.h
@@ -28,6 +28,13 @@ static inline void smp_send_reschedule(int cpu)
{
smp_ops.smp_send_reschedule(cpu);
}
+
+static inline int smp_call_function_mask(cpumask_t mask,
+ void (*func) (void *info), void *info,
+ int wait)
+{
+ return smp_ops.smp_call_function_mask(mask, func, info, wait);
+}
#endif
#ifdef CONFIG_X86_32
diff --git a/include/asm-x86/smp_32.h b/include/asm-x86/smp_32.h
index c60a3dd..d9337ee 100644
--- a/include/asm-x86/smp_32.h
+++ b/include/asm-x86/smp_32.h
@@ -60,12 +60,6 @@ static inline void smp_send_stop(void)
{
smp_ops.smp_send_stop();
}
-static inline int smp_call_function_mask(cpumask_t mask,
- void (*func) (void *info), void *info,
- int wait)
-{
- return smp_ops.smp_call_function_mask(mask, func, info, wait);
-}
void native_smp_prepare_boot_cpu(void);
void native_smp_prepare_cpus(unsigned int max_cpus);
--
1.5.0.6
move extern definitions that are the same between smp_{32,64}.h
to smp.h
Signed-off-by: Glauber Costa <[email protected]>
---
include/asm-x86/smp.h | 6 ++++++
include/asm-x86/smp_32.h | 4 ----
include/asm-x86/smp_64.h | 4 ----
3 files changed, 6 insertions(+), 8 deletions(-)
diff --git a/include/asm-x86/smp.h b/include/asm-x86/smp.h
index ad7b99d..c130a87 100644
--- a/include/asm-x86/smp.h
+++ b/include/asm-x86/smp.h
@@ -1,6 +1,12 @@
#ifndef _ASM_X86_SMP_H_
#define _ASM_X86_SMP_H_
#ifndef __ASSEMBLY__
+#include <linux/cpumask.h>
+
+extern cpumask_t cpu_callout_map;
+
+extern int smp_num_siblings;
+extern unsigned int num_processors;
#ifdef CONFIG_X86_32
# include "smp_32.h"
diff --git a/include/asm-x86/smp_32.h b/include/asm-x86/smp_32.h
index 2781225..9a4057d 100644
--- a/include/asm-x86/smp_32.h
+++ b/include/asm-x86/smp_32.h
@@ -16,12 +16,8 @@
# endif
#endif
-extern cpumask_t cpu_callout_map;
extern cpumask_t cpu_callin_map;
-extern int smp_num_siblings;
-extern unsigned int num_processors;
-
extern void (*mtrr_hook) (void);
extern void zap_low_mappings (void);
diff --git a/include/asm-x86/smp_64.h b/include/asm-x86/smp_64.h
index 2c21df2..284f701 100644
--- a/include/asm-x86/smp_64.h
+++ b/include/asm-x86/smp_64.h
@@ -13,12 +13,8 @@
#include <asm/pda.h>
#include <asm/thread_info.h>
-extern cpumask_t cpu_callout_map;
extern cpumask_t cpu_initialized;
-extern int smp_num_siblings;
-extern unsigned int num_processors;
-
extern int smp_call_function_mask(cpumask_t mask, void (*func)(void *),
void *info, int wait);
--
1.5.0.6
function definition is moved to common header, x86_64 version is now called
native_smp_send_reschedule
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smp_64.c | 7 +++++--
include/asm-x86/smp.h | 5 +++++
include/asm-x86/smp_32.h | 4 ----
include/asm-x86/smp_64.h | 2 --
4 files changed, 10 insertions(+), 8 deletions(-)
diff --git a/arch/x86/kernel/smp_64.c b/arch/x86/kernel/smp_64.c
index 80dba12..fd18161 100644
--- a/arch/x86/kernel/smp_64.c
+++ b/arch/x86/kernel/smp_64.c
@@ -290,8 +290,9 @@ void flush_tlb_all(void)
* anything. Worst case is that we lose a reschedule ...
*/
-void smp_send_reschedule(int cpu)
+static void native_smp_send_reschedule(int cpu)
{
+ WARN_ON(cpu_is_offline(cpu));
send_IPI_mask(cpumask_of_cpu(cpu), RESCHEDULE_VECTOR);
}
@@ -528,5 +529,7 @@ asmlinkage void smp_call_function_interrupt(void)
}
}
-struct smp_ops smp_ops;
+struct smp_ops smp_ops = {
+ .smp_send_reschedule = native_smp_send_reschedule,
+};
EXPORT_SYMBOL_GPL(smp_ops);
diff --git a/include/asm-x86/smp.h b/include/asm-x86/smp.h
index ee98bee..28f33c0 100644
--- a/include/asm-x86/smp.h
+++ b/include/asm-x86/smp.h
@@ -23,6 +23,11 @@ struct smp_ops {
#ifdef CONFIG_SMP
extern struct smp_ops smp_ops;
+
+static inline void smp_send_reschedule(int cpu)
+{
+ smp_ops.smp_send_reschedule(cpu);
+}
#endif
#ifdef CONFIG_X86_32
diff --git a/include/asm-x86/smp_32.h b/include/asm-x86/smp_32.h
index 74755e8..c60a3dd 100644
--- a/include/asm-x86/smp_32.h
+++ b/include/asm-x86/smp_32.h
@@ -60,10 +60,6 @@ static inline void smp_send_stop(void)
{
smp_ops.smp_send_stop();
}
-static inline void smp_send_reschedule(int cpu)
-{
- smp_ops.smp_send_reschedule(cpu);
-}
static inline int smp_call_function_mask(cpumask_t mask,
void (*func) (void *info), void *info,
int wait)
diff --git a/include/asm-x86/smp_64.h b/include/asm-x86/smp_64.h
index 284f701..b920458 100644
--- a/include/asm-x86/smp_64.h
+++ b/include/asm-x86/smp_64.h
@@ -65,8 +65,6 @@ static inline int num_booting_cpus(void)
return cpus_weight(cpu_callout_map);
}
-extern void smp_send_reschedule(int cpu);
-
#else /* CONFIG_SMP */
extern unsigned int boot_cpu_id;
--
1.5.0.6
function definition is moved to common header. x86_64 version
is now called native_cpu_up
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smp_64.c | 1 +
arch/x86/kernel/smpboot_64.c | 2 +-
include/asm-x86/smp.h | 7 +++++++
include/asm-x86/smp_32.h | 5 -----
4 files changed, 9 insertions(+), 6 deletions(-)
diff --git a/arch/x86/kernel/smp_64.c b/arch/x86/kernel/smp_64.c
index 225b765..7cc20a3 100644
--- a/arch/x86/kernel/smp_64.c
+++ b/arch/x86/kernel/smp_64.c
@@ -532,5 +532,6 @@ asmlinkage void smp_call_function_interrupt(void)
struct smp_ops smp_ops = {
.smp_send_reschedule = native_smp_send_reschedule,
.smp_call_function_mask = native_smp_call_function_mask,
+ .cpu_up = native_cpu_up,
};
EXPORT_SYMBOL_GPL(smp_ops);
diff --git a/arch/x86/kernel/smpboot_64.c b/arch/x86/kernel/smpboot_64.c
index 0880f2c..e381fe7 100644
--- a/arch/x86/kernel/smpboot_64.c
+++ b/arch/x86/kernel/smpboot_64.c
@@ -929,7 +929,7 @@ void __init smp_prepare_boot_cpu(void)
/*
* Entry point to boot a CPU.
*/
-int __cpuinit __cpu_up(unsigned int cpu)
+int __cpuinit native_cpu_up(unsigned int cpu)
{
int apicid = cpu_present_to_apicid(cpu);
unsigned long flags;
diff --git a/include/asm-x86/smp.h b/include/asm-x86/smp.h
index d9782f4..7dd7141 100644
--- a/include/asm-x86/smp.h
+++ b/include/asm-x86/smp.h
@@ -24,6 +24,11 @@ struct smp_ops {
#ifdef CONFIG_SMP
extern struct smp_ops smp_ops;
+static inline int __cpu_up(unsigned int cpu)
+{
+ return smp_ops.cpu_up(cpu);
+}
+
static inline void smp_send_reschedule(int cpu)
{
smp_ops.smp_send_reschedule(cpu);
@@ -35,6 +40,8 @@ static inline int smp_call_function_mask(cpumask_t mask,
{
return smp_ops.smp_call_function_mask(mask, func, info, wait);
}
+
+int native_cpu_up(unsigned int cpunum);
#endif
#ifdef CONFIG_X86_32
diff --git a/include/asm-x86/smp_32.h b/include/asm-x86/smp_32.h
index d9337ee..a7fab8e 100644
--- a/include/asm-x86/smp_32.h
+++ b/include/asm-x86/smp_32.h
@@ -47,10 +47,6 @@ static inline void smp_prepare_cpus(unsigned int max_cpus)
{
smp_ops.smp_prepare_cpus(max_cpus);
}
-static inline int __cpu_up(unsigned int cpu)
-{
- return smp_ops.cpu_up(cpu);
-}
static inline void smp_cpus_done(unsigned int max_cpus)
{
smp_ops.smp_cpus_done(max_cpus);
@@ -63,7 +59,6 @@ static inline void smp_send_stop(void)
void native_smp_prepare_boot_cpu(void);
void native_smp_prepare_cpus(unsigned int max_cpus);
-int native_cpu_up(unsigned int cpunum);
void native_smp_cpus_done(unsigned int max_cpus);
#ifndef CONFIG_PARAVIRT
--
1.5.0.6
definition is moved to common header. x86_64 version is now called
native_prepare_boot_cpu
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smp_64.c | 1 +
arch/x86/kernel/smpboot_64.c | 2 +-
include/asm-x86/smp.h | 6 ++++++
include/asm-x86/smp_32.h | 5 -----
4 files changed, 8 insertions(+), 6 deletions(-)
diff --git a/arch/x86/kernel/smp_64.c b/arch/x86/kernel/smp_64.c
index 7cc20a3..05116c1 100644
--- a/arch/x86/kernel/smp_64.c
+++ b/arch/x86/kernel/smp_64.c
@@ -530,6 +530,7 @@ asmlinkage void smp_call_function_interrupt(void)
}
struct smp_ops smp_ops = {
+ .smp_prepare_boot_cpu = native_smp_prepare_boot_cpu,
.smp_send_reschedule = native_smp_send_reschedule,
.smp_call_function_mask = native_smp_call_function_mask,
.cpu_up = native_cpu_up,
diff --git a/arch/x86/kernel/smpboot_64.c b/arch/x86/kernel/smpboot_64.c
index e381fe7..47e654c 100644
--- a/arch/x86/kernel/smpboot_64.c
+++ b/arch/x86/kernel/smpboot_64.c
@@ -918,7 +918,7 @@ void __init smp_prepare_cpus(unsigned int max_cpus)
/*
* Early setup to make printk work.
*/
-void __init smp_prepare_boot_cpu(void)
+void __init native_smp_prepare_boot_cpu(void)
{
int me = smp_processor_id();
/* already set me in cpu_online_map in boot_cpu_init() */
diff --git a/include/asm-x86/smp.h b/include/asm-x86/smp.h
index 7dd7141..a2d69a1 100644
--- a/include/asm-x86/smp.h
+++ b/include/asm-x86/smp.h
@@ -24,6 +24,11 @@ struct smp_ops {
#ifdef CONFIG_SMP
extern struct smp_ops smp_ops;
+static inline void smp_prepare_boot_cpu(void)
+{
+ smp_ops.smp_prepare_boot_cpu();
+}
+
static inline int __cpu_up(unsigned int cpu)
{
return smp_ops.cpu_up(cpu);
@@ -41,6 +46,7 @@ static inline int smp_call_function_mask(cpumask_t mask,
return smp_ops.smp_call_function_mask(mask, func, info, wait);
}
+void native_smp_prepare_boot_cpu(void);
int native_cpu_up(unsigned int cpunum);
#endif
diff --git a/include/asm-x86/smp_32.h b/include/asm-x86/smp_32.h
index a7fab8e..2c7ecfa 100644
--- a/include/asm-x86/smp_32.h
+++ b/include/asm-x86/smp_32.h
@@ -39,10 +39,6 @@ extern void remove_siblinginfo(int cpu);
extern void set_cpu_sibling_map(int cpu);
#ifdef CONFIG_SMP
-static inline void smp_prepare_boot_cpu(void)
-{
- smp_ops.smp_prepare_boot_cpu();
-}
static inline void smp_prepare_cpus(unsigned int max_cpus)
{
smp_ops.smp_prepare_cpus(max_cpus);
@@ -57,7 +53,6 @@ static inline void smp_send_stop(void)
smp_ops.smp_send_stop();
}
-void native_smp_prepare_boot_cpu(void);
void native_smp_prepare_cpus(unsigned int max_cpus);
void native_smp_cpus_done(unsigned int max_cpus);
--
1.5.0.6
definition is moved to common header. x86_64 version is now called
native_smp_prepare_cpus
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smp_64.c | 1 +
arch/x86/kernel/smpboot_64.c | 2 +-
include/asm-x86/smp.h | 6 ++++++
include/asm-x86/smp_32.h | 5 -----
4 files changed, 8 insertions(+), 6 deletions(-)
diff --git a/arch/x86/kernel/smp_64.c b/arch/x86/kernel/smp_64.c
index 05116c1..c520374 100644
--- a/arch/x86/kernel/smp_64.c
+++ b/arch/x86/kernel/smp_64.c
@@ -531,6 +531,7 @@ asmlinkage void smp_call_function_interrupt(void)
struct smp_ops smp_ops = {
.smp_prepare_boot_cpu = native_smp_prepare_boot_cpu,
+ .smp_prepare_cpus = native_smp_prepare_cpus,
.smp_send_reschedule = native_smp_send_reschedule,
.smp_call_function_mask = native_smp_call_function_mask,
.cpu_up = native_cpu_up,
diff --git a/arch/x86/kernel/smpboot_64.c b/arch/x86/kernel/smpboot_64.c
index 47e654c..b106983 100644
--- a/arch/x86/kernel/smpboot_64.c
+++ b/arch/x86/kernel/smpboot_64.c
@@ -867,7 +867,7 @@ static void __init smp_cpu_index_default(void)
* Prepare for SMP bootup. The MP table or ACPI has been read
* earlier. Just do some sanity checking here and enable APIC mode.
*/
-void __init smp_prepare_cpus(unsigned int max_cpus)
+void __init native_smp_prepare_cpus(unsigned int max_cpus)
{
nmi_watchdog_default();
smp_cpu_index_default();
diff --git a/include/asm-x86/smp.h b/include/asm-x86/smp.h
index a2d69a1..31bd99d 100644
--- a/include/asm-x86/smp.h
+++ b/include/asm-x86/smp.h
@@ -29,6 +29,11 @@ static inline void smp_prepare_boot_cpu(void)
smp_ops.smp_prepare_boot_cpu();
}
+static inline void smp_prepare_cpus(unsigned int max_cpus)
+{
+ smp_ops.smp_prepare_cpus(max_cpus);
+}
+
static inline int __cpu_up(unsigned int cpu)
{
return smp_ops.cpu_up(cpu);
@@ -47,6 +52,7 @@ static inline int smp_call_function_mask(cpumask_t mask,
}
void native_smp_prepare_boot_cpu(void);
+void native_smp_prepare_cpus(unsigned int max_cpus);
int native_cpu_up(unsigned int cpunum);
#endif
diff --git a/include/asm-x86/smp_32.h b/include/asm-x86/smp_32.h
index 2c7ecfa..5078538 100644
--- a/include/asm-x86/smp_32.h
+++ b/include/asm-x86/smp_32.h
@@ -39,10 +39,6 @@ extern void remove_siblinginfo(int cpu);
extern void set_cpu_sibling_map(int cpu);
#ifdef CONFIG_SMP
-static inline void smp_prepare_cpus(unsigned int max_cpus)
-{
- smp_ops.smp_prepare_cpus(max_cpus);
-}
static inline void smp_cpus_done(unsigned int max_cpus)
{
smp_ops.smp_cpus_done(max_cpus);
@@ -53,7 +49,6 @@ static inline void smp_send_stop(void)
smp_ops.smp_send_stop();
}
-void native_smp_prepare_cpus(unsigned int max_cpus);
void native_smp_cpus_done(unsigned int max_cpus);
#ifndef CONFIG_PARAVIRT
--
1.5.0.6
with this removal, exports for both i386 and x86_64,
regarding the "smp_call_function" series are now the same.
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smp_64.c | 1 -
1 files changed, 0 insertions(+), 1 deletions(-)
diff --git a/arch/x86/kernel/smp_64.c b/arch/x86/kernel/smp_64.c
index 275101a..a434f6c 100644
--- a/arch/x86/kernel/smp_64.c
+++ b/arch/x86/kernel/smp_64.c
@@ -400,7 +400,6 @@ int native_smp_call_function_mask(cpumask_t mask,
spin_unlock(&call_lock);
return ret;
}
-EXPORT_SYMBOL(smp_call_function_mask);
/*
* smp_call_function_single - Run a function on a specific CPU
--
1.5.0.6
definition is moved to common header. x86_64 version is now called
native_smp_cpus_done
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smp_64.c | 2 ++
arch/x86/kernel/smpboot_64.c | 2 +-
include/asm-x86/smp.h | 6 ++++++
include/asm-x86/smp_32.h | 7 -------
4 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/arch/x86/kernel/smp_64.c b/arch/x86/kernel/smp_64.c
index c520374..275101a 100644
--- a/arch/x86/kernel/smp_64.c
+++ b/arch/x86/kernel/smp_64.c
@@ -532,6 +532,8 @@ asmlinkage void smp_call_function_interrupt(void)
struct smp_ops smp_ops = {
.smp_prepare_boot_cpu = native_smp_prepare_boot_cpu,
.smp_prepare_cpus = native_smp_prepare_cpus,
+ .smp_cpus_done = native_smp_cpus_done,
+
.smp_send_reschedule = native_smp_send_reschedule,
.smp_call_function_mask = native_smp_call_function_mask,
.cpu_up = native_cpu_up,
diff --git a/arch/x86/kernel/smpboot_64.c b/arch/x86/kernel/smpboot_64.c
index b106983..fd0d3a9 100644
--- a/arch/x86/kernel/smpboot_64.c
+++ b/arch/x86/kernel/smpboot_64.c
@@ -987,7 +987,7 @@ int __cpuinit native_cpu_up(unsigned int cpu)
/*
* Finish the SMP boot.
*/
-void __init smp_cpus_done(unsigned int max_cpus)
+void __init native_smp_cpus_done(unsigned int max_cpus)
{
smp_cleanup_boot();
setup_ioapic_dest();
diff --git a/include/asm-x86/smp.h b/include/asm-x86/smp.h
index 31bd99d..9620165 100644
--- a/include/asm-x86/smp.h
+++ b/include/asm-x86/smp.h
@@ -34,6 +34,11 @@ static inline void smp_prepare_cpus(unsigned int max_cpus)
smp_ops.smp_prepare_cpus(max_cpus);
}
+static inline void smp_cpus_done(unsigned int max_cpus)
+{
+ smp_ops.smp_cpus_done(max_cpus);
+}
+
static inline int __cpu_up(unsigned int cpu)
{
return smp_ops.cpu_up(cpu);
@@ -53,6 +58,7 @@ static inline int smp_call_function_mask(cpumask_t mask,
void native_smp_prepare_boot_cpu(void);
void native_smp_prepare_cpus(unsigned int max_cpus);
+void native_smp_cpus_done(unsigned int max_cpus);
int native_cpu_up(unsigned int cpunum);
#endif
diff --git a/include/asm-x86/smp_32.h b/include/asm-x86/smp_32.h
index 5078538..bc90a4e 100644
--- a/include/asm-x86/smp_32.h
+++ b/include/asm-x86/smp_32.h
@@ -39,18 +39,11 @@ extern void remove_siblinginfo(int cpu);
extern void set_cpu_sibling_map(int cpu);
#ifdef CONFIG_SMP
-static inline void smp_cpus_done(unsigned int max_cpus)
-{
- smp_ops.smp_cpus_done(max_cpus);
-}
-
static inline void smp_send_stop(void)
{
smp_ops.smp_send_stop();
}
-void native_smp_cpus_done(unsigned int max_cpus);
-
#ifndef CONFIG_PARAVIRT
#define startup_ipi_hook(phys_apicid, start_eip, start_esp) do { } while (0)
#endif
--
1.5.0.6
disabled_cpus is (up to now) a x86_64-only contruction.
But it's extern declaration can be moved to common header anyway
Signed-off-by: Glauber Costa <[email protected]>
---
include/asm-x86/smp.h | 3 +++
include/asm-x86/smp_64.h | 1 -
2 files changed, 3 insertions(+), 1 deletions(-)
diff --git a/include/asm-x86/smp.h b/include/asm-x86/smp.h
index 9620165..be8a511 100644
--- a/include/asm-x86/smp.h
+++ b/include/asm-x86/smp.h
@@ -2,6 +2,7 @@
#define _ASM_X86_SMP_H_
#ifndef __ASSEMBLY__
#include <linux/cpumask.h>
+#include <linux/init.h>
extern cpumask_t cpu_callout_map;
@@ -60,6 +61,8 @@ void native_smp_prepare_boot_cpu(void);
void native_smp_prepare_cpus(unsigned int max_cpus);
void native_smp_cpus_done(unsigned int max_cpus);
int native_cpu_up(unsigned int cpunum);
+
+extern unsigned disabled_cpus;
#endif
#ifdef CONFIG_X86_32
diff --git a/include/asm-x86/smp_64.h b/include/asm-x86/smp_64.h
index b920458..c7a00ca 100644
--- a/include/asm-x86/smp_64.h
+++ b/include/asm-x86/smp_64.h
@@ -44,7 +44,6 @@ static inline int cpu_present_to_apicid(int mps_cpu)
extern int __cpu_disable(void);
extern void __cpu_die(unsigned int cpu);
extern void prefill_possible_map(void);
-extern unsigned __cpuinitdata disabled_cpus;
#define raw_smp_processor_id() read_pda(cpunumber)
#define cpu_physical_id(cpu) per_cpu(x86_cpu_to_apicid, cpu)
--
1.5.0.6
this patch allows a cpu to be marked as present but disabled in i386,
just as x86_64 currently does.
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/mpparse_32.c | 6 +++++-
1 files changed, 5 insertions(+), 1 deletions(-)
diff --git a/arch/x86/kernel/mpparse_32.c b/arch/x86/kernel/mpparse_32.c
index f349e68..b2aded3 100644
--- a/arch/x86/kernel/mpparse_32.c
+++ b/arch/x86/kernel/mpparse_32.c
@@ -70,6 +70,8 @@ unsigned int boot_cpu_physical_apicid = -1U;
/* Internal processor count */
unsigned int num_processors;
+unsigned disabled_cpus __cpuinitdata;
+
/* Bitmask of physically existing CPUs */
physid_mask_t phys_cpu_present_map;
@@ -108,8 +110,10 @@ static void __cpuinit MP_processor_info (struct mpc_config_processor *m)
int ver, apicid;
physid_mask_t phys_cpu;
- if (!(m->mpc_cpuflag & CPU_ENABLED))
+ if (!(m->mpc_cpuflag & CPU_ENABLED)) {
+ disabled_cpus++;
return;
+ }
apicid = mpc_apic_id(m, translation_table[mpc_record]);
--
1.5.0.6
this patches moves prefill_possible_map() to smpboot.c
Right now it is x86_64-specific, but nothing intrinsically
prevents it to be used by i386
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/Makefile | 2 +-
arch/x86/kernel/smpboot.c | 53 ++++++++++++++++++++++++++++++++++++++++++
arch/x86/kernel/smpboot_64.c | 51 ----------------------------------------
include/asm-x86/smp.h | 1 +
include/asm-x86/smp_64.h | 1 -
5 files changed, 55 insertions(+), 53 deletions(-)
create mode 100644 arch/x86/kernel/smpboot.c
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index df10327..4c68bfc 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -46,7 +46,7 @@ obj-$(CONFIG_MICROCODE) += microcode.o
obj-$(CONFIG_PCI) += early-quirks.o
apm-y := apm_32.o
obj-$(CONFIG_APM) += apm.o
-obj-$(CONFIG_X86_SMP) += smp_$(BITS).o smpboot_$(BITS).o tsc_sync.o
+obj-$(CONFIG_X86_SMP) += smp_$(BITS).o smpboot_$(BITS).o smpboot.o tsc_sync.o
obj-$(CONFIG_X86_32_SMP) += smpcommon_32.o
obj-$(CONFIG_X86_64_SMP) += smp_64.o smpboot_64.o tsc_sync.o
obj-$(CONFIG_X86_TRAMPOLINE) += trampoline_$(BITS).o
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
new file mode 100644
index 0000000..bffe108
--- /dev/null
+++ b/arch/x86/kernel/smpboot.c
@@ -0,0 +1,53 @@
+#include <linux/init.h>
+#include <linux/smp.h>
+
+#ifdef CONFIG_HOTPLUG_CPU
+
+int additional_cpus __initdata = -1;
+
+static __init int setup_additional_cpus(char *s)
+{
+ return s && get_option(&s, &additional_cpus) ? 0 : -EINVAL;
+}
+early_param("additional_cpus", setup_additional_cpus);
+
+/*
+ * cpu_possible_map should be static, it cannot change as cpu's
+ * are onlined, or offlined. The reason is per-cpu data-structures
+ * are allocated by some modules at init time, and dont expect to
+ * do this dynamically on cpu arrival/departure.
+ * cpu_present_map on the other hand can change dynamically.
+ * In case when cpu_hotplug is not compiled, then we resort to current
+ * behaviour, which is cpu_possible == cpu_present.
+ * - Ashok Raj
+ *
+ * Three ways to find out the number of additional hotplug CPUs:
+ * - If the BIOS specified disabled CPUs in ACPI/mptables use that.
+ * - The user can overwrite it with additional_cpus=NUM
+ * - Otherwise don't reserve additional CPUs.
+ * We do this because additional CPUs waste a lot of memory.
+ * -AK
+ */
+__init void prefill_possible_map(void)
+{
+ int i;
+ int possible;
+
+ if (additional_cpus == -1) {
+ if (disabled_cpus > 0)
+ additional_cpus = disabled_cpus;
+ else
+ additional_cpus = 0;
+ }
+ possible = num_processors + additional_cpus;
+ if (possible > NR_CPUS)
+ possible = NR_CPUS;
+
+ printk(KERN_INFO "SMP: Allowing %d CPUs, %d hotplug CPUs\n",
+ possible, max_t(int, possible - num_processors, 0));
+
+ for (i = 0; i < possible; i++)
+ cpu_set(i, cpu_possible_map);
+}
+#endif
+
diff --git a/arch/x86/kernel/smpboot_64.c b/arch/x86/kernel/smpboot_64.c
index fd0d3a9..953b0ff 100644
--- a/arch/x86/kernel/smpboot_64.c
+++ b/arch/x86/kernel/smpboot_64.c
@@ -749,51 +749,6 @@ static __init void disable_smp(void)
cpu_set(0, per_cpu(cpu_core_map, 0));
}
-#ifdef CONFIG_HOTPLUG_CPU
-
-int additional_cpus __initdata = -1;
-
-/*
- * cpu_possible_map should be static, it cannot change as cpu's
- * are onlined, or offlined. The reason is per-cpu data-structures
- * are allocated by some modules at init time, and dont expect to
- * do this dynamically on cpu arrival/departure.
- * cpu_present_map on the other hand can change dynamically.
- * In case when cpu_hotplug is not compiled, then we resort to current
- * behaviour, which is cpu_possible == cpu_present.
- * - Ashok Raj
- *
- * Three ways to find out the number of additional hotplug CPUs:
- * - If the BIOS specified disabled CPUs in ACPI/mptables use that.
- * - The user can overwrite it with additional_cpus=NUM
- * - Otherwise don't reserve additional CPUs.
- * We do this because additional CPUs waste a lot of memory.
- * -AK
- */
-__init void prefill_possible_map(void)
-{
- int i;
- int possible;
-
- if (additional_cpus == -1) {
- if (disabled_cpus > 0)
- additional_cpus = disabled_cpus;
- else
- additional_cpus = 0;
- }
- possible = num_processors + additional_cpus;
- if (possible > NR_CPUS)
- possible = NR_CPUS;
-
- printk(KERN_INFO "SMP: Allowing %d CPUs, %d hotplug CPUs\n",
- possible,
- max_t(int, possible - num_processors, 0));
-
- for (i = 0; i < possible; i++)
- cpu_set(i, cpu_possible_map);
-}
-#endif
-
/*
* Various sanity checks.
*/
@@ -1087,12 +1042,6 @@ void __cpu_die(unsigned int cpu)
printk(KERN_ERR "CPU %u didn't die...\n", cpu);
}
-static __init int setup_additional_cpus(char *s)
-{
- return s && get_option(&s, &additional_cpus) ? 0 : -EINVAL;
-}
-early_param("additional_cpus", setup_additional_cpus);
-
#else /* ... !CONFIG_HOTPLUG_CPU */
int __cpu_disable(void)
diff --git a/include/asm-x86/smp.h b/include/asm-x86/smp.h
index be8a511..28cb1f8 100644
--- a/include/asm-x86/smp.h
+++ b/include/asm-x86/smp.h
@@ -63,6 +63,7 @@ void native_smp_cpus_done(unsigned int max_cpus);
int native_cpu_up(unsigned int cpunum);
extern unsigned disabled_cpus;
+extern void prefill_possible_map(void);
#endif
#ifdef CONFIG_X86_32
diff --git a/include/asm-x86/smp_64.h b/include/asm-x86/smp_64.h
index c7a00ca..e5bc1be 100644
--- a/include/asm-x86/smp_64.h
+++ b/include/asm-x86/smp_64.h
@@ -43,7 +43,6 @@ static inline int cpu_present_to_apicid(int mps_cpu)
extern int __cpu_disable(void);
extern void __cpu_die(unsigned int cpu);
-extern void prefill_possible_map(void);
#define raw_smp_processor_id() read_pda(cpunumber)
#define cpu_physical_id(cpu) per_cpu(x86_cpu_to_apicid, cpu)
--
1.5.0.6
there's already a warning in the topmost function
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smp_64.c | 3 ---
1 files changed, 0 insertions(+), 3 deletions(-)
diff --git a/arch/x86/kernel/smp_64.c b/arch/x86/kernel/smp_64.c
index a434f6c..b040224 100644
--- a/arch/x86/kernel/smp_64.c
+++ b/arch/x86/kernel/smp_64.c
@@ -420,9 +420,6 @@ int smp_call_function_single (int cpu, void (*func) (void *info), void *info,
/* prevent preemption and reschedule on another processor */
int ret, me = get_cpu();
- /* Can deadlock when called with interrupts disabled */
- WARN_ON(irqs_disabled());
-
if (cpu == me) {
local_irq_disable();
func(info);
--
1.5.0.6
with the hlt_works change, it is possible to have i386
and x86_64 stop_this_cpu() looking exactly the same. They
can, after that, be merged.
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smp_32.c | 2 +-
arch/x86/kernel/smp_64.c | 5 +++--
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/arch/x86/kernel/smp_32.c b/arch/x86/kernel/smp_32.c
index e4a6b66..cde3a0e 100644
--- a/arch/x86/kernel/smp_32.c
+++ b/arch/x86/kernel/smp_32.c
@@ -611,7 +611,7 @@ static void stop_this_cpu (void * dummy)
*/
cpu_clear(smp_processor_id(), cpu_online_map);
disable_local_APIC();
- if (cpu_data(smp_processor_id()).hlt_works_ok)
+ if (hlt_works(smp_processor_id()))
for(;;) halt();
for (;;);
}
diff --git a/arch/x86/kernel/smp_64.c b/arch/x86/kernel/smp_64.c
index e4494e8..4e1e2bc 100644
--- a/arch/x86/kernel/smp_64.c
+++ b/arch/x86/kernel/smp_64.c
@@ -416,8 +416,9 @@ static void stop_this_cpu(void *dummy)
*/
cpu_clear(smp_processor_id(), cpu_online_map);
disable_local_APIC();
- for (;;)
- halt();
+ if (hlt_works(smp_processor_id()))
+ for (;;) halt();
+ for (;;);
}
void smp_send_stop(void)
--
1.5.0.6
the two versions (the inner version, and the outer version, that takes
the locks) of smp_call_function_mask are made into one. With the changes,
i386 and x86_64 versions look exactly the same.
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smp_32.c | 2 +-
arch/x86/kernel/smp_64.c | 57 +++++++++++++---------------------------------
2 files changed, 17 insertions(+), 42 deletions(-)
diff --git a/arch/x86/kernel/smp_32.c b/arch/x86/kernel/smp_32.c
index dc0cde9..e4a6b66 100644
--- a/arch/x86/kernel/smp_32.c
+++ b/arch/x86/kernel/smp_32.c
@@ -583,7 +583,7 @@ native_smp_call_function_mask(cpumask_t mask,
atomic_set(&data.finished, 0);
call_data = &data;
- mb();
+ wmb();
/* Send a message to other CPUs */
if (cpus_equal(mask, allbutself))
diff --git a/arch/x86/kernel/smp_64.c b/arch/x86/kernel/smp_64.c
index aa2edb7..e4494e8 100644
--- a/arch/x86/kernel/smp_64.c
+++ b/arch/x86/kernel/smp_64.c
@@ -354,26 +354,30 @@ static void __smp_call_function(void (*func) (void *info), void *info,
}
-/*
- * this function sends a 'generic call function' IPI to all other CPU
- * of the system defined in the mask.
- */
-static int __smp_call_function_mask(cpumask_t mask,
- void (*func)(void *), void *info,
- int wait)
+int native_smp_call_function_mask(cpumask_t mask,
+ void (*func)(void *), void *info,
+ int wait)
{
struct call_data_struct data;
cpumask_t allbutself;
int cpus;
+ /* Can deadlock when called with interrupts disabled */
+ WARN_ON(irqs_disabled());
+
+ /* Holding any lock stops cpus from going down. */
+ spin_lock(&call_lock);
+
allbutself = cpu_online_map;
cpu_clear(smp_processor_id(), allbutself);
cpus_and(mask, mask, allbutself);
cpus = cpus_weight(mask);
- if (!cpus)
+ if (!cpus) {
+ spin_unlock(&call_lock);
return 0;
+ }
data.func = func;
data.info = info;
@@ -395,43 +399,14 @@ static int __smp_call_function_mask(cpumask_t mask,
while (atomic_read(&data.started) != cpus)
cpu_relax();
- if (!wait)
- return 0;
+ if (wait)
+ while (atomic_read(&data.finished) != cpus)
+ cpu_relax();
- while (atomic_read(&data.finished) != cpus)
- cpu_relax();
+ spin_unlock(&call_lock);
return 0;
}
-/**
- * smp_call_function_mask(): Run a function on a set of other CPUs.
- * @mask: The set of cpus to run on. Must not include the current cpu.
- * @func: The function to run. This must be fast and non-blocking.
- * @info: An arbitrary pointer to pass to the function.
- * @wait: If true, wait (atomically) until function has completed on other CPUs.
- *
- * Returns 0 on success, else a negative status code.
- *
- * If @wait is true, then returns once @func has returned; otherwise
- * it returns just before the target cpu calls @func.
- *
- * You must not call this function with disabled interrupts or from a
- * hardware interrupt handler or from a bottom half handler.
- */
-int native_smp_call_function_mask(cpumask_t mask,
- void (*func)(void *), void *info,
- int wait)
-{
- int ret;
-
- /* Can deadlock when called with interrupts disabled */
- WARN_ON(irqs_disabled());
-
- spin_lock(&call_lock);
- ret = __smp_call_function_mask(mask, func, info, wait);
- spin_unlock(&call_lock);
- return ret;
-}
static void stop_this_cpu(void *dummy)
{
--
1.5.0.6
This patch creates smpcommon.c with functions that are
equal between architectures. The i386-only init_gdt
is ifdef'd.
Note that smpcommon.o figures twice in the Makefile:
this is because sub-architectures like voyager that does
not use the normal smp_$(BITS) files also have to access them
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/Makefile | 4 +-
arch/x86/kernel/smp_64.c | 56 ---------------------------
arch/x86/kernel/smpcommon.c | 83 ++++++++++++++++++++++++++++++++++++++++
arch/x86/kernel/smpcommon_32.c | 81 ---------------------------------------
4 files changed, 85 insertions(+), 139 deletions(-)
create mode 100644 arch/x86/kernel/smpcommon.c
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index 4c68bfc..018d04d 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -47,8 +47,8 @@ obj-$(CONFIG_PCI) += early-quirks.o
apm-y := apm_32.o
obj-$(CONFIG_APM) += apm.o
obj-$(CONFIG_X86_SMP) += smp_$(BITS).o smpboot_$(BITS).o smpboot.o tsc_sync.o
-obj-$(CONFIG_X86_32_SMP) += smpcommon_32.o
-obj-$(CONFIG_X86_64_SMP) += smp_64.o smpboot_64.o tsc_sync.o
+obj-$(CONFIG_X86_32_SMP) += smpcommon.o
+obj-$(CONFIG_X86_64_SMP) += smp_64.o smpboot_64.o tsc_sync.o smpcommon.o
obj-$(CONFIG_X86_TRAMPOLINE) += trampoline_$(BITS).o
obj-$(CONFIG_X86_MPPARSE) += mpparse_$(BITS).o
obj-$(CONFIG_X86_LOCAL_APIC) += apic_$(BITS).o nmi_$(BITS).o
diff --git a/arch/x86/kernel/smp_64.c b/arch/x86/kernel/smp_64.c
index b040224..1d8b863 100644
--- a/arch/x86/kernel/smp_64.c
+++ b/arch/x86/kernel/smp_64.c
@@ -401,62 +401,6 @@ int native_smp_call_function_mask(cpumask_t mask,
return ret;
}
-/*
- * smp_call_function_single - Run a function on a specific CPU
- * @func: The function to run. This must be fast and non-blocking.
- * @info: An arbitrary pointer to pass to the function.
- * @nonatomic: Currently unused.
- * @wait: If true, wait until function has completed on other CPUs.
- *
- * Retrurns 0 on success, else a negative status code.
- *
- * Does not return until the remote CPU is nearly ready to execute <func>
- * or is or has executed.
- */
-
-int smp_call_function_single (int cpu, void (*func) (void *info), void *info,
- int nonatomic, int wait)
-{
- /* prevent preemption and reschedule on another processor */
- int ret, me = get_cpu();
-
- if (cpu == me) {
- local_irq_disable();
- func(info);
- local_irq_enable();
- put_cpu();
- return 0;
- }
-
- ret = smp_call_function_mask(cpumask_of_cpu(cpu), func, info, wait);
-
- put_cpu();
- return ret;
-}
-EXPORT_SYMBOL(smp_call_function_single);
-
-/*
- * smp_call_function - run a function on all other CPUs.
- * @func: The function to run. This must be fast and non-blocking.
- * @info: An arbitrary pointer to pass to the function.
- * @nonatomic: currently unused.
- * @wait: If true, wait (atomically) until function has completed on other
- * CPUs.
- *
- * Returns 0 on success, else a negative status code. Does not return until
- * remote CPUs are nearly ready to execute func or are or have executed.
- *
- * You must not call this function with disabled interrupts or from a
- * hardware interrupt handler or from a bottom half handler.
- * Actually there are a few legal cases, like panic.
- */
-int smp_call_function (void (*func) (void *info), void *info, int nonatomic,
- int wait)
-{
- return smp_call_function_mask(cpu_online_map, func, info, wait);
-}
-EXPORT_SYMBOL(smp_call_function);
-
static void stop_this_cpu(void *dummy)
{
local_irq_disable();
diff --git a/arch/x86/kernel/smpcommon.c b/arch/x86/kernel/smpcommon.c
new file mode 100644
index 0000000..3449064
--- /dev/null
+++ b/arch/x86/kernel/smpcommon.c
@@ -0,0 +1,83 @@
+/*
+ * SMP stuff which is common to all sub-architectures.
+ */
+#include <linux/module.h>
+#include <asm/smp.h>
+
+#ifdef CONFIG_X86_32
+DEFINE_PER_CPU(unsigned long, this_cpu_off);
+EXPORT_PER_CPU_SYMBOL(this_cpu_off);
+
+/* Initialize the CPU's GDT. This is either the boot CPU doing itself
+ (still using the master per-cpu area), or a CPU doing it for a
+ secondary which will soon come up. */
+__cpuinit void init_gdt(int cpu)
+{
+ struct desc_struct *gdt = get_cpu_gdt_table(cpu);
+
+ pack_descriptor(&gdt[GDT_ENTRY_PERCPU],
+ __per_cpu_offset[cpu], 0xFFFFF,
+ 0x2 | DESCTYPE_S, 0x8);
+
+ gdt[GDT_ENTRY_PERCPU].s = 1;
+
+ per_cpu(this_cpu_off, cpu) = __per_cpu_offset[cpu];
+ per_cpu(cpu_number, cpu) = cpu;
+}
+#endif
+
+/**
+ * smp_call_function(): Run a function on all other CPUs.
+ * @func: The function to run. This must be fast and non-blocking.
+ * @info: An arbitrary pointer to pass to the function.
+ * @nonatomic: Unused.
+ * @wait: If true, wait (atomically) until function has completed on other CPUs.
+ *
+ * Returns 0 on success, else a negative status code.
+ *
+ * If @wait is true, then returns once @func has returned; otherwise
+ * it returns just before the target cpu calls @func.
+ *
+ * You must not call this function with disabled interrupts or from a
+ * hardware interrupt handler or from a bottom half handler.
+ */
+int smp_call_function(void (*func) (void *info), void *info, int nonatomic,
+ int wait)
+{
+ return smp_call_function_mask(cpu_online_map, func, info, wait);
+}
+EXPORT_SYMBOL(smp_call_function);
+
+/**
+ * smp_call_function_single - Run a function on a specific CPU
+ * @cpu: The target CPU. Cannot be the calling CPU.
+ * @func: The function to run. This must be fast and non-blocking.
+ * @info: An arbitrary pointer to pass to the function.
+ * @nonatomic: Unused.
+ * @wait: If true, wait until function has completed on other CPUs.
+ *
+ * Returns 0 on success, else a negative status code.
+ *
+ * If @wait is true, then returns once @func has returned; otherwise
+ * it returns just before the target cpu calls @func.
+ */
+int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
+ int nonatomic, int wait)
+{
+ /* prevent preemption and reschedule on another processor */
+ int ret;
+ int me = get_cpu();
+ if (cpu == me) {
+ local_irq_disable();
+ func(info);
+ local_irq_enable();
+ put_cpu();
+ return 0;
+ }
+
+ ret = smp_call_function_mask(cpumask_of_cpu(cpu), func, info, wait);
+
+ put_cpu();
+ return ret;
+}
+EXPORT_SYMBOL(smp_call_function_single);
diff --git a/arch/x86/kernel/smpcommon_32.c b/arch/x86/kernel/smpcommon_32.c
index 8bc38af..8b13789 100644
--- a/arch/x86/kernel/smpcommon_32.c
+++ b/arch/x86/kernel/smpcommon_32.c
@@ -1,82 +1 @@
-/*
- * SMP stuff which is common to all sub-architectures.
- */
-#include <linux/module.h>
-#include <asm/smp.h>
-DEFINE_PER_CPU(unsigned long, this_cpu_off);
-EXPORT_PER_CPU_SYMBOL(this_cpu_off);
-
-/* Initialize the CPU's GDT. This is either the boot CPU doing itself
- (still using the master per-cpu area), or a CPU doing it for a
- secondary which will soon come up. */
-__cpuinit void init_gdt(int cpu)
-{
- struct desc_struct *gdt = get_cpu_gdt_table(cpu);
-
- pack_descriptor(&gdt[GDT_ENTRY_PERCPU],
- __per_cpu_offset[cpu], 0xFFFFF,
- 0x2 | DESCTYPE_S, 0x8);
-
- gdt[GDT_ENTRY_PERCPU].s = 1;
-
- per_cpu(this_cpu_off, cpu) = __per_cpu_offset[cpu];
- per_cpu(cpu_number, cpu) = cpu;
-}
-
-
-/**
- * smp_call_function(): Run a function on all other CPUs.
- * @func: The function to run. This must be fast and non-blocking.
- * @info: An arbitrary pointer to pass to the function.
- * @nonatomic: Unused.
- * @wait: If true, wait (atomically) until function has completed on other CPUs.
- *
- * Returns 0 on success, else a negative status code.
- *
- * If @wait is true, then returns once @func has returned; otherwise
- * it returns just before the target cpu calls @func.
- *
- * You must not call this function with disabled interrupts or from a
- * hardware interrupt handler or from a bottom half handler.
- */
-int smp_call_function(void (*func) (void *info), void *info, int nonatomic,
- int wait)
-{
- return smp_call_function_mask(cpu_online_map, func, info, wait);
-}
-EXPORT_SYMBOL(smp_call_function);
-
-/**
- * smp_call_function_single - Run a function on a specific CPU
- * @cpu: The target CPU. Cannot be the calling CPU.
- * @func: The function to run. This must be fast and non-blocking.
- * @info: An arbitrary pointer to pass to the function.
- * @nonatomic: Unused.
- * @wait: If true, wait until function has completed on other CPUs.
- *
- * Returns 0 on success, else a negative status code.
- *
- * If @wait is true, then returns once @func has returned; otherwise
- * it returns just before the target cpu calls @func.
- */
-int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
- int nonatomic, int wait)
-{
- /* prevent preemption and reschedule on another processor */
- int ret;
- int me = get_cpu();
- if (cpu == me) {
- local_irq_disable();
- func(info);
- local_irq_enable();
- put_cpu();
- return 0;
- }
-
- ret = smp_call_function_mask(cpumask_of_cpu(cpu), func, info, wait);
-
- put_cpu();
- return ret;
-}
-EXPORT_SYMBOL(smp_call_function_single);
--
1.5.0.6
this patch moves all the functions and data structures that look
like exactly the same from smp_{32,64}.c to smp.c
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/Makefile | 3 +-
arch/x86/kernel/smp.c | 253 ++++++++++++++++++++++++++++++++++++++++++++++
arch/x86/kernel/smp_32.c | 223 ----------------------------------------
arch/x86/kernel/smp_64.c | 205 -------------------------------------
4 files changed, 255 insertions(+), 429 deletions(-)
create mode 100644 arch/x86/kernel/smp.c
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index 018d04d..0a4b088 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -46,7 +46,8 @@ obj-$(CONFIG_MICROCODE) += microcode.o
obj-$(CONFIG_PCI) += early-quirks.o
apm-y := apm_32.o
obj-$(CONFIG_APM) += apm.o
-obj-$(CONFIG_X86_SMP) += smp_$(BITS).o smpboot_$(BITS).o smpboot.o tsc_sync.o
+obj-$(CONFIG_X86_SMP) += smp_$(BITS).o smpboot_$(BITS).o smp.o
+obj-$(CONFIG_X86_SMP) += smpboot.o tsc_sync.o
obj-$(CONFIG_X86_32_SMP) += smpcommon.o
obj-$(CONFIG_X86_64_SMP) += smp_64.o smpboot_64.o tsc_sync.o smpcommon.o
obj-$(CONFIG_X86_TRAMPOLINE) += trampoline_$(BITS).o
diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c
new file mode 100644
index 0000000..b662300
--- /dev/null
+++ b/arch/x86/kernel/smp.c
@@ -0,0 +1,253 @@
+#include <linux/init.h>
+
+#include <linux/mm.h>
+#include <linux/delay.h>
+#include <linux/spinlock.h>
+#include <linux/kernel_stat.h>
+#include <linux/mc146818rtc.h>
+#include <linux/cache.h>
+#include <linux/interrupt.h>
+#include <linux/cpu.h>
+
+#include <asm/mtrr.h>
+#include <asm/tlbflush.h>
+#include <asm/mmu_context.h>
+#include <asm/proto.h>
+#ifdef CONFIG_X86_32
+#include <mach_apic.h>
+#include <mach_ipi.h>
+#else
+#include <asm/mach_apic.h>
+#endif
+
+/*
+ * this function sends a 'reschedule' IPI to another CPU.
+ * it goes straight through and wastes no time serializing
+ * anything. Worst case is that we lose a reschedule ...
+ */
+static void native_smp_send_reschedule(int cpu)
+{
+ WARN_ON(cpu_is_offline(cpu));
+ send_IPI_mask(cpumask_of_cpu(cpu), RESCHEDULE_VECTOR);
+}
+
+/*
+ * Structure and data for smp_call_function(). This is designed to minimise
+ * static memory requirements. It also looks cleaner.
+ */
+static DEFINE_SPINLOCK(call_lock);
+
+struct call_data_struct {
+ void (*func) (void *info);
+ void *info;
+ atomic_t started;
+ atomic_t finished;
+ int wait;
+};
+
+void lock_ipi_call_lock(void)
+{
+ spin_lock_irq(&call_lock);
+}
+
+void unlock_ipi_call_lock(void)
+{
+ spin_unlock_irq(&call_lock);
+}
+
+static struct call_data_struct *call_data;
+
+static void __smp_call_function(void (*func) (void *info), void *info,
+ int nonatomic, int wait)
+{
+ struct call_data_struct data;
+ int cpus = num_online_cpus() - 1;
+
+ if (!cpus)
+ return;
+
+ data.func = func;
+ data.info = info;
+ atomic_set(&data.started, 0);
+ data.wait = wait;
+ if (wait)
+ atomic_set(&data.finished, 0);
+
+ call_data = &data;
+ mb();
+
+ /* Send a message to all other CPUs and wait for them to respond */
+ send_IPI_allbutself(CALL_FUNCTION_VECTOR);
+
+ /* Wait for response */
+ while (atomic_read(&data.started) != cpus)
+ cpu_relax();
+
+ if (wait)
+ while (atomic_read(&data.finished) != cpus)
+ cpu_relax();
+}
+
+
+/**
+ * smp_call_function_mask(): Run a function on a set of other CPUs.
+ * @mask: The set of cpus to run on. Must not include the current cpu.
+ * @func: The function to run. This must be fast and non-blocking.
+ * @info: An arbitrary pointer to pass to the function.
+ * @wait: If true, wait (atomically) until function has completed on other CPUs.
+ *
+ * Returns 0 on success, else a negative status code.
+ *
+ * If @wait is true, then returns once @func has returned; otherwise
+ * it returns just before the target cpu calls @func.
+ *
+ * You must not call this function with disabled interrupts or from a
+ * hardware interrupt handler or from a bottom half handler.
+ */
+static int
+native_smp_call_function_mask(cpumask_t mask,
+ void (*func)(void *), void *info,
+ int wait)
+{
+ struct call_data_struct data;
+ cpumask_t allbutself;
+ int cpus;
+
+ /* Can deadlock when called with interrupts disabled */
+ WARN_ON(irqs_disabled());
+
+ /* Holding any lock stops cpus from going down. */
+ spin_lock(&call_lock);
+
+ allbutself = cpu_online_map;
+ cpu_clear(smp_processor_id(), allbutself);
+
+ cpus_and(mask, mask, allbutself);
+ cpus = cpus_weight(mask);
+
+ if (!cpus) {
+ spin_unlock(&call_lock);
+ return 0;
+ }
+
+ data.func = func;
+ data.info = info;
+ atomic_set(&data.started, 0);
+ data.wait = wait;
+ if (wait)
+ atomic_set(&data.finished, 0);
+
+ call_data = &data;
+ wmb();
+
+ /* Send a message to other CPUs */
+ if (cpus_equal(mask, allbutself))
+ send_IPI_allbutself(CALL_FUNCTION_VECTOR);
+ else
+ send_IPI_mask(mask, CALL_FUNCTION_VECTOR);
+
+ /* Wait for response */
+ while (atomic_read(&data.started) != cpus)
+ cpu_relax();
+
+ if (wait)
+ while (atomic_read(&data.finished) != cpus)
+ cpu_relax();
+ spin_unlock(&call_lock);
+
+ return 0;
+}
+
+static void stop_this_cpu(void *dummy)
+{
+ local_irq_disable();
+ /*
+ * Remove this CPU:
+ */
+ cpu_clear(smp_processor_id(), cpu_online_map);
+ disable_local_APIC();
+ if (hlt_works(smp_processor_id()))
+ for (;;) halt();
+ for (;;);
+}
+
+/*
+ * this function calls the 'stop' function on all other CPUs in the system.
+ */
+
+static void native_smp_send_stop(void)
+{
+ int nolock;
+ unsigned long flags;
+
+ if (reboot_force)
+ return;
+
+ /* Don't deadlock on the call lock in panic */
+ nolock = !spin_trylock(&call_lock);
+ local_irq_save(flags);
+ __smp_call_function(stop_this_cpu, NULL, 0, 0);
+ if (!nolock)
+ spin_unlock(&call_lock);
+ disable_local_APIC();
+ local_irq_restore(flags);
+}
+
+/*
+ * Reschedule call back. Nothing to do,
+ * all the work is done automatically when
+ * we return from the interrupt.
+ */
+void smp_reschedule_interrupt(struct pt_regs *regs)
+{
+ ack_APIC_irq();
+#ifdef CONFIG_X86_32
+ __get_cpu_var(irq_stat).irq_resched_count++;
+#else
+ add_pda(irq_resched_count, 1);
+#endif
+}
+
+void smp_call_function_interrupt(struct pt_regs *regs)
+{
+ void (*func) (void *info) = call_data->func;
+ void *info = call_data->info;
+ int wait = call_data->wait;
+
+ ack_APIC_irq();
+ /*
+ * Notify initiating CPU that I've grabbed the data and am
+ * about to execute the function
+ */
+ mb();
+ atomic_inc(&call_data->started);
+ /*
+ * At this point the info structure may be out of scope unless wait==1
+ */
+ irq_enter();
+ (*func)(info);
+#ifdef CONFIG_X86_32
+ __get_cpu_var(irq_stat).irq_call_count++;
+#else
+ add_pda(irq_call_count, 1);
+#endif
+ irq_exit();
+
+ if (wait) {
+ mb();
+ atomic_inc(&call_data->finished);
+ }
+}
+
+struct smp_ops smp_ops = {
+ .smp_prepare_boot_cpu = native_smp_prepare_boot_cpu,
+ .smp_prepare_cpus = native_smp_prepare_cpus,
+ .cpu_up = native_cpu_up,
+ .smp_cpus_done = native_smp_cpus_done,
+
+ .smp_send_stop = native_smp_send_stop,
+ .smp_send_reschedule = native_smp_send_reschedule,
+ .smp_call_function_mask = native_smp_call_function_mask,
+};
+EXPORT_SYMBOL_GPL(smp_ops);
+
diff --git a/arch/x86/kernel/smp_32.c b/arch/x86/kernel/smp_32.c
index 8be3e09..61e546e 100644
--- a/arch/x86/kernel/smp_32.c
+++ b/arch/x86/kernel/smp_32.c
@@ -466,217 +466,6 @@ void flush_tlb_all(void)
on_each_cpu(do_flush_tlb_all, NULL, 1, 1);
}
-/*
- * this function sends a 'reschedule' IPI to another CPU.
- * it goes straight through and wastes no time serializing
- * anything. Worst case is that we lose a reschedule ...
- */
-static void native_smp_send_reschedule(int cpu)
-{
- WARN_ON(cpu_is_offline(cpu));
- send_IPI_mask(cpumask_of_cpu(cpu), RESCHEDULE_VECTOR);
-}
-
-/*
- * Structure and data for smp_call_function(). This is designed to minimise
- * static memory requirements. It also looks cleaner.
- */
-static DEFINE_SPINLOCK(call_lock);
-
-struct call_data_struct {
- void (*func) (void *info);
- void *info;
- atomic_t started;
- atomic_t finished;
- int wait;
-};
-
-void lock_ipi_call_lock(void)
-{
- spin_lock_irq(&call_lock);
-}
-
-void unlock_ipi_call_lock(void)
-{
- spin_unlock_irq(&call_lock);
-}
-
-static struct call_data_struct *call_data;
-
-static void __smp_call_function(void (*func) (void *info), void *info,
- int nonatomic, int wait)
-{
- struct call_data_struct data;
- int cpus = num_online_cpus() - 1;
-
- if (!cpus)
- return;
-
- data.func = func;
- data.info = info;
- atomic_set(&data.started, 0);
- data.wait = wait;
- if (wait)
- atomic_set(&data.finished, 0);
-
- call_data = &data;
- mb();
-
- /* Send a message to all other CPUs and wait for them to respond */
- send_IPI_allbutself(CALL_FUNCTION_VECTOR);
-
- /* Wait for response */
- while (atomic_read(&data.started) != cpus)
- cpu_relax();
-
- if (wait)
- while (atomic_read(&data.finished) != cpus)
- cpu_relax();
-}
-
-
-/**
- * smp_call_function_mask(): Run a function on a set of other CPUs.
- * @mask: The set of cpus to run on. Must not include the current cpu.
- * @func: The function to run. This must be fast and non-blocking.
- * @info: An arbitrary pointer to pass to the function.
- * @wait: If true, wait (atomically) until function has completed on other CPUs.
- *
- * Returns 0 on success, else a negative status code.
- *
- * If @wait is true, then returns once @func has returned; otherwise
- * it returns just before the target cpu calls @func.
- *
- * You must not call this function with disabled interrupts or from a
- * hardware interrupt handler or from a bottom half handler.
- */
-static int
-native_smp_call_function_mask(cpumask_t mask,
- void (*func)(void *), void *info,
- int wait)
-{
- struct call_data_struct data;
- cpumask_t allbutself;
- int cpus;
-
- /* Can deadlock when called with interrupts disabled */
- WARN_ON(irqs_disabled());
-
- /* Holding any lock stops cpus from going down. */
- spin_lock(&call_lock);
-
- allbutself = cpu_online_map;
- cpu_clear(smp_processor_id(), allbutself);
-
- cpus_and(mask, mask, allbutself);
- cpus = cpus_weight(mask);
-
- if (!cpus) {
- spin_unlock(&call_lock);
- return 0;
- }
-
- data.func = func;
- data.info = info;
- atomic_set(&data.started, 0);
- data.wait = wait;
- if (wait)
- atomic_set(&data.finished, 0);
-
- call_data = &data;
- wmb();
-
- /* Send a message to other CPUs */
- if (cpus_equal(mask, allbutself))
- send_IPI_allbutself(CALL_FUNCTION_VECTOR);
- else
- send_IPI_mask(mask, CALL_FUNCTION_VECTOR);
-
- /* Wait for response */
- while (atomic_read(&data.started) != cpus)
- cpu_relax();
-
- if (wait)
- while (atomic_read(&data.finished) != cpus)
- cpu_relax();
- spin_unlock(&call_lock);
-
- return 0;
-}
-
-static void stop_this_cpu (void * dummy)
-{
- local_irq_disable();
- /*
- * Remove this CPU:
- */
- cpu_clear(smp_processor_id(), cpu_online_map);
- disable_local_APIC();
- if (hlt_works(smp_processor_id()))
- for(;;) halt();
- for (;;);
-}
-
-/*
- * this function calls the 'stop' function on all other CPUs in the system.
- */
-
-static void native_smp_send_stop(void)
-{
- int nolock;
- unsigned long flags;
-
- if (reboot_force)
- return;
-
- /* Don't deadlock on the call lock in panic */
- nolock = !spin_trylock(&call_lock);
- local_irq_save(flags);
- __smp_call_function(stop_this_cpu, NULL, 0, 0);
- if (!nolock)
- spin_unlock(&call_lock);
- disable_local_APIC();
- local_irq_restore(flags);
-}
-
-/*
- * Reschedule call back. Nothing to do,
- * all the work is done automatically when
- * we return from the interrupt.
- */
-void smp_reschedule_interrupt(struct pt_regs *regs)
-{
- ack_APIC_irq();
- __get_cpu_var(irq_stat).irq_resched_count++;
-}
-
-void smp_call_function_interrupt(struct pt_regs *regs)
-{
- void (*func) (void *info) = call_data->func;
- void *info = call_data->info;
- int wait = call_data->wait;
-
- ack_APIC_irq();
- /*
- * Notify initiating CPU that I've grabbed the data and am
- * about to execute the function
- */
- mb();
- atomic_inc(&call_data->started);
- /*
- * At this point the info structure may be out of scope unless wait==1
- */
- irq_enter();
- (*func)(info);
- __get_cpu_var(irq_stat).irq_call_count++;
- irq_exit();
-
- if (wait) {
- mb();
- atomic_inc(&call_data->finished);
- }
-}
-
static int convert_apicid_to_cpu(int apic_id)
{
int i;
@@ -703,15 +492,3 @@ int safe_smp_processor_id(void)
return cpuid >= 0 ? cpuid : 0;
}
-
-struct smp_ops smp_ops = {
- .smp_prepare_boot_cpu = native_smp_prepare_boot_cpu,
- .smp_prepare_cpus = native_smp_prepare_cpus,
- .cpu_up = native_cpu_up,
- .smp_cpus_done = native_smp_cpus_done,
-
- .smp_send_stop = native_smp_send_stop,
- .smp_send_reschedule = native_smp_send_reschedule,
- .smp_call_function_mask = native_smp_call_function_mask,
-};
-EXPORT_SYMBOL_GPL(smp_ops);
diff --git a/arch/x86/kernel/smp_64.c b/arch/x86/kernel/smp_64.c
index ad11ef0..d28e868 100644
--- a/arch/x86/kernel/smp_64.c
+++ b/arch/x86/kernel/smp_64.c
@@ -283,208 +283,3 @@ void flush_tlb_all(void)
{
on_each_cpu(do_flush_tlb_all, NULL, 1, 1);
}
-
-/*
- * this function sends a 'reschedule' IPI to another CPU.
- * it goes straight through and wastes no time serializing
- * anything. Worst case is that we lose a reschedule ...
- */
-
-static void native_smp_send_reschedule(int cpu)
-{
- WARN_ON(cpu_is_offline(cpu));
- send_IPI_mask(cpumask_of_cpu(cpu), RESCHEDULE_VECTOR);
-}
-
-/*
- * Structure and data for smp_call_function(). This is designed to minimise
- * static memory requirements. It also looks cleaner.
- */
-static DEFINE_SPINLOCK(call_lock);
-
-struct call_data_struct {
- void (*func) (void *info);
- void *info;
- atomic_t started;
- atomic_t finished;
- int wait;
-};
-
-static struct call_data_struct * call_data;
-
-void lock_ipi_call_lock(void)
-{
- spin_lock_irq(&call_lock);
-}
-
-void unlock_ipi_call_lock(void)
-{
- spin_unlock_irq(&call_lock);
-}
-
-static void __smp_call_function(void (*func) (void *info), void *info,
- int nonatomic, int wait)
-{
- struct call_data_struct data;
- int cpus = num_online_cpus() - 1;
-
- if (!cpus)
- return;
-
- data.func = func;
- data.info = info;
- atomic_set(&data.started, 0);
- data.wait = wait;
- if (wait)
- atomic_set(&data.finished, 0);
-
- call_data = &data;
- mb();
-
- /* Send a message to all other CPUs and wait for them to respond */
- send_IPI_allbutself(CALL_FUNCTION_VECTOR);
-
- /* Wait for response */
- while (atomic_read(&data.started) != cpus)
- cpu_relax();
-
- if (wait)
- while (atomic_read(&data.finished) != cpus)
- cpu_relax();
-}
-
-
-int native_smp_call_function_mask(cpumask_t mask,
- void (*func)(void *), void *info,
- int wait)
-{
- struct call_data_struct data;
- cpumask_t allbutself;
- int cpus;
-
- /* Can deadlock when called with interrupts disabled */
- WARN_ON(irqs_disabled());
-
- /* Holding any lock stops cpus from going down. */
- spin_lock(&call_lock);
-
- allbutself = cpu_online_map;
- cpu_clear(smp_processor_id(), allbutself);
-
- cpus_and(mask, mask, allbutself);
- cpus = cpus_weight(mask);
-
- if (!cpus) {
- spin_unlock(&call_lock);
- return 0;
- }
-
- data.func = func;
- data.info = info;
- atomic_set(&data.started, 0);
- data.wait = wait;
- if (wait)
- atomic_set(&data.finished, 0);
-
- call_data = &data;
- wmb();
-
- /* Send a message to other CPUs */
- if (cpus_equal(mask, allbutself))
- send_IPI_allbutself(CALL_FUNCTION_VECTOR);
- else
- send_IPI_mask(mask, CALL_FUNCTION_VECTOR);
-
- /* Wait for response */
- while (atomic_read(&data.started) != cpus)
- cpu_relax();
-
- if (wait)
- while (atomic_read(&data.finished) != cpus)
- cpu_relax();
-
- spin_unlock(&call_lock);
-
- return 0;
-}
-
-static void stop_this_cpu(void *dummy)
-{
- local_irq_disable();
- /*
- * Remove this CPU:
- */
- cpu_clear(smp_processor_id(), cpu_online_map);
- disable_local_APIC();
- if (hlt_works(smp_processor_id()))
- for (;;) halt();
- for (;;);
-}
-
-void native_smp_send_stop(void)
-{
- int nolock;
- unsigned long flags;
-
- if (reboot_force)
- return;
-
- /* Don't deadlock on the call lock in panic */
- nolock = !spin_trylock(&call_lock);
- local_irq_save(flags);
- __smp_call_function(stop_this_cpu, NULL, 0, 0);
- if (!nolock)
- spin_unlock(&call_lock);
- disable_local_APIC();
- local_irq_restore(flags);
-}
-
-/*
- * Reschedule call back. Nothing to do,
- * all the work is done automatically when
- * we return from the interrupt.
- */
-asmlinkage void smp_reschedule_interrupt(void)
-{
- ack_APIC_irq();
- add_pda(irq_resched_count, 1);
-}
-
-asmlinkage void smp_call_function_interrupt(void)
-{
- void (*func) (void *info) = call_data->func;
- void *info = call_data->info;
- int wait = call_data->wait;
-
- ack_APIC_irq();
- /*
- * Notify initiating CPU that I've grabbed the data and am
- * about to execute the function
- */
- mb();
- atomic_inc(&call_data->started);
- /*
- * At this point the info structure may be out of scope unless wait==1
- */
- exit_idle();
- irq_enter();
- (*func)(info);
- add_pda(irq_call_count, 1);
- irq_exit();
- if (wait) {
- mb();
- atomic_inc(&call_data->finished);
- }
-}
-
-struct smp_ops smp_ops = {
- .smp_prepare_boot_cpu = native_smp_prepare_boot_cpu,
- .smp_prepare_cpus = native_smp_prepare_cpus,
- .smp_cpus_done = native_smp_cpus_done,
-
- .smp_send_stop = native_smp_send_stop,
- .smp_send_reschedule = native_smp_send_reschedule,
- .smp_call_function_mask = native_smp_call_function_mask,
- .cpu_up = native_cpu_up,
-};
-EXPORT_SYMBOL_GPL(smp_ops);
--
1.5.0.6
it is already defined in smp.h
Signed-off-by: Glauber Costa <[email protected]>
---
include/asm-x86/processor.h | 4 +---
1 files changed, 1 insertions(+), 3 deletions(-)
diff --git a/include/asm-x86/processor.h b/include/asm-x86/processor.h
index 8bec23c..e49e5e6 100644
--- a/include/asm-x86/processor.h
+++ b/include/asm-x86/processor.h
@@ -355,9 +355,7 @@ union i387_union {
struct i387_soft_struct soft;
};
-#ifdef CONFIG_X86_32
-DECLARE_PER_CPU(u8, cpu_llc_id);
-#else
+#ifdef CONFIG_X86_64
DECLARE_PER_CPU(struct orig_ist, orig_ist);
#endif
--
1.5.0.6
move definitions that are now equal in type from
smpboot_{32,64}.c to smpboot.c
cpu_callin_map is put temporarily in smp_64.h (already
exists in smp_32.h), and will soon be merged.
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smpboot.c | 28 ++++++++++++++++++++++++++++
arch/x86/kernel/smpboot_32.c | 27 ---------------------------
arch/x86/kernel/smpboot_64.c | 33 ---------------------------------
include/asm-x86/smp_64.h | 1 +
4 files changed, 29 insertions(+), 60 deletions(-)
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index bffe108..40a3b56 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -1,6 +1,34 @@
#include <linux/init.h>
#include <linux/smp.h>
+#include <linux/module.h>
+/* Number of siblings per CPU package */
+int smp_num_siblings = 1;
+EXPORT_SYMBOL(smp_num_siblings);
+
+/* Last level cache ID of each logical CPU */
+DEFINE_PER_CPU(u16, cpu_llc_id) = BAD_APICID;
+
+/* bitmap of online cpus */
+cpumask_t cpu_online_map __read_mostly;
+EXPORT_SYMBOL(cpu_online_map);
+
+cpumask_t cpu_callin_map;
+cpumask_t cpu_callout_map;
+cpumask_t cpu_possible_map;
+EXPORT_SYMBOL(cpu_possible_map);
+
+/* representing HT siblings of each logical CPU */
+DEFINE_PER_CPU(cpumask_t, cpu_sibling_map);
+EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
+
+/* representing HT and core siblings of each logical CPU */
+DEFINE_PER_CPU(cpumask_t, cpu_core_map);
+EXPORT_PER_CPU_SYMBOL(cpu_core_map);
+
+/* Per CPU bogomips and other parameters */
+DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);
+EXPORT_PER_CPU_SYMBOL(cpu_info);
#ifdef CONFIG_HOTPLUG_CPU
int additional_cpus __initdata = -1;
diff --git a/arch/x86/kernel/smpboot_32.c b/arch/x86/kernel/smpboot_32.c
index 5a446f0..0fbc981 100644
--- a/arch/x86/kernel/smpboot_32.c
+++ b/arch/x86/kernel/smpboot_32.c
@@ -62,35 +62,8 @@
/* Set if we find a B stepping CPU */
static int __cpuinitdata smp_b_stepping;
-/* Number of siblings per CPU package */
-int smp_num_siblings = 1;
-EXPORT_SYMBOL(smp_num_siblings);
-
-/* Last level cache ID of each logical CPU */
-DEFINE_PER_CPU(u16, cpu_llc_id) = BAD_APICID;
-
-/* representing HT siblings of each logical CPU */
-DEFINE_PER_CPU(cpumask_t, cpu_sibling_map);
-EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
-
-/* representing HT and core siblings of each logical CPU */
-DEFINE_PER_CPU(cpumask_t, cpu_core_map);
-EXPORT_PER_CPU_SYMBOL(cpu_core_map);
-
-/* bitmap of online cpus */
-cpumask_t cpu_online_map __read_mostly;
-EXPORT_SYMBOL(cpu_online_map);
-
-cpumask_t cpu_callin_map;
-cpumask_t cpu_callout_map;
-cpumask_t cpu_possible_map;
-EXPORT_SYMBOL(cpu_possible_map);
static cpumask_t smp_commenced_mask;
-/* Per CPU bogomips and other parameters */
-DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);
-EXPORT_PER_CPU_SYMBOL(cpu_info);
-
/* which logical CPU number maps to which CPU (physical APIC ID) */
u16 x86_cpu_to_apicid_init[NR_CPUS] __initdata =
{ [0 ... NR_CPUS-1] = BAD_APICID };
diff --git a/arch/x86/kernel/smpboot_64.c b/arch/x86/kernel/smpboot_64.c
index 953b0ff..c51279f 100644
--- a/arch/x86/kernel/smpboot_64.c
+++ b/arch/x86/kernel/smpboot_64.c
@@ -60,42 +60,9 @@
#include <asm/hw_irq.h>
#include <asm/numa.h>
-/* Number of siblings per CPU package */
-int smp_num_siblings = 1;
-EXPORT_SYMBOL(smp_num_siblings);
-
-/* Last level cache ID of each logical CPU */
-DEFINE_PER_CPU(u16, cpu_llc_id) = BAD_APICID;
-
-/* Bitmask of currently online CPUs */
-cpumask_t cpu_online_map __read_mostly;
-
-EXPORT_SYMBOL(cpu_online_map);
-
-/*
- * Private maps to synchronize booting between AP and BP.
- * Probably not needed anymore, but it makes for easier debugging. -AK
- */
-cpumask_t cpu_callin_map;
-cpumask_t cpu_callout_map;
-cpumask_t cpu_possible_map;
-EXPORT_SYMBOL(cpu_possible_map);
-
-/* Per CPU bogomips and other parameters */
-DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);
-EXPORT_PER_CPU_SYMBOL(cpu_info);
-
/* Set when the idlers are all forked */
int smp_threads_ready;
-/* representing HT siblings of each logical CPU */
-DEFINE_PER_CPU(cpumask_t, cpu_sibling_map);
-EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
-
-/* representing HT and core siblings of each logical CPU */
-DEFINE_PER_CPU(cpumask_t, cpu_core_map);
-EXPORT_PER_CPU_SYMBOL(cpu_core_map);
-
/*
* Trampoline 80x86 program as an array.
*/
diff --git a/include/asm-x86/smp_64.h b/include/asm-x86/smp_64.h
index e5bc1be..1ecf813 100644
--- a/include/asm-x86/smp_64.h
+++ b/include/asm-x86/smp_64.h
@@ -14,6 +14,7 @@
#include <asm/thread_info.h>
extern cpumask_t cpu_initialized;
+extern cpumask_t cpu_callin_map;
extern int smp_call_function_mask(cpumask_t mask, void (*func)(void *),
void *info, int wait);
--
1.5.0.6
so they can have the same type as x86_64
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smpboot_32.c | 6 +++---
include/asm-x86/smp_32.h | 6 +++---
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/arch/x86/kernel/smpboot_32.c b/arch/x86/kernel/smpboot_32.c
index 579b9b7..5a446f0 100644
--- a/arch/x86/kernel/smpboot_32.c
+++ b/arch/x86/kernel/smpboot_32.c
@@ -67,7 +67,7 @@ int smp_num_siblings = 1;
EXPORT_SYMBOL(smp_num_siblings);
/* Last level cache ID of each logical CPU */
-DEFINE_PER_CPU(u8, cpu_llc_id) = BAD_APICID;
+DEFINE_PER_CPU(u16, cpu_llc_id) = BAD_APICID;
/* representing HT siblings of each logical CPU */
DEFINE_PER_CPU(cpumask_t, cpu_sibling_map);
@@ -92,10 +92,10 @@ DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);
EXPORT_PER_CPU_SYMBOL(cpu_info);
/* which logical CPU number maps to which CPU (physical APIC ID) */
-u8 x86_cpu_to_apicid_init[NR_CPUS] __initdata =
+u16 x86_cpu_to_apicid_init[NR_CPUS] __initdata =
{ [0 ... NR_CPUS-1] = BAD_APICID };
void *x86_cpu_to_apicid_early_ptr;
-DEFINE_PER_CPU(u8, x86_cpu_to_apicid) = BAD_APICID;
+DEFINE_PER_CPU(u16, x86_cpu_to_apicid) = BAD_APICID;
EXPORT_PER_CPU_SYMBOL(x86_cpu_to_apicid);
u8 apicid_2_node[MAX_APICID];
diff --git a/include/asm-x86/smp_32.h b/include/asm-x86/smp_32.h
index 41b58e0..29f6195 100644
--- a/include/asm-x86/smp_32.h
+++ b/include/asm-x86/smp_32.h
@@ -21,13 +21,13 @@ extern cpumask_t cpu_callin_map;
extern void (*mtrr_hook) (void);
extern void zap_low_mappings (void);
-extern u8 __initdata x86_cpu_to_apicid_init[];
+extern u16 __initdata x86_cpu_to_apicid_init[];
extern void *x86_cpu_to_apicid_early_ptr;
DECLARE_PER_CPU(cpumask_t, cpu_sibling_map);
DECLARE_PER_CPU(cpumask_t, cpu_core_map);
-DECLARE_PER_CPU(u8, cpu_llc_id);
-DECLARE_PER_CPU(u8, x86_cpu_to_apicid);
+DECLARE_PER_CPU(u16, cpu_llc_id);
+DECLARE_PER_CPU(u16, x86_cpu_to_apicid);
#ifdef CONFIG_HOTPLUG_CPU
extern void cpu_exit_clear(void);
--
1.5.0.6
definitions that are inside CONFIG_HOTPLUG_CPU in
the arch-specific smp*.h files are moved to common
header
Signed-off-by: Glauber Costa <[email protected]>
---
include/asm-x86/smp.h | 6 ++++++
include/asm-x86/smp_32.h | 6 ------
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/include/asm-x86/smp.h b/include/asm-x86/smp.h
index 1b4481a..c800b81 100644
--- a/include/asm-x86/smp.h
+++ b/include/asm-x86/smp.h
@@ -80,6 +80,12 @@ extern void prefill_possible_map(void);
# include "smp_64.h"
#endif
+#ifdef CONFIG_HOTPLUG_CPU
+extern void cpu_exit_clear(void);
+extern void cpu_uninit(void);
+extern void remove_siblinginfo(int cpu);
+#endif
+
extern void smp_alloc_memory(void);
extern void lock_ipi_call_lock(void);
extern void unlock_ipi_call_lock(void);
diff --git a/include/asm-x86/smp_32.h b/include/asm-x86/smp_32.h
index 76247a9..0b25134 100644
--- a/include/asm-x86/smp_32.h
+++ b/include/asm-x86/smp_32.h
@@ -29,12 +29,6 @@ DECLARE_PER_CPU(cpumask_t, cpu_core_map);
DECLARE_PER_CPU(u16, cpu_llc_id);
DECLARE_PER_CPU(u16, x86_cpu_to_apicid);
-#ifdef CONFIG_HOTPLUG_CPU
-extern void cpu_exit_clear(void);
-extern void cpu_uninit(void);
-extern void remove_siblinginfo(int cpu);
-#endif
-
#ifdef CONFIG_SMP
#ifndef CONFIG_PARAVIRT
#define startup_ipi_hook(phys_apicid, start_eip, start_esp) do { } while (0)
--
1.5.0.6
this is done to match i386
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smpboot_64.c | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/arch/x86/kernel/smpboot_64.c b/arch/x86/kernel/smpboot_64.c
index 1e8f00a..20f1c7d 100644
--- a/arch/x86/kernel/smpboot_64.c
+++ b/arch/x86/kernel/smpboot_64.c
@@ -918,7 +918,7 @@ void __init native_smp_cpus_done(unsigned int max_cpus)
#ifdef CONFIG_HOTPLUG_CPU
-static void remove_siblinginfo(int cpu)
+void remove_siblinginfo(int cpu)
{
int sibling;
struct cpuinfo_x86 *c = &cpu_data(cpu);
--
1.5.0.6
And move its extern definition to smp.h, the common header
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smpboot_64.c | 2 +-
include/asm-x86/smp.h | 3 +++
include/asm-x86/smp_32.h | 3 ---
3 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/arch/x86/kernel/smpboot_64.c b/arch/x86/kernel/smpboot_64.c
index c51279f..1e8f00a 100644
--- a/arch/x86/kernel/smpboot_64.c
+++ b/arch/x86/kernel/smpboot_64.c
@@ -228,7 +228,7 @@ cpumask_t cpu_coregroup_map(int cpu)
/* representing cpus for which sibling maps can be computed */
static cpumask_t cpu_sibling_setup_map;
-static inline void set_cpu_sibling_map(int cpu)
+void __cpuinit set_cpu_sibling_map(int cpu)
{
int i;
struct cpuinfo_x86 *c = &cpu_data(cpu);
diff --git a/include/asm-x86/smp.h b/include/asm-x86/smp.h
index 2ab8ed4..1b4481a 100644
--- a/include/asm-x86/smp.h
+++ b/include/asm-x86/smp.h
@@ -22,6 +22,9 @@ struct smp_ops {
int wait);
};
+/* Globals due to paravirt */
+extern void set_cpu_sibling_map(int cpu);
+
#ifdef CONFIG_SMP
extern struct smp_ops smp_ops;
diff --git a/include/asm-x86/smp_32.h b/include/asm-x86/smp_32.h
index 29f6195..76247a9 100644
--- a/include/asm-x86/smp_32.h
+++ b/include/asm-x86/smp_32.h
@@ -35,9 +35,6 @@ extern void cpu_uninit(void);
extern void remove_siblinginfo(int cpu);
#endif
-/* Globals due to paravirt */
-extern void set_cpu_sibling_map(int cpu);
-
#ifdef CONFIG_SMP
#ifndef CONFIG_PARAVIRT
#define startup_ipi_hook(phys_apicid, start_eip, start_esp) do { } while (0)
--
1.5.0.6
This patch moves all ipi and apic related functions
from smp_32.c to ipi.c
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/Makefile | 2 +-
arch/x86/kernel/ipi.c | 178 ++++++++++++++++++++++++++++++++++++++++++++++
arch/x86/kernel/smp_32.c | 153 ---------------------------------------
3 files changed, 179 insertions(+), 154 deletions(-)
create mode 100644 arch/x86/kernel/ipi.c
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index 0a4b088..e3b01f9 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -47,7 +47,7 @@ obj-$(CONFIG_PCI) += early-quirks.o
apm-y := apm_32.o
obj-$(CONFIG_APM) += apm.o
obj-$(CONFIG_X86_SMP) += smp_$(BITS).o smpboot_$(BITS).o smp.o
-obj-$(CONFIG_X86_SMP) += smpboot.o tsc_sync.o
+obj-$(CONFIG_X86_SMP) += smpboot.o tsc_sync.o ipi.o
obj-$(CONFIG_X86_32_SMP) += smpcommon.o
obj-$(CONFIG_X86_64_SMP) += smp_64.o smpboot_64.o tsc_sync.o smpcommon.o
obj-$(CONFIG_X86_TRAMPOLINE) += trampoline_$(BITS).o
diff --git a/arch/x86/kernel/ipi.c b/arch/x86/kernel/ipi.c
new file mode 100644
index 0000000..c0df7b8
--- /dev/null
+++ b/arch/x86/kernel/ipi.c
@@ -0,0 +1,178 @@
+#include <linux/cpumask.h>
+#include <linux/interrupt.h>
+#include <linux/init.h>
+
+#include <linux/mm.h>
+#include <linux/delay.h>
+#include <linux/spinlock.h>
+#include <linux/kernel_stat.h>
+#include <linux/mc146818rtc.h>
+#include <linux/cache.h>
+#include <linux/interrupt.h>
+#include <linux/cpu.h>
+#include <linux/module.h>
+
+#include <asm/smp.h>
+#include <asm/mtrr.h>
+#include <asm/tlbflush.h>
+#include <asm/mmu_context.h>
+#include <asm/apic.h>
+#include <asm/proto.h>
+
+#ifdef CONFIG_X86_32
+#include <mach_apic.h>
+/*
+ * the following functions deal with sending IPIs between CPUs.
+ *
+ * We use 'broadcast', CPU->CPU IPIs and self-IPIs too.
+ */
+
+static inline int __prepare_ICR(unsigned int shortcut, int vector)
+{
+ unsigned int icr = shortcut | APIC_DEST_LOGICAL;
+
+ switch (vector) {
+ default:
+ icr |= APIC_DM_FIXED | vector;
+ break;
+ case NMI_VECTOR:
+ icr |= APIC_DM_NMI;
+ break;
+ }
+ return icr;
+}
+
+static inline int __prepare_ICR2(unsigned int mask)
+{
+ return SET_APIC_DEST_FIELD(mask);
+}
+
+void __send_IPI_shortcut(unsigned int shortcut, int vector)
+{
+ /*
+ * Subtle. In the case of the 'never do double writes' workaround
+ * we have to lock out interrupts to be safe. As we don't care
+ * of the value read we use an atomic rmw access to avoid costly
+ * cli/sti. Otherwise we use an even cheaper single atomic write
+ * to the APIC.
+ */
+ unsigned int cfg;
+
+ /*
+ * Wait for idle.
+ */
+ apic_wait_icr_idle();
+
+ /*
+ * No need to touch the target chip field
+ */
+ cfg = __prepare_ICR(shortcut, vector);
+
+ /*
+ * Send the IPI. The write to APIC_ICR fires this off.
+ */
+ apic_write_around(APIC_ICR, cfg);
+}
+
+void send_IPI_self(int vector)
+{
+ __send_IPI_shortcut(APIC_DEST_SELF, vector);
+}
+
+/*
+ * This is used to send an IPI with no shorthand notation (the destination is
+ * specified in bits 56 to 63 of the ICR).
+ */
+static inline void __send_IPI_dest_field(unsigned long mask, int vector)
+{
+ unsigned long cfg;
+
+ /*
+ * Wait for idle.
+ */
+ if (unlikely(vector == NMI_VECTOR))
+ safe_apic_wait_icr_idle();
+ else
+ apic_wait_icr_idle();
+
+ /*
+ * prepare target chip field
+ */
+ cfg = __prepare_ICR2(mask);
+ apic_write_around(APIC_ICR2, cfg);
+
+ /*
+ * program the ICR
+ */
+ cfg = __prepare_ICR(0, vector);
+
+ /*
+ * Send the IPI. The write to APIC_ICR fires this off.
+ */
+ apic_write_around(APIC_ICR, cfg);
+}
+
+/*
+ * This is only used on smaller machines.
+ */
+void send_IPI_mask_bitmask(cpumask_t cpumask, int vector)
+{
+ unsigned long mask = cpus_addr(cpumask)[0];
+ unsigned long flags;
+
+ local_irq_save(flags);
+ WARN_ON(mask & ~cpus_addr(cpu_online_map)[0]);
+ __send_IPI_dest_field(mask, vector);
+ local_irq_restore(flags);
+}
+
+void send_IPI_mask_sequence(cpumask_t mask, int vector)
+{
+ unsigned long flags;
+ unsigned int query_cpu;
+
+ /*
+ * Hack. The clustered APIC addressing mode doesn't allow us to send
+ * to an arbitrary mask, so I do a unicasts to each CPU instead. This
+ * should be modified to do 1 message per cluster ID - mbligh
+ */
+
+ local_irq_save(flags);
+ for_each_possible_cpu(query_cpu) {
+ if (cpu_isset(query_cpu, mask)) {
+ __send_IPI_dest_field(cpu_to_logical_apicid(query_cpu),
+ vector);
+ }
+ }
+ local_irq_restore(flags);
+}
+
+/* must come after the send_IPI functions above for inlining */
+#include <mach_ipi.h>
+static int convert_apicid_to_cpu(int apic_id)
+{
+ int i;
+
+ for_each_possible_cpu(i) {
+ if (per_cpu(x86_cpu_to_apicid, i) == apic_id)
+ return i;
+ }
+ return -1;
+}
+
+int safe_smp_processor_id(void)
+{
+ int apicid, cpuid;
+
+ if (!boot_cpu_has(X86_FEATURE_APIC))
+ return 0;
+
+ apicid = hard_smp_processor_id();
+ if (apicid == BAD_APICID)
+ return 0;
+
+ cpuid = convert_apicid_to_cpu(apicid);
+
+ return cpuid >= 0 ? cpuid : 0;
+}
+#endif
diff --git a/arch/x86/kernel/smp_32.c b/arch/x86/kernel/smp_32.c
index 61e546e..d80623a 100644
--- a/arch/x86/kernel/smp_32.c
+++ b/arch/x86/kernel/smp_32.c
@@ -107,132 +107,6 @@
DEFINE_PER_CPU(struct tlb_state, cpu_tlbstate) ____cacheline_aligned = { &init_mm, 0, };
-/*
- * the following functions deal with sending IPIs between CPUs.
- *
- * We use 'broadcast', CPU->CPU IPIs and self-IPIs too.
- */
-
-static inline int __prepare_ICR (unsigned int shortcut, int vector)
-{
- unsigned int icr = shortcut | APIC_DEST_LOGICAL;
-
- switch (vector) {
- default:
- icr |= APIC_DM_FIXED | vector;
- break;
- case NMI_VECTOR:
- icr |= APIC_DM_NMI;
- break;
- }
- return icr;
-}
-
-static inline int __prepare_ICR2 (unsigned int mask)
-{
- return SET_APIC_DEST_FIELD(mask);
-}
-
-void __send_IPI_shortcut(unsigned int shortcut, int vector)
-{
- /*
- * Subtle. In the case of the 'never do double writes' workaround
- * we have to lock out interrupts to be safe. As we don't care
- * of the value read we use an atomic rmw access to avoid costly
- * cli/sti. Otherwise we use an even cheaper single atomic write
- * to the APIC.
- */
- unsigned int cfg;
-
- /*
- * Wait for idle.
- */
- apic_wait_icr_idle();
-
- /*
- * No need to touch the target chip field
- */
- cfg = __prepare_ICR(shortcut, vector);
-
- /*
- * Send the IPI. The write to APIC_ICR fires this off.
- */
- apic_write_around(APIC_ICR, cfg);
-}
-
-void send_IPI_self(int vector)
-{
- __send_IPI_shortcut(APIC_DEST_SELF, vector);
-}
-
-/*
- * This is used to send an IPI with no shorthand notation (the destination is
- * specified in bits 56 to 63 of the ICR).
- */
-static inline void __send_IPI_dest_field(unsigned long mask, int vector)
-{
- unsigned long cfg;
-
- /*
- * Wait for idle.
- */
- if (unlikely(vector == NMI_VECTOR))
- safe_apic_wait_icr_idle();
- else
- apic_wait_icr_idle();
-
- /*
- * prepare target chip field
- */
- cfg = __prepare_ICR2(mask);
- apic_write_around(APIC_ICR2, cfg);
-
- /*
- * program the ICR
- */
- cfg = __prepare_ICR(0, vector);
-
- /*
- * Send the IPI. The write to APIC_ICR fires this off.
- */
- apic_write_around(APIC_ICR, cfg);
-}
-
-/*
- * This is only used on smaller machines.
- */
-void send_IPI_mask_bitmask(cpumask_t cpumask, int vector)
-{
- unsigned long mask = cpus_addr(cpumask)[0];
- unsigned long flags;
-
- local_irq_save(flags);
- WARN_ON(mask & ~cpus_addr(cpu_online_map)[0]);
- __send_IPI_dest_field(mask, vector);
- local_irq_restore(flags);
-}
-
-void send_IPI_mask_sequence(cpumask_t mask, int vector)
-{
- unsigned long flags;
- unsigned int query_cpu;
-
- /*
- * Hack. The clustered APIC addressing mode doesn't allow us to send
- * to an arbitrary mask, so I do a unicasts to each CPU instead. This
- * should be modified to do 1 message per cluster ID - mbligh
- */
-
- local_irq_save(flags);
- for_each_possible_cpu(query_cpu) {
- if (cpu_isset(query_cpu, mask)) {
- __send_IPI_dest_field(cpu_to_logical_apicid(query_cpu),
- vector);
- }
- }
- local_irq_restore(flags);
-}
-
#include <mach_ipi.h> /* must come after the send_IPI functions above for inlining */
/*
@@ -465,30 +339,3 @@ void flush_tlb_all(void)
{
on_each_cpu(do_flush_tlb_all, NULL, 1, 1);
}
-
-static int convert_apicid_to_cpu(int apic_id)
-{
- int i;
-
- for_each_possible_cpu(i) {
- if (per_cpu(x86_cpu_to_apicid, i) == apic_id)
- return i;
- }
- return -1;
-}
-
-int safe_smp_processor_id(void)
-{
- int apicid, cpuid;
-
- if (!boot_cpu_has(X86_FEATURE_APIC))
- return 0;
-
- apicid = hard_smp_processor_id();
- if (apicid == BAD_APICID)
- return 0;
-
- cpuid = convert_apicid_to_cpu(apicid);
-
- return cpuid >= 0 ? cpuid : 0;
-}
--
1.5.0.6
it is equal between architectures
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smpboot.c | 16 ++++++++++++++++
arch/x86/kernel/smpboot_32.c | 14 --------------
arch/x86/kernel/smpboot_64.c | 14 --------------
3 files changed, 16 insertions(+), 28 deletions(-)
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index d774520..644e609 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -1,6 +1,7 @@
#include <linux/init.h>
#include <linux/smp.h>
#include <linux/module.h>
+#include <linux/sched.h>
/* Number of siblings per CPU package */
int smp_num_siblings = 1;
@@ -95,6 +96,21 @@ void __cpuinit set_cpu_sibling_map(int cpu)
}
}
+/* maps the cpu to the sched domain representing multi-core */
+cpumask_t cpu_coregroup_map(int cpu)
+{
+ struct cpuinfo_x86 *c = &cpu_data(cpu);
+ /*
+ * For perf, we return last level cache shared map.
+ * And for power savings, we return cpu_core_map
+ */
+ if (sched_mc_power_savings || sched_smt_power_savings)
+ return per_cpu(cpu_core_map, cpu);
+ else
+ return c->llc_shared_map;
+}
+
+
#ifdef CONFIG_HOTPLUG_CPU
void remove_siblinginfo(int cpu)
{
diff --git a/arch/x86/kernel/smpboot_32.c b/arch/x86/kernel/smpboot_32.c
index 322f466..a58ca7f 100644
--- a/arch/x86/kernel/smpboot_32.c
+++ b/arch/x86/kernel/smpboot_32.c
@@ -260,20 +260,6 @@ static void __cpuinit smp_callin(void)
static int cpucount;
-/* maps the cpu to the sched domain representing multi-core */
-cpumask_t cpu_coregroup_map(int cpu)
-{
- struct cpuinfo_x86 *c = &cpu_data(cpu);
- /*
- * For perf, we return last level cache shared map.
- * And for power savings, we return cpu_core_map
- */
- if (sched_mc_power_savings || sched_smt_power_savings)
- return per_cpu(cpu_core_map, cpu);
- else
- return c->llc_shared_map;
-}
-
/*
* Activate a secondary processor.
*/
diff --git a/arch/x86/kernel/smpboot_64.c b/arch/x86/kernel/smpboot_64.c
index 329f9c5..1a59240 100644
--- a/arch/x86/kernel/smpboot_64.c
+++ b/arch/x86/kernel/smpboot_64.c
@@ -211,20 +211,6 @@ void __cpuinit smp_callin(void)
cpu_set(cpuid, cpu_callin_map);
}
-/* maps the cpu to the sched domain representing multi-core */
-cpumask_t cpu_coregroup_map(int cpu)
-{
- struct cpuinfo_x86 *c = &cpu_data(cpu);
- /*
- * For perf, we return last level cache shared map.
- * And for power savings, we return cpu_core_map
- */
- if (sched_mc_power_savings || sched_smt_power_savings)
- return per_cpu(cpu_core_map, cpu);
- else
- return c->llc_shared_map;
-}
-
/*
* Setup code on secondary processor (after comming out of the trampoline)
*/
--
1.5.0.6
This lock does not protect cpu_online_map, so its
length can be shortened, and in some cases, removed.
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smpboot_64.c | 4 +---
1 files changed, 1 insertions(+), 3 deletions(-)
diff --git a/arch/x86/kernel/smpboot_64.c b/arch/x86/kernel/smpboot_64.c
index 1a59240..ca3a3c5 100644
--- a/arch/x86/kernel/smpboot_64.c
+++ b/arch/x86/kernel/smpboot_64.c
@@ -261,9 +261,9 @@ void __cpuinit start_secondary(void)
/*
* Allow the master to continue.
*/
+ spin_unlock(&vector_lock);
cpu_set(smp_processor_id(), cpu_online_map);
per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
- spin_unlock(&vector_lock);
unlock_ipi_call_lock();
@@ -879,10 +879,8 @@ int __cpu_disable(void)
local_irq_disable();
remove_siblinginfo(cpu);
- spin_lock(&vector_lock);
/* It's now safe to remove this processor from the online map */
cpu_clear(cpu, cpu_online_map);
- spin_unlock(&vector_lock);
remove_cpu_from_maps();
fixup_irqs(cpu_online_map);
return 0;
--
1.5.0.6
They are now equal, and are moved to a common file
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smpboot.c | 85 ++++++++++++++++++++++++++++++++++++++++++
arch/x86/kernel/smpboot_32.c | 67 ---------------------------------
arch/x86/kernel/smpboot_64.c | 79 ---------------------------------------
include/asm-x86/smp.h | 3 +
include/asm-x86/smp_32.h | 3 -
include/asm-x86/smp_64.h | 3 -
6 files changed, 88 insertions(+), 152 deletions(-)
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 644e609..c35cd31 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -2,6 +2,13 @@
#include <linux/smp.h>
#include <linux/module.h>
#include <linux/sched.h>
+#include <linux/percpu.h>
+
+#include <asm/nmi.h>
+#include <asm/irq.h>
+#include <asm/smp.h>
+#include <asm/cpu.h>
+#include <asm/numa.h>
/* Number of siblings per CPU package */
int smp_num_siblings = 1;
@@ -181,5 +188,83 @@ __init void prefill_possible_map(void)
for (i = 0; i < possible; i++)
cpu_set(i, cpu_possible_map);
}
+
+static void __ref remove_cpu_from_maps(int cpu)
+{
+ cpu_clear(cpu, cpu_online_map);
+#ifdef CONFIG_X86_64
+ cpu_clear(cpu, cpu_callout_map);
+ cpu_clear(cpu, cpu_callin_map);
+ /* was set by cpu_init() */
+ clear_bit(cpu, (unsigned long *)&cpu_initialized);
+ clear_node_cpumask(cpu);
+#endif
+}
+
+int __cpu_disable(void)
+{
+ int cpu = smp_processor_id();
+
+ /*
+ * Perhaps use cpufreq to drop frequency, but that could go
+ * into generic code.
+ *
+ * We won't take down the boot processor on i386 due to some
+ * interrupts only being able to be serviced by the BSP.
+ * Especially so if we're not using an IOAPIC -zwane
+ */
+ if (cpu == 0)
+ return -EBUSY;
+
+ if (nmi_watchdog == NMI_LOCAL_APIC)
+ stop_apic_nmi_watchdog(NULL);
+ clear_local_APIC();
+
+ /*
+ * HACK:
+ * Allow any queued timer interrupts to get serviced
+ * This is only a temporary solution until we cleanup
+ * fixup_irqs as we do for IA64.
+ */
+ local_irq_enable();
+ mdelay(1);
+
+ local_irq_disable();
+ remove_siblinginfo(cpu);
+
+ /* It's now safe to remove this processor from the online map */
+ remove_cpu_from_maps(cpu);
+ fixup_irqs(cpu_online_map);
+ return 0;
+}
+
+void __cpu_die(unsigned int cpu)
+{
+ /* We don't do anything here: idle task is faking death itself. */
+ unsigned int i;
+
+ for (i = 0; i < 10; i++) {
+ /* They ack this in play_dead by setting CPU_DEAD */
+ if (per_cpu(cpu_state, cpu) == CPU_DEAD) {
+ printk(KERN_INFO "CPU %d is now offline\n", cpu);
+ if (1 == num_online_cpus())
+ alternatives_smp_switch(0);
+ return;
+ }
+ msleep(100);
+ }
+ printk(KERN_ERR "CPU %u didn't die...\n", cpu);
+}
+#else /* ... !CONFIG_HOTPLUG_CPU */
+int __cpu_disable(void)
+{
+ return -ENOSYS;
+}
+
+void __cpu_die(unsigned int cpu)
+{
+ /* We said "no" in __cpu_disable */
+ BUG();
+}
#endif
diff --git a/arch/x86/kernel/smpboot_32.c b/arch/x86/kernel/smpboot_32.c
index 3d21c66..00b1b59 100644
--- a/arch/x86/kernel/smpboot_32.c
+++ b/arch/x86/kernel/smpboot_32.c
@@ -1040,73 +1040,6 @@ void __init native_smp_prepare_boot_cpu(void)
__get_cpu_var(cpu_state) = CPU_ONLINE;
}
-#ifdef CONFIG_HOTPLUG_CPU
-static void __ref remove_cpu_from_maps(int cpu)
-{
- cpu_clear(cpu, cpu_online_map);
-}
-
-int __cpu_disable(void)
-{
- cpumask_t map = cpu_online_map;
- int cpu = smp_processor_id();
-
- /*
- * Perhaps use cpufreq to drop frequency, but that could go
- * into generic code.
- *
- * We won't take down the boot processor on i386 due to some
- * interrupts only being able to be serviced by the BSP.
- * Especially so if we're not using an IOAPIC -zwane
- */
- if (cpu == 0)
- return -EBUSY;
- if (nmi_watchdog == NMI_LOCAL_APIC)
- stop_apic_nmi_watchdog(NULL);
- clear_local_APIC();
- /* Allow any queued timer interrupts to get serviced */
- local_irq_enable();
- mdelay(1);
- local_irq_disable();
-
- remove_siblinginfo(cpu);
-
- remove_cpu_from_maps(cpu);
- fixup_irqs(map);
-
- return 0;
-}
-
-void __cpu_die(unsigned int cpu)
-{
- /* We don't do anything here: idle task is faking death itself. */
- unsigned int i;
-
- for (i = 0; i < 10; i++) {
- /* They ack this in play_dead by setting CPU_DEAD */
- if (per_cpu(cpu_state, cpu) == CPU_DEAD) {
- printk ("CPU %d is now offline\n", cpu);
- if (1 == num_online_cpus())
- alternatives_smp_switch(0);
- return;
- }
- msleep(100);
- }
- printk(KERN_ERR "CPU %u didn't die...\n", cpu);
-}
-#else /* ... !CONFIG_HOTPLUG_CPU */
-int __cpu_disable(void)
-{
- return -ENOSYS;
-}
-
-void __cpu_die(unsigned int cpu)
-{
- /* We said "no" in __cpu_disable */
- BUG();
-}
-#endif /* CONFIG_HOTPLUG_CPU */
-
int __cpuinit native_cpu_up(unsigned int cpu)
{
unsigned long flags;
diff --git a/arch/x86/kernel/smpboot_64.c b/arch/x86/kernel/smpboot_64.c
index 6509d3c..0c67e5a 100644
--- a/arch/x86/kernel/smpboot_64.c
+++ b/arch/x86/kernel/smpboot_64.c
@@ -836,82 +836,3 @@ void __init native_smp_cpus_done(unsigned int max_cpus)
setup_ioapic_dest();
check_nmi_watchdog();
}
-
-#ifdef CONFIG_HOTPLUG_CPU
-static void __ref remove_cpu_from_maps(int cpu)
-{
- cpu_clear(cpu, cpu_online_map);
- cpu_clear(cpu, cpu_callout_map);
- cpu_clear(cpu, cpu_callin_map);
- clear_bit(cpu, (unsigned long *)&cpu_initialized); /* was set by cpu_init() */
- clear_node_cpumask(cpu);
-}
-
-int __cpu_disable(void)
-{
- int cpu = smp_processor_id();
-
- /*
- * Perhaps use cpufreq to drop frequency, but that could go
- * into generic code.
- *
- * We won't take down the boot processor on i386 due to some
- * interrupts only being able to be serviced by the BSP.
- * Especially so if we're not using an IOAPIC -zwane
- */
- if (cpu == 0)
- return -EBUSY;
-
- if (nmi_watchdog == NMI_LOCAL_APIC)
- stop_apic_nmi_watchdog(NULL);
- clear_local_APIC();
-
- /*
- * HACK:
- * Allow any queued timer interrupts to get serviced
- * This is only a temporary solution until we cleanup
- * fixup_irqs as we do for IA64.
- */
- local_irq_enable();
- mdelay(1);
-
- local_irq_disable();
- remove_siblinginfo(cpu);
-
- /* It's now safe to remove this processor from the online map */
- remove_cpu_from_maps(cpu);
- fixup_irqs(cpu_online_map);
- return 0;
-}
-
-void __cpu_die(unsigned int cpu)
-{
- /* We don't do anything here: idle task is faking death itself. */
- unsigned int i;
-
- for (i = 0; i < 10; i++) {
- /* They ack this in play_dead by setting CPU_DEAD */
- if (per_cpu(cpu_state, cpu) == CPU_DEAD) {
- printk ("CPU %d is now offline\n", cpu);
- if (1 == num_online_cpus())
- alternatives_smp_switch(0);
- return;
- }
- msleep(100);
- }
- printk(KERN_ERR "CPU %u didn't die...\n", cpu);
-}
-
-#else /* ... !CONFIG_HOTPLUG_CPU */
-
-int __cpu_disable(void)
-{
- return -ENOSYS;
-}
-
-void __cpu_die(unsigned int cpu)
-{
- /* We said "no" in __cpu_disable */
- BUG();
-}
-#endif /* CONFIG_HOTPLUG_CPU */
diff --git a/include/asm-x86/smp.h b/include/asm-x86/smp.h
index c800b81..27d9f65 100644
--- a/include/asm-x86/smp.h
+++ b/include/asm-x86/smp.h
@@ -70,6 +70,9 @@ void native_smp_prepare_cpus(unsigned int max_cpus);
void native_smp_cpus_done(unsigned int max_cpus);
int native_cpu_up(unsigned int cpunum);
+extern int __cpu_disable(void);
+extern void __cpu_die(unsigned int cpu);
+
extern unsigned disabled_cpus;
extern void prefill_possible_map(void);
#endif
diff --git a/include/asm-x86/smp_32.h b/include/asm-x86/smp_32.h
index 0b25134..4fec2fe 100644
--- a/include/asm-x86/smp_32.h
+++ b/include/asm-x86/smp_32.h
@@ -34,9 +34,6 @@ DECLARE_PER_CPU(u16, x86_cpu_to_apicid);
#define startup_ipi_hook(phys_apicid, start_eip, start_esp) do { } while (0)
#endif
-extern int __cpu_disable(void);
-extern void __cpu_die(unsigned int cpu);
-
/*
* This function is needed by all SMP systems. It must _always_ be valid
* from the initial startup. We map APIC_BASE very early in page_setup(),
diff --git a/include/asm-x86/smp_64.h b/include/asm-x86/smp_64.h
index 1ecf813..d554d7d 100644
--- a/include/asm-x86/smp_64.h
+++ b/include/asm-x86/smp_64.h
@@ -42,9 +42,6 @@ static inline int cpu_present_to_apicid(int mps_cpu)
#define SMP_TRAMPOLINE_BASE 0x6000
-extern int __cpu_disable(void);
-extern void __cpu_die(unsigned int cpu);
-
#define raw_smp_processor_id() read_pda(cpunumber)
#define cpu_physical_id(cpu) per_cpu(x86_cpu_to_apicid, cpu)
--
1.5.0.6
The parameter passing parsing is done in the common smpboot.c
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/mpparse_64.c | 7 +++++++
arch/x86/kernel/smpboot.c | 12 ++++++++++++
arch/x86/kernel/smpboot_32.c | 13 -------------
3 files changed, 19 insertions(+), 13 deletions(-)
diff --git a/arch/x86/kernel/mpparse_64.c b/arch/x86/kernel/mpparse_64.c
index 72ab140..2a1f788 100644
--- a/arch/x86/kernel/mpparse_64.c
+++ b/arch/x86/kernel/mpparse_64.c
@@ -32,6 +32,7 @@
/* Have we found an MP table */
int smp_found_config;
+unsigned int __cpuinitdata maxcpus = NR_CPUS;
/*
* Various Linux-internal data structures created from the
@@ -115,6 +116,12 @@ static void __cpuinit MP_processor_info(struct mpc_config_processor *m)
return;
}
+ if (num_processors >= maxcpus) {
+ printk(KERN_WARNING "WARNING: maxcpus limit of %i reached."
+ " Processor ignored.\n", maxcpus);
+ return;
+ }
+
num_processors++;
cpus_complement(tmp_map, cpu_present_map);
cpu = first_cpu(tmp_map);
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index c35cd31..34c3117 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -268,3 +268,15 @@ void __cpu_die(unsigned int cpu)
}
#endif
+/*
+ * If the BIOS enumerates physical processors before logical,
+ * maxcpus=N at enumeration-time can be used to disable HT.
+ */
+static int __init parse_maxcpus(char *arg)
+{
+ extern unsigned int maxcpus;
+
+ maxcpus = simple_strtoul(arg, NULL, 0);
+ return 0;
+}
+early_param("maxcpus", parse_maxcpus);
diff --git a/arch/x86/kernel/smpboot_32.c b/arch/x86/kernel/smpboot_32.c
index 00b1b59..3236e84 100644
--- a/arch/x86/kernel/smpboot_32.c
+++ b/arch/x86/kernel/smpboot_32.c
@@ -1113,16 +1113,3 @@ void __init smp_intr_init(void)
/* IPI for generic function call */
set_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt);
}
-
-/*
- * If the BIOS enumerates physical processors before logical,
- * maxcpus=N at enumeration-time can be used to disable HT.
- */
-static int __init parse_maxcpus(char *arg)
-{
- extern unsigned int maxcpus;
-
- maxcpus = simple_strtoul(arg, NULL, 0);
- return 0;
-}
-early_param("maxcpus", parse_maxcpus);
--
1.5.0.6
it was already cleared two lines above, and so, this removal
is bogus
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smpboot_32.c | 3 +--
1 files changed, 1 insertions(+), 2 deletions(-)
diff --git a/arch/x86/kernel/smpboot_32.c b/arch/x86/kernel/smpboot_32.c
index 4939b3a..3d21c66 100644
--- a/arch/x86/kernel/smpboot_32.c
+++ b/arch/x86/kernel/smpboot_32.c
@@ -1073,8 +1073,7 @@ int __cpu_disable(void)
remove_cpu_from_maps(cpu);
fixup_irqs(map);
- /* It's now safe to remove this processor from the online map */
- cpu_clear(cpu, cpu_online_map);
+
return 0;
}
--
1.5.0.6
In here, they can serve both architectures
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smpboot_32.c | 6 ------
arch/x86/kernel/smpboot_64.c | 7 -------
include/asm-x86/smp.h | 6 ++++++
3 files changed, 6 insertions(+), 13 deletions(-)
diff --git a/arch/x86/kernel/smpboot_32.c b/arch/x86/kernel/smpboot_32.c
index 3236e84..a21f254 100644
--- a/arch/x86/kernel/smpboot_32.c
+++ b/arch/x86/kernel/smpboot_32.c
@@ -73,12 +73,6 @@ EXPORT_PER_CPU_SYMBOL(x86_cpu_to_apicid);
u8 apicid_2_node[MAX_APICID];
-/*
- * Trampoline 80x86 program as an array.
- */
-
-extern const unsigned char trampoline_data [];
-extern const unsigned char trampoline_end [];
static unsigned char *trampoline_base;
static void map_cpu_to_logical_apicid(void);
diff --git a/arch/x86/kernel/smpboot_64.c b/arch/x86/kernel/smpboot_64.c
index 0c67e5a..2cc1b8b 100644
--- a/arch/x86/kernel/smpboot_64.c
+++ b/arch/x86/kernel/smpboot_64.c
@@ -63,13 +63,6 @@
/* Set when the idlers are all forked */
int smp_threads_ready;
-/*
- * Trampoline 80x86 program as an array.
- */
-
-extern const unsigned char trampoline_data[];
-extern const unsigned char trampoline_end[];
-
/* State of each CPU */
DEFINE_PER_CPU(int, cpu_state) = { 0 };
diff --git a/include/asm-x86/smp.h b/include/asm-x86/smp.h
index 27d9f65..b2a1697 100644
--- a/include/asm-x86/smp.h
+++ b/include/asm-x86/smp.h
@@ -9,6 +9,12 @@ extern cpumask_t cpu_callout_map;
extern int smp_num_siblings;
extern unsigned int num_processors;
+/*
+ * Trampoline 80x86 program as an array.
+ */
+extern const unsigned char trampoline_data [];
+extern const unsigned char trampoline_end [];
+
struct smp_ops {
void (*smp_prepare_boot_cpu)(void);
void (*smp_prepare_cpus)(unsigned max_cpus);
--
1.5.0.6
setup_trampoline() looks very similar between architectures, and this
patch unifies them. The i386 version allocates bootmem memory, while
the x86_64 version uses a fixed address.
In this patch, we initialize the global trampoline_base to the x86_64 version,
and i386 allocation can later override it.
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smpboot.c | 33 +++++++++++++++++++++++++++++++++
arch/x86/kernel/smpboot_32.c | 29 -----------------------------
arch/x86/kernel/smpboot_64.c | 14 --------------
include/asm-x86/smp.h | 4 ++++
include/asm-x86/smp_64.h | 2 --
5 files changed, 37 insertions(+), 45 deletions(-)
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 34c3117..b13b9d5 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -3,6 +3,7 @@
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/percpu.h>
+#include <linux/bootmem.h>
#include <asm/nmi.h>
#include <asm/irq.h>
@@ -38,6 +39,9 @@ EXPORT_PER_CPU_SYMBOL(cpu_core_map);
DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);
EXPORT_PER_CPU_SYMBOL(cpu_info);
+/* ready for x86_64, no harm for x86, since it will overwrite after alloc */
+unsigned char *trampoline_base = __va(SMP_TRAMPOLINE_BASE);
+
/* representing cpus for which sibling maps can be computed */
static cpumask_t cpu_sibling_setup_map;
@@ -117,6 +121,35 @@ cpumask_t cpu_coregroup_map(int cpu)
return c->llc_shared_map;
}
+/*
+ * Currently trivial. Write the real->protected mode
+ * bootstrap into the page concerned. The caller
+ * has made sure it's suitably aligned.
+ */
+
+unsigned long __cpuinit setup_trampoline(void)
+{
+ memcpy(trampoline_base, trampoline_data,
+ trampoline_end - trampoline_data);
+ return virt_to_phys(trampoline_base);
+}
+
+#ifdef CONFIG_X86_32
+/*
+ * We are called very early to get the low memory for the
+ * SMP bootup trampoline page.
+ */
+void __init smp_alloc_memory(void)
+{
+ trampoline_base = alloc_bootmem_low_pages(PAGE_SIZE);
+ /*
+ * Has to be in very low memory so we can execute
+ * real-mode AP code.
+ */
+ if (__pa(trampoline_base) >= 0x9F000)
+ BUG();
+}
+#endif
#ifdef CONFIG_HOTPLUG_CPU
void remove_siblinginfo(int cpu)
diff --git a/arch/x86/kernel/smpboot_32.c b/arch/x86/kernel/smpboot_32.c
index a21f254..ee82659 100644
--- a/arch/x86/kernel/smpboot_32.c
+++ b/arch/x86/kernel/smpboot_32.c
@@ -73,41 +73,12 @@ EXPORT_PER_CPU_SYMBOL(x86_cpu_to_apicid);
u8 apicid_2_node[MAX_APICID];
-static unsigned char *trampoline_base;
-
static void map_cpu_to_logical_apicid(void);
/* State of each CPU. */
DEFINE_PER_CPU(int, cpu_state) = { 0 };
/*
- * Currently trivial. Write the real->protected mode
- * bootstrap into the page concerned. The caller
- * has made sure it's suitably aligned.
- */
-
-static unsigned long __cpuinit setup_trampoline(void)
-{
- memcpy(trampoline_base, trampoline_data, trampoline_end - trampoline_data);
- return virt_to_phys(trampoline_base);
-}
-
-/*
- * We are called very early to get the low memory for the
- * SMP bootup trampoline page.
- */
-void __init smp_alloc_memory(void)
-{
- trampoline_base = alloc_bootmem_low_pages(PAGE_SIZE);
- /*
- * Has to be in very low memory so we can execute
- * real-mode AP code.
- */
- if (__pa(trampoline_base) >= 0x9F000)
- BUG();
-}
-
-/*
* The bootstrap kernel entry code has set these up. Save them for
* a given CPU
*/
diff --git a/arch/x86/kernel/smpboot_64.c b/arch/x86/kernel/smpboot_64.c
index 2cc1b8b..9f4935e 100644
--- a/arch/x86/kernel/smpboot_64.c
+++ b/arch/x86/kernel/smpboot_64.c
@@ -85,20 +85,6 @@ struct task_struct *idle_thread_array[NR_CPUS] __cpuinitdata ;
#define set_idle_for_cpu(x,p) (idle_thread_array[(x)] = (p))
#endif
-
-/*
- * Currently trivial. Write the real->protected mode
- * bootstrap into the page concerned. The caller
- * has made sure it's suitably aligned.
- */
-
-static unsigned long __cpuinit setup_trampoline(void)
-{
- void *tramp = __va(SMP_TRAMPOLINE_BASE);
- memcpy(tramp, trampoline_data, trampoline_end - trampoline_data);
- return virt_to_phys(tramp);
-}
-
/*
* The bootstrap kernel entry code has set these up. Save them for
* a given CPU
diff --git a/include/asm-x86/smp.h b/include/asm-x86/smp.h
index b2a1697..513c857 100644
--- a/include/asm-x86/smp.h
+++ b/include/asm-x86/smp.h
@@ -14,6 +14,7 @@ extern unsigned int num_processors;
*/
extern const unsigned char trampoline_data [];
extern const unsigned char trampoline_end [];
+extern unsigned char *trampoline_base;
struct smp_ops {
void (*smp_prepare_boot_cpu)(void);
@@ -81,6 +82,9 @@ extern void __cpu_die(unsigned int cpu);
extern unsigned disabled_cpus;
extern void prefill_possible_map(void);
+
+#define SMP_TRAMPOLINE_BASE 0x6000
+extern unsigned long setup_trampoline(void);
#endif
#ifdef CONFIG_X86_32
diff --git a/include/asm-x86/smp_64.h b/include/asm-x86/smp_64.h
index d554d7d..394c785 100644
--- a/include/asm-x86/smp_64.h
+++ b/include/asm-x86/smp_64.h
@@ -40,8 +40,6 @@ static inline int cpu_present_to_apicid(int mps_cpu)
#ifdef CONFIG_SMP
-#define SMP_TRAMPOLINE_BASE 0x6000
-
#define raw_smp_processor_id() read_pda(cpunumber)
#define cpu_physical_id(cpu) per_cpu(x86_cpu_to_apicid, cpu)
--
1.5.0.6
make setup_trampoline non-static. This way, it won't conflict
with the extern declaration in smp.h
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/mach-voyager/voyager_smp.c | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/arch/x86/mach-voyager/voyager_smp.c b/arch/x86/mach-voyager/voyager_smp.c
index 0ab236e..9ff687a 100644
--- a/arch/x86/mach-voyager/voyager_smp.c
+++ b/arch/x86/mach-voyager/voyager_smp.c
@@ -429,7 +429,7 @@ void __init smp_store_cpu_info(int id)
}
/* set up the trampoline and return the physical address of the code */
-static __u32 __init setup_trampoline(void)
+unsigned long __init setup_trampoline(void)
{
/* these two are global symbols in trampoline.S */
extern const __u8 trampoline_end[];
--
1.5.0.6
wraps the busy loop for wait_for_init_deasserted() in a function,
so smp_callin in x86_64 looks like more i386
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smpboot_64.c | 10 ++++++++--
1 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/arch/x86/kernel/smpboot_64.c b/arch/x86/kernel/smpboot_64.c
index 9f4935e..4f6d976 100644
--- a/arch/x86/kernel/smpboot_64.c
+++ b/arch/x86/kernel/smpboot_64.c
@@ -100,6 +100,13 @@ static void __cpuinit smp_store_cpu_info(int id)
print_cpu_info(c);
}
+static inline void wait_for_init_deassert(atomic_t *deassert)
+{
+ while (!atomic_read(deassert))
+ cpu_relax();
+ return;
+}
+
static atomic_t init_deasserted __cpuinitdata;
/*
@@ -117,8 +124,7 @@ void __cpuinit smp_callin(void)
* our local APIC. We have to wait for the IPI or we'll
* lock up on an APIC access.
*/
- while (!atomic_read(&init_deasserted))
- cpu_relax();
+ wait_for_init_deassert(&init_deasserted);
/*
* (This works even if the APIC is not enabled.)
--
1.5.0.6
Change voyager's trampoline base to unsigned char *
instead of u32. This way, it won't conflict with
the other architectures when including smp.h
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/mach-voyager/voyager_smp.c | 8 ++++----
1 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/arch/x86/mach-voyager/voyager_smp.c b/arch/x86/mach-voyager/voyager_smp.c
index 0b3f52a..0ab236e 100644
--- a/arch/x86/mach-voyager/voyager_smp.c
+++ b/arch/x86/mach-voyager/voyager_smp.c
@@ -210,7 +210,7 @@ static int cpucount = 0;
/* steal a page from the bottom of memory for the trampoline and
* squirrel its address away here. This will be in kernel virtual
* space */
-static __u32 trampoline_base;
+unsigned char *trampoline_base;
/* The per cpu profile stuff - used in smp_local_timer_interrupt */
static DEFINE_PER_CPU(int, prof_multiplier) = 1;
@@ -435,9 +435,9 @@ static __u32 __init setup_trampoline(void)
extern const __u8 trampoline_end[];
extern const __u8 trampoline_data[];
- memcpy((__u8 *) trampoline_base, trampoline_data,
+ memcpy(trampoline_base, trampoline_data,
trampoline_end - trampoline_data);
- return virt_to_phys((__u8 *) trampoline_base);
+ return virt_to_phys(trampoline_base);
}
/* Routine initially called when a non-boot CPU is brought online */
@@ -1166,7 +1166,7 @@ void flush_tlb_all(void)
* is sorted out */
void __init smp_alloc_memory(void)
{
- trampoline_base = (__u32) alloc_bootmem_low_pages(PAGE_SIZE);
+ trampoline_base = alloc_bootmem_low_pages(PAGE_SIZE);
if (__pa(trampoline_base) >= 0x93000)
BUG();
}
--
1.5.0.6
This is done for smpboot_32.c
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smpboot_32.c | 4 ++--
1 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/arch/x86/kernel/smpboot_32.c b/arch/x86/kernel/smpboot_32.c
index ee82659..2dd95ba 100644
--- a/arch/x86/kernel/smpboot_32.c
+++ b/arch/x86/kernel/smpboot_32.c
@@ -185,7 +185,7 @@ static void __cpuinit smp_callin(void)
*/
if (cpu_isset(cpuid, cpu_callout_map))
break;
- rep_nop();
+ cpu_relax();
}
if (!time_before(jiffies, timeout)) {
@@ -242,7 +242,7 @@ static void __cpuinit start_secondary(void *unused)
preempt_disable();
smp_callin();
while (!cpu_isset(smp_processor_id(), smp_commenced_mask))
- rep_nop();
+ cpu_relax();
/*
* Check TSC synchronization with the BP:
*/
--
1.5.0.6
In x86_64, hlt always work. in i386, we'll query the cpuinfo associated
with this cpu
Signed-off-by: Glauber Costa <[email protected]>
---
include/asm-x86/processor.h | 9 +++++++++
1 files changed, 9 insertions(+), 0 deletions(-)
diff --git a/include/asm-x86/processor.h b/include/asm-x86/processor.h
index 9054734..8bec23c 100644
--- a/include/asm-x86/processor.h
+++ b/include/asm-x86/processor.h
@@ -144,6 +144,15 @@ DECLARE_PER_CPU(struct cpuinfo_x86, cpu_info);
#define current_cpu_data boot_cpu_data
#endif
+static inline int hlt_works(int cpu)
+{
+#ifdef CONFIG_X86_32
+ return cpu_data(cpu).hlt_works_ok;
+#else
+ return 1;
+#endif
+}
+
#define cache_line_size() (boot_cpu_data.x86_cache_alignment)
extern void cpu_detect(struct cpuinfo_x86 *c);
--
1.5.0.6
function definition is moved to common header.
x86_64 version is now called native_smp_send_stop
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smp_64.c | 3 ++-
include/asm-x86/smp.h | 5 +++++
include/asm-x86/smp_32.h | 5 -----
3 files changed, 7 insertions(+), 6 deletions(-)
diff --git a/arch/x86/kernel/smp_64.c b/arch/x86/kernel/smp_64.c
index 4e1e2bc..ad11ef0 100644
--- a/arch/x86/kernel/smp_64.c
+++ b/arch/x86/kernel/smp_64.c
@@ -421,7 +421,7 @@ static void stop_this_cpu(void *dummy)
for (;;);
}
-void smp_send_stop(void)
+void native_smp_send_stop(void)
{
int nolock;
unsigned long flags;
@@ -482,6 +482,7 @@ struct smp_ops smp_ops = {
.smp_prepare_cpus = native_smp_prepare_cpus,
.smp_cpus_done = native_smp_cpus_done,
+ .smp_send_stop = native_smp_send_stop,
.smp_send_reschedule = native_smp_send_reschedule,
.smp_call_function_mask = native_smp_call_function_mask,
.cpu_up = native_cpu_up,
diff --git a/include/asm-x86/smp.h b/include/asm-x86/smp.h
index 28cb1f8..2ab8ed4 100644
--- a/include/asm-x86/smp.h
+++ b/include/asm-x86/smp.h
@@ -25,6 +25,11 @@ struct smp_ops {
#ifdef CONFIG_SMP
extern struct smp_ops smp_ops;
+static inline void smp_send_stop(void)
+{
+ smp_ops.smp_send_stop();
+}
+
static inline void smp_prepare_boot_cpu(void)
{
smp_ops.smp_prepare_boot_cpu();
diff --git a/include/asm-x86/smp_32.h b/include/asm-x86/smp_32.h
index bc90a4e..41b58e0 100644
--- a/include/asm-x86/smp_32.h
+++ b/include/asm-x86/smp_32.h
@@ -39,11 +39,6 @@ extern void remove_siblinginfo(int cpu);
extern void set_cpu_sibling_map(int cpu);
#ifdef CONFIG_SMP
-static inline void smp_send_stop(void)
-{
- smp_ops.smp_send_stop();
-}
-
#ifndef CONFIG_PARAVIRT
#define startup_ipi_hook(phys_apicid, start_eip, start_esp) do { } while (0)
#endif
--
1.5.0.6
This can be safely added to i386. After that,
functions look exactly the same for both arches
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smp_32.c | 9 +++++++--
1 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/arch/x86/kernel/smp_32.c b/arch/x86/kernel/smp_32.c
index cde3a0e..8be3e09 100644
--- a/arch/x86/kernel/smp_32.c
+++ b/arch/x86/kernel/smp_32.c
@@ -24,6 +24,7 @@
#include <asm/tlbflush.h>
#include <asm/mmu_context.h>
#include <mach_apic.h>
+#include <asm/proto.h>
/*
* Some notes on x86 processor bugs affecting SMP operation:
@@ -622,10 +623,14 @@ static void stop_this_cpu (void * dummy)
static void native_smp_send_stop(void)
{
- /* Don't deadlock on the call lock in panic */
- int nolock = !spin_trylock(&call_lock);
+ int nolock;
unsigned long flags;
+ if (reboot_force)
+ return;
+
+ /* Don't deadlock on the call lock in panic */
+ nolock = !spin_trylock(&call_lock);
local_irq_save(flags);
__smp_call_function(stop_this_cpu, NULL, 0, 0);
if (!nolock)
--
1.5.0.6
This function is used in smp_send_stop(). It's like
smp_call_function_mask, but always go to all online cpus,
and does not take any locks.
It is added to x86_64, but will soon be unified in a common file
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smp_64.c | 34 +++++++++++++++++++++++++++++++++-
1 files changed, 33 insertions(+), 1 deletions(-)
diff --git a/arch/x86/kernel/smp_64.c b/arch/x86/kernel/smp_64.c
index 1d8b863..aa2edb7 100644
--- a/arch/x86/kernel/smp_64.c
+++ b/arch/x86/kernel/smp_64.c
@@ -322,6 +322,38 @@ void unlock_ipi_call_lock(void)
spin_unlock_irq(&call_lock);
}
+static void __smp_call_function(void (*func) (void *info), void *info,
+ int nonatomic, int wait)
+{
+ struct call_data_struct data;
+ int cpus = num_online_cpus() - 1;
+
+ if (!cpus)
+ return;
+
+ data.func = func;
+ data.info = info;
+ atomic_set(&data.started, 0);
+ data.wait = wait;
+ if (wait)
+ atomic_set(&data.finished, 0);
+
+ call_data = &data;
+ mb();
+
+ /* Send a message to all other CPUs and wait for them to respond */
+ send_IPI_allbutself(CALL_FUNCTION_VECTOR);
+
+ /* Wait for response */
+ while (atomic_read(&data.started) != cpus)
+ cpu_relax();
+
+ if (wait)
+ while (atomic_read(&data.finished) != cpus)
+ cpu_relax();
+}
+
+
/*
* this function sends a 'generic call function' IPI to all other CPU
* of the system defined in the mask.
@@ -424,7 +456,7 @@ void smp_send_stop(void)
/* Don't deadlock on the call lock in panic */
nolock = !spin_trylock(&call_lock);
local_irq_save(flags);
- __smp_call_function_mask(cpu_online_map, stop_this_cpu, NULL, 0);
+ __smp_call_function(stop_this_cpu, NULL, 0, 0);
if (!nolock)
spin_unlock(&call_lock);
disable_local_APIC();
--
1.5.0.6
This patch merges the copyright notices, and valuable
comments that were left back on smp_{32,64}.c. With that,
files are empty, and are deleted
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/Makefile | 2 +-
arch/x86/kernel/smp.c | 91 +++++++++++++++++++++++++++++++++++++++
arch/x86/kernel/smp_32.c | 106 ----------------------------------------------
arch/x86/kernel/smp_64.c | 10 ----
4 files changed, 92 insertions(+), 117 deletions(-)
delete mode 100644 arch/x86/kernel/smp_32.c
delete mode 100644 arch/x86/kernel/smp_64.c
Index: linux-2.6-x86/arch/x86/kernel/Makefile
===================================================================
--- linux-2.6-x86.orig/arch/x86/kernel/Makefile 2008-03-03 11:58:38.000000000 -0800
+++ linux-2.6-x86/arch/x86/kernel/Makefile 2008-03-03 11:59:55.000000000 -0800
@@ -46,10 +46,10 @@
obj-$(CONFIG_PCI) += early-quirks.o
apm-y := apm_32.o
obj-$(CONFIG_APM) += apm.o
-obj-$(CONFIG_X86_SMP) += smp_$(BITS).o smpboot_$(BITS).o smp.o
+obj-$(CONFIG_X86_SMP) += smpboot_$(BITS).o smp.o
obj-$(CONFIG_X86_SMP) += smpboot.o tsc_sync.o ipi.o tlb_$(BITS).o
obj-$(CONFIG_X86_32_SMP) += smpcommon.o
-obj-$(CONFIG_X86_64_SMP) += smp_64.o smpboot_64.o tsc_sync.o smpcommon.o
+obj-$(CONFIG_X86_64_SMP) += smpboot_64.o tsc_sync.o smpcommon.o
obj-$(CONFIG_X86_TRAMPOLINE) += trampoline_$(BITS).o
obj-$(CONFIG_X86_MPPARSE) += mpparse_$(BITS).o
obj-$(CONFIG_X86_LOCAL_APIC) += apic_$(BITS).o nmi_$(BITS).o
Index: linux-2.6-x86/arch/x86/kernel/smp.c
===================================================================
--- linux-2.6-x86.orig/arch/x86/kernel/smp.c 2008-03-03 11:58:38.000000000 -0800
+++ linux-2.6-x86/arch/x86/kernel/smp.c 2008-03-03 11:58:38.000000000 -0800
@@ -1,3 +1,16 @@
+/*
+ * Intel SMP support routines.
+ *
+ * (c) 1995 Alan Cox, Building #3 <[email protected]>
+ * (c) 1998-99, 2000 Ingo Molnar <[email protected]>
+ * (c) 2002,2003 Andi Kleen, SuSE Labs.
+ *
+ * i386 and x86_64 integration by Glauber Costa <[email protected]>
+ *
+ * This code is released under the GNU General Public License version 2 or
+ * later.
+ */
+
#include <linux/init.h>
#include <linux/mm.h>
@@ -19,6 +32,84 @@
#else
#include <asm/mach_apic.h>
#endif
+/*
+ * Some notes on x86 processor bugs affecting SMP operation:
+ *
+ * Pentium, Pentium Pro, II, III (and all CPUs) have bugs.
+ * The Linux implications for SMP are handled as follows:
+ *
+ * Pentium III / [Xeon]
+ * None of the E1AP-E3AP errata are visible to the user.
+ *
+ * E1AP. see PII A1AP
+ * E2AP. see PII A2AP
+ * E3AP. see PII A3AP
+ *
+ * Pentium II / [Xeon]
+ * None of the A1AP-A3AP errata are visible to the user.
+ *
+ * A1AP. see PPro 1AP
+ * A2AP. see PPro 2AP
+ * A3AP. see PPro 7AP
+ *
+ * Pentium Pro
+ * None of 1AP-9AP errata are visible to the normal user,
+ * except occasional delivery of 'spurious interrupt' as trap #15.
+ * This is very rare and a non-problem.
+ *
+ * 1AP. Linux maps APIC as non-cacheable
+ * 2AP. worked around in hardware
+ * 3AP. fixed in C0 and above steppings microcode update.
+ * Linux does not use excessive STARTUP_IPIs.
+ * 4AP. worked around in hardware
+ * 5AP. symmetric IO mode (normal Linux operation) not affected.
+ * 'noapic' mode has vector 0xf filled out properly.
+ * 6AP. 'noapic' mode might be affected - fixed in later steppings
+ * 7AP. We do not assume writes to the LVT deassering IRQs
+ * 8AP. We do not enable low power mode (deep sleep) during MP bootup
+ * 9AP. We do not use mixed mode
+ *
+ * Pentium
+ * There is a marginal case where REP MOVS on 100MHz SMP
+ * machines with B stepping processors can fail. XXX should provide
+ * an L1cache=Writethrough or L1cache=off option.
+ *
+ * B stepping CPUs may hang. There are hardware work arounds
+ * for this. We warn about it in case your board doesn't have the work
+ * arounds. Basically that's so I can tell anyone with a B stepping
+ * CPU and SMP problems "tough".
+ *
+ * Specific items [From Pentium Processor Specification Update]
+ *
+ * 1AP. Linux doesn't use remote read
+ * 2AP. Linux doesn't trust APIC errors
+ * 3AP. We work around this
+ * 4AP. Linux never generated 3 interrupts of the same priority
+ * to cause a lost local interrupt.
+ * 5AP. Remote read is never used
+ * 6AP. not affected - worked around in hardware
+ * 7AP. not affected - worked around in hardware
+ * 8AP. worked around in hardware - we get explicit CS errors if not
+ * 9AP. only 'noapic' mode affected. Might generate spurious
+ * interrupts, we log only the first one and count the
+ * rest silently.
+ * 10AP. not affected - worked around in hardware
+ * 11AP. Linux reads the APIC between writes to avoid this, as per
+ * the documentation. Make sure you preserve this as it affects
+ * the C stepping chips too.
+ * 12AP. not affected - worked around in hardware
+ * 13AP. not affected - worked around in hardware
+ * 14AP. we always deassert INIT during bootup
+ * 15AP. not affected - worked around in hardware
+ * 16AP. not affected - worked around in hardware
+ * 17AP. not affected - worked around in hardware
+ * 18AP. not affected - worked around in hardware
+ * 19AP. not affected - worked around in BIOS
+ *
+ * If this sounds worrying believe me these bugs are either ___RARE___,
+ * or are signal timing bugs worked around in hardware and there's
+ * about nothing of note with C stepping upwards.
+ */
/*
* this function sends a 'reschedule' IPI to another CPU.
Index: linux-2.6-x86/arch/x86/kernel/smp_32.c
===================================================================
--- linux-2.6-x86.orig/arch/x86/kernel/smp_32.c 2008-03-03 11:58:38.000000000 -0800
+++ /dev/null 1970-01-01 00:00:00.000000000 +0000
@@ -1,106 +0,0 @@
-/*
- * Intel SMP support routines.
- *
- * (c) 1995 Alan Cox, Building #3 <[email protected]>
- * (c) 1998-99, 2000 Ingo Molnar <[email protected]>
- *
- * This code is released under the GNU General Public License version 2 or
- * later.
- */
-
-#include <linux/init.h>
-
-#include <linux/mm.h>
-#include <linux/delay.h>
-#include <linux/spinlock.h>
-#include <linux/kernel_stat.h>
-#include <linux/mc146818rtc.h>
-#include <linux/cache.h>
-#include <linux/interrupt.h>
-#include <linux/cpu.h>
-#include <linux/module.h>
-
-#include <asm/mtrr.h>
-#include <asm/tlbflush.h>
-#include <asm/mmu_context.h>
-#include <mach_apic.h>
-#include <asm/proto.h>
-
-/*
- * Some notes on x86 processor bugs affecting SMP operation:
- *
- * Pentium, Pentium Pro, II, III (and all CPUs) have bugs.
- * The Linux implications for SMP are handled as follows:
- *
- * Pentium III / [Xeon]
- * None of the E1AP-E3AP errata are visible to the user.
- *
- * E1AP. see PII A1AP
- * E2AP. see PII A2AP
- * E3AP. see PII A3AP
- *
- * Pentium II / [Xeon]
- * None of the A1AP-A3AP errata are visible to the user.
- *
- * A1AP. see PPro 1AP
- * A2AP. see PPro 2AP
- * A3AP. see PPro 7AP
- *
- * Pentium Pro
- * None of 1AP-9AP errata are visible to the normal user,
- * except occasional delivery of 'spurious interrupt' as trap #15.
- * This is very rare and a non-problem.
- *
- * 1AP. Linux maps APIC as non-cacheable
- * 2AP. worked around in hardware
- * 3AP. fixed in C0 and above steppings microcode update.
- * Linux does not use excessive STARTUP_IPIs.
- * 4AP. worked around in hardware
- * 5AP. symmetric IO mode (normal Linux operation) not affected.
- * 'noapic' mode has vector 0xf filled out properly.
- * 6AP. 'noapic' mode might be affected - fixed in later steppings
- * 7AP. We do not assume writes to the LVT deassering IRQs
- * 8AP. We do not enable low power mode (deep sleep) during MP bootup
- * 9AP. We do not use mixed mode
- *
- * Pentium
- * There is a marginal case where REP MOVS on 100MHz SMP
- * machines with B stepping processors can fail. XXX should provide
- * an L1cache=Writethrough or L1cache=off option.
- *
- * B stepping CPUs may hang. There are hardware work arounds
- * for this. We warn about it in case your board doesn't have the work
- * arounds. Basically that's so I can tell anyone with a B stepping
- * CPU and SMP problems "tough".
- *
- * Specific items [From Pentium Processor Specification Update]
- *
- * 1AP. Linux doesn't use remote read
- * 2AP. Linux doesn't trust APIC errors
- * 3AP. We work around this
- * 4AP. Linux never generated 3 interrupts of the same priority
- * to cause a lost local interrupt.
- * 5AP. Remote read is never used
- * 6AP. not affected - worked around in hardware
- * 7AP. not affected - worked around in hardware
- * 8AP. worked around in hardware - we get explicit CS errors if not
- * 9AP. only 'noapic' mode affected. Might generate spurious
- * interrupts, we log only the first one and count the
- * rest silently.
- * 10AP. not affected - worked around in hardware
- * 11AP. Linux reads the APIC between writes to avoid this, as per
- * the documentation. Make sure you preserve this as it affects
- * the C stepping chips too.
- * 12AP. not affected - worked around in hardware
- * 13AP. not affected - worked around in hardware
- * 14AP. we always deassert INIT during bootup
- * 15AP. not affected - worked around in hardware
- * 16AP. not affected - worked around in hardware
- * 17AP. not affected - worked around in hardware
- * 18AP. not affected - worked around in hardware
- * 19AP. not affected - worked around in BIOS
- *
- * If this sounds worrying believe me these bugs are either ___RARE___,
- * or are signal timing bugs worked around in hardware and there's
- * about nothing of note with C stepping upwards.
- */
Index: linux-2.6-x86/arch/x86/kernel/smp_64.c
===================================================================
--- linux-2.6-x86.orig/arch/x86/kernel/smp_64.c 2008-03-03 11:58:38.000000000 -0800
+++ /dev/null 1970-01-01 00:00:00.000000000 +0000
@@ -1,10 +0,0 @@
-/*
- * Intel SMP support routines.
- *
- * (c) 1995 Alan Cox, Building #3 <[email protected]>
- * (c) 1998-99, 2000 Ingo Molnar <[email protected]>
- * (c) 2002,2003 Andi Kleen, SuSE Labs.
- *
- * This code is released under the GNU General Public License version 2 or
- * later.
- */
this patch creates tlb_32.c and tlb_64.c, with
tlb-related functions that used to live in smp*.c files.
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/Makefile | 2 +-
arch/x86/kernel/smp_32.c | 235 ---------------------------------------
arch/x86/kernel/smp_64.c | 275 ----------------------------------------------
arch/x86/kernel/tlb_32.c | 243 ++++++++++++++++++++++++++++++++++++++++
arch/x86/kernel/tlb_64.c | 273 +++++++++++++++++++++++++++++++++++++++++++++
5 files changed, 517 insertions(+), 511 deletions(-)
create mode 100644 arch/x86/kernel/tlb_32.c
create mode 100644 arch/x86/kernel/tlb_64.c
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index e3b01f9..362ab6a 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -47,7 +47,7 @@ obj-$(CONFIG_PCI) += early-quirks.o
apm-y := apm_32.o
obj-$(CONFIG_APM) += apm.o
obj-$(CONFIG_X86_SMP) += smp_$(BITS).o smpboot_$(BITS).o smp.o
-obj-$(CONFIG_X86_SMP) += smpboot.o tsc_sync.o ipi.o
+obj-$(CONFIG_X86_SMP) += smpboot.o tsc_sync.o ipi.o tlb_$(BITS).o
obj-$(CONFIG_X86_32_SMP) += smpcommon.o
obj-$(CONFIG_X86_64_SMP) += smp_64.o smpboot_64.o tsc_sync.o smpcommon.o
obj-$(CONFIG_X86_TRAMPOLINE) += trampoline_$(BITS).o
diff --git a/arch/x86/kernel/smp_32.c b/arch/x86/kernel/smp_32.c
index d80623a..d8fdec5 100644
--- a/arch/x86/kernel/smp_32.c
+++ b/arch/x86/kernel/smp_32.c
@@ -104,238 +104,3 @@
* or are signal timing bugs worked around in hardware and there's
* about nothing of note with C stepping upwards.
*/
-
-DEFINE_PER_CPU(struct tlb_state, cpu_tlbstate) ____cacheline_aligned = { &init_mm, 0, };
-
-#include <mach_ipi.h> /* must come after the send_IPI functions above for inlining */
-
-/*
- * Smarter SMP flushing macros.
- * c/o Linus Torvalds.
- *
- * These mean you can really definitely utterly forget about
- * writing to user space from interrupts. (Its not allowed anyway).
- *
- * Optimizations Manfred Spraul <[email protected]>
- */
-
-static cpumask_t flush_cpumask;
-static struct mm_struct * flush_mm;
-static unsigned long flush_va;
-static DEFINE_SPINLOCK(tlbstate_lock);
-
-/*
- * We cannot call mmdrop() because we are in interrupt context,
- * instead update mm->cpu_vm_mask.
- *
- * We need to reload %cr3 since the page tables may be going
- * away from under us..
- */
-void leave_mm(int cpu)
-{
- if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_OK)
- BUG();
- cpu_clear(cpu, per_cpu(cpu_tlbstate, cpu).active_mm->cpu_vm_mask);
- load_cr3(swapper_pg_dir);
-}
-EXPORT_SYMBOL_GPL(leave_mm);
-
-/*
- *
- * The flush IPI assumes that a thread switch happens in this order:
- * [cpu0: the cpu that switches]
- * 1) switch_mm() either 1a) or 1b)
- * 1a) thread switch to a different mm
- * 1a1) cpu_clear(cpu, old_mm->cpu_vm_mask);
- * Stop ipi delivery for the old mm. This is not synchronized with
- * the other cpus, but smp_invalidate_interrupt ignore flush ipis
- * for the wrong mm, and in the worst case we perform a superfluous
- * tlb flush.
- * 1a2) set cpu_tlbstate to TLBSTATE_OK
- * Now the smp_invalidate_interrupt won't call leave_mm if cpu0
- * was in lazy tlb mode.
- * 1a3) update cpu_tlbstate[].active_mm
- * Now cpu0 accepts tlb flushes for the new mm.
- * 1a4) cpu_set(cpu, new_mm->cpu_vm_mask);
- * Now the other cpus will send tlb flush ipis.
- * 1a4) change cr3.
- * 1b) thread switch without mm change
- * cpu_tlbstate[].active_mm is correct, cpu0 already handles
- * flush ipis.
- * 1b1) set cpu_tlbstate to TLBSTATE_OK
- * 1b2) test_and_set the cpu bit in cpu_vm_mask.
- * Atomically set the bit [other cpus will start sending flush ipis],
- * and test the bit.
- * 1b3) if the bit was 0: leave_mm was called, flush the tlb.
- * 2) switch %%esp, ie current
- *
- * The interrupt must handle 2 special cases:
- * - cr3 is changed before %%esp, ie. it cannot use current->{active_,}mm.
- * - the cpu performs speculative tlb reads, i.e. even if the cpu only
- * runs in kernel space, the cpu could load tlb entries for user space
- * pages.
- *
- * The good news is that cpu_tlbstate is local to each cpu, no
- * write/read ordering problems.
- */
-
-/*
- * TLB flush IPI:
- *
- * 1) Flush the tlb entries if the cpu uses the mm that's being flushed.
- * 2) Leave the mm if we are in the lazy tlb mode.
- */
-
-void smp_invalidate_interrupt(struct pt_regs *regs)
-{
- unsigned long cpu;
-
- cpu = get_cpu();
-
- if (!cpu_isset(cpu, flush_cpumask))
- goto out;
- /*
- * This was a BUG() but until someone can quote me the
- * line from the intel manual that guarantees an IPI to
- * multiple CPUs is retried _only_ on the erroring CPUs
- * its staying as a return
- *
- * BUG();
- */
-
- if (flush_mm == per_cpu(cpu_tlbstate, cpu).active_mm) {
- if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_OK) {
- if (flush_va == TLB_FLUSH_ALL)
- local_flush_tlb();
- else
- __flush_tlb_one(flush_va);
- } else
- leave_mm(cpu);
- }
- ack_APIC_irq();
- smp_mb__before_clear_bit();
- cpu_clear(cpu, flush_cpumask);
- smp_mb__after_clear_bit();
-out:
- put_cpu_no_resched();
- __get_cpu_var(irq_stat).irq_tlb_count++;
-}
-
-void native_flush_tlb_others(const cpumask_t *cpumaskp, struct mm_struct *mm,
- unsigned long va)
-{
- cpumask_t cpumask = *cpumaskp;
-
- /*
- * A couple of (to be removed) sanity checks:
- *
- * - current CPU must not be in mask
- * - mask must exist :)
- */
- BUG_ON(cpus_empty(cpumask));
- BUG_ON(cpu_isset(smp_processor_id(), cpumask));
- BUG_ON(!mm);
-
-#ifdef CONFIG_HOTPLUG_CPU
- /* If a CPU which we ran on has gone down, OK. */
- cpus_and(cpumask, cpumask, cpu_online_map);
- if (unlikely(cpus_empty(cpumask)))
- return;
-#endif
-
- /*
- * i'm not happy about this global shared spinlock in the
- * MM hot path, but we'll see how contended it is.
- * AK: x86-64 has a faster method that could be ported.
- */
- spin_lock(&tlbstate_lock);
-
- flush_mm = mm;
- flush_va = va;
- cpus_or(flush_cpumask, cpumask, flush_cpumask);
- /*
- * We have to send the IPI only to
- * CPUs affected.
- */
- send_IPI_mask(cpumask, INVALIDATE_TLB_VECTOR);
-
- while (!cpus_empty(flush_cpumask))
- /* nothing. lockup detection does not belong here */
- cpu_relax();
-
- flush_mm = NULL;
- flush_va = 0;
- spin_unlock(&tlbstate_lock);
-}
-
-void flush_tlb_current_task(void)
-{
- struct mm_struct *mm = current->mm;
- cpumask_t cpu_mask;
-
- preempt_disable();
- cpu_mask = mm->cpu_vm_mask;
- cpu_clear(smp_processor_id(), cpu_mask);
-
- local_flush_tlb();
- if (!cpus_empty(cpu_mask))
- flush_tlb_others(cpu_mask, mm, TLB_FLUSH_ALL);
- preempt_enable();
-}
-
-void flush_tlb_mm (struct mm_struct * mm)
-{
- cpumask_t cpu_mask;
-
- preempt_disable();
- cpu_mask = mm->cpu_vm_mask;
- cpu_clear(smp_processor_id(), cpu_mask);
-
- if (current->active_mm == mm) {
- if (current->mm)
- local_flush_tlb();
- else
- leave_mm(smp_processor_id());
- }
- if (!cpus_empty(cpu_mask))
- flush_tlb_others(cpu_mask, mm, TLB_FLUSH_ALL);
-
- preempt_enable();
-}
-
-void flush_tlb_page(struct vm_area_struct * vma, unsigned long va)
-{
- struct mm_struct *mm = vma->vm_mm;
- cpumask_t cpu_mask;
-
- preempt_disable();
- cpu_mask = mm->cpu_vm_mask;
- cpu_clear(smp_processor_id(), cpu_mask);
-
- if (current->active_mm == mm) {
- if(current->mm)
- __flush_tlb_one(va);
- else
- leave_mm(smp_processor_id());
- }
-
- if (!cpus_empty(cpu_mask))
- flush_tlb_others(cpu_mask, mm, va);
-
- preempt_enable();
-}
-EXPORT_SYMBOL(flush_tlb_page);
-
-static void do_flush_tlb_all(void* info)
-{
- unsigned long cpu = smp_processor_id();
-
- __flush_tlb_all();
- if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_LAZY)
- leave_mm(cpu);
-}
-
-void flush_tlb_all(void)
-{
- on_each_cpu(do_flush_tlb_all, NULL, 1, 1);
-}
diff --git a/arch/x86/kernel/smp_64.c b/arch/x86/kernel/smp_64.c
index d28e868..26448ff 100644
--- a/arch/x86/kernel/smp_64.c
+++ b/arch/x86/kernel/smp_64.c
@@ -8,278 +8,3 @@
* This code is released under the GNU General Public License version 2 or
* later.
*/
-
-#include <linux/init.h>
-
-#include <linux/mm.h>
-#include <linux/delay.h>
-#include <linux/spinlock.h>
-#include <linux/smp.h>
-#include <linux/kernel_stat.h>
-#include <linux/mc146818rtc.h>
-#include <linux/interrupt.h>
-
-#include <asm/mtrr.h>
-#include <asm/pgalloc.h>
-#include <asm/tlbflush.h>
-#include <asm/mach_apic.h>
-#include <asm/mmu_context.h>
-#include <asm/proto.h>
-#include <asm/apicdef.h>
-#include <asm/idle.h>
-
-/*
- * Smarter SMP flushing macros.
- * c/o Linus Torvalds.
- *
- * These mean you can really definitely utterly forget about
- * writing to user space from interrupts. (Its not allowed anyway).
- *
- * Optimizations Manfred Spraul <[email protected]>
- *
- * More scalable flush, from Andi Kleen
- *
- * To avoid global state use 8 different call vectors.
- * Each CPU uses a specific vector to trigger flushes on other
- * CPUs. Depending on the received vector the target CPUs look into
- * the right per cpu variable for the flush data.
- *
- * With more than 8 CPUs they are hashed to the 8 available
- * vectors. The limited global vector space forces us to this right now.
- * In future when interrupts are split into per CPU domains this could be
- * fixed, at the cost of triggering multiple IPIs in some cases.
- */
-
-union smp_flush_state {
- struct {
- cpumask_t flush_cpumask;
- struct mm_struct *flush_mm;
- unsigned long flush_va;
- spinlock_t tlbstate_lock;
- };
- char pad[SMP_CACHE_BYTES];
-} ____cacheline_aligned;
-
-/* State is put into the per CPU data section, but padded
- to a full cache line because other CPUs can access it and we don't
- want false sharing in the per cpu data segment. */
-static DEFINE_PER_CPU(union smp_flush_state, flush_state);
-
-/*
- * We cannot call mmdrop() because we are in interrupt context,
- * instead update mm->cpu_vm_mask.
- */
-void leave_mm(int cpu)
-{
- if (read_pda(mmu_state) == TLBSTATE_OK)
- BUG();
- cpu_clear(cpu, read_pda(active_mm)->cpu_vm_mask);
- load_cr3(swapper_pg_dir);
-}
-EXPORT_SYMBOL_GPL(leave_mm);
-
-/*
- *
- * The flush IPI assumes that a thread switch happens in this order:
- * [cpu0: the cpu that switches]
- * 1) switch_mm() either 1a) or 1b)
- * 1a) thread switch to a different mm
- * 1a1) cpu_clear(cpu, old_mm->cpu_vm_mask);
- * Stop ipi delivery for the old mm. This is not synchronized with
- * the other cpus, but smp_invalidate_interrupt ignore flush ipis
- * for the wrong mm, and in the worst case we perform a superfluous
- * tlb flush.
- * 1a2) set cpu mmu_state to TLBSTATE_OK
- * Now the smp_invalidate_interrupt won't call leave_mm if cpu0
- * was in lazy tlb mode.
- * 1a3) update cpu active_mm
- * Now cpu0 accepts tlb flushes for the new mm.
- * 1a4) cpu_set(cpu, new_mm->cpu_vm_mask);
- * Now the other cpus will send tlb flush ipis.
- * 1a4) change cr3.
- * 1b) thread switch without mm change
- * cpu active_mm is correct, cpu0 already handles
- * flush ipis.
- * 1b1) set cpu mmu_state to TLBSTATE_OK
- * 1b2) test_and_set the cpu bit in cpu_vm_mask.
- * Atomically set the bit [other cpus will start sending flush ipis],
- * and test the bit.
- * 1b3) if the bit was 0: leave_mm was called, flush the tlb.
- * 2) switch %%esp, ie current
- *
- * The interrupt must handle 2 special cases:
- * - cr3 is changed before %%esp, ie. it cannot use current->{active_,}mm.
- * - the cpu performs speculative tlb reads, i.e. even if the cpu only
- * runs in kernel space, the cpu could load tlb entries for user space
- * pages.
- *
- * The good news is that cpu mmu_state is local to each cpu, no
- * write/read ordering problems.
- */
-
-/*
- * TLB flush IPI:
- *
- * 1) Flush the tlb entries if the cpu uses the mm that's being flushed.
- * 2) Leave the mm if we are in the lazy tlb mode.
- *
- * Interrupts are disabled.
- */
-
-asmlinkage void smp_invalidate_interrupt(struct pt_regs *regs)
-{
- int cpu;
- int sender;
- union smp_flush_state *f;
-
- cpu = smp_processor_id();
- /*
- * orig_rax contains the negated interrupt vector.
- * Use that to determine where the sender put the data.
- */
- sender = ~regs->orig_ax - INVALIDATE_TLB_VECTOR_START;
- f = &per_cpu(flush_state, sender);
-
- if (!cpu_isset(cpu, f->flush_cpumask))
- goto out;
- /*
- * This was a BUG() but until someone can quote me the
- * line from the intel manual that guarantees an IPI to
- * multiple CPUs is retried _only_ on the erroring CPUs
- * its staying as a return
- *
- * BUG();
- */
-
- if (f->flush_mm == read_pda(active_mm)) {
- if (read_pda(mmu_state) == TLBSTATE_OK) {
- if (f->flush_va == TLB_FLUSH_ALL)
- local_flush_tlb();
- else
- __flush_tlb_one(f->flush_va);
- } else
- leave_mm(cpu);
- }
-out:
- ack_APIC_irq();
- cpu_clear(cpu, f->flush_cpumask);
- add_pda(irq_tlb_count, 1);
-}
-
-void native_flush_tlb_others(const cpumask_t *cpumaskp, struct mm_struct *mm,
- unsigned long va)
-{
- int sender;
- union smp_flush_state *f;
- cpumask_t cpumask = *cpumaskp;
-
- /* Caller has disabled preemption */
- sender = smp_processor_id() % NUM_INVALIDATE_TLB_VECTORS;
- f = &per_cpu(flush_state, sender);
-
- /*
- * Could avoid this lock when
- * num_online_cpus() <= NUM_INVALIDATE_TLB_VECTORS, but it is
- * probably not worth checking this for a cache-hot lock.
- */
- spin_lock(&f->tlbstate_lock);
-
- f->flush_mm = mm;
- f->flush_va = va;
- cpus_or(f->flush_cpumask, cpumask, f->flush_cpumask);
-
- /*
- * We have to send the IPI only to
- * CPUs affected.
- */
- send_IPI_mask(cpumask, INVALIDATE_TLB_VECTOR_START + sender);
-
- while (!cpus_empty(f->flush_cpumask))
- cpu_relax();
-
- f->flush_mm = NULL;
- f->flush_va = 0;
- spin_unlock(&f->tlbstate_lock);
-}
-
-int __cpuinit init_smp_flush(void)
-{
- int i;
-
- for_each_cpu_mask(i, cpu_possible_map) {
- spin_lock_init(&per_cpu(flush_state, i).tlbstate_lock);
- }
- return 0;
-}
-core_initcall(init_smp_flush);
-
-void flush_tlb_current_task(void)
-{
- struct mm_struct *mm = current->mm;
- cpumask_t cpu_mask;
-
- preempt_disable();
- cpu_mask = mm->cpu_vm_mask;
- cpu_clear(smp_processor_id(), cpu_mask);
-
- local_flush_tlb();
- if (!cpus_empty(cpu_mask))
- flush_tlb_others(cpu_mask, mm, TLB_FLUSH_ALL);
- preempt_enable();
-}
-
-void flush_tlb_mm (struct mm_struct * mm)
-{
- cpumask_t cpu_mask;
-
- preempt_disable();
- cpu_mask = mm->cpu_vm_mask;
- cpu_clear(smp_processor_id(), cpu_mask);
-
- if (current->active_mm == mm) {
- if (current->mm)
- local_flush_tlb();
- else
- leave_mm(smp_processor_id());
- }
- if (!cpus_empty(cpu_mask))
- flush_tlb_others(cpu_mask, mm, TLB_FLUSH_ALL);
-
- preempt_enable();
-}
-
-void flush_tlb_page(struct vm_area_struct * vma, unsigned long va)
-{
- struct mm_struct *mm = vma->vm_mm;
- cpumask_t cpu_mask;
-
- preempt_disable();
- cpu_mask = mm->cpu_vm_mask;
- cpu_clear(smp_processor_id(), cpu_mask);
-
- if (current->active_mm == mm) {
- if(current->mm)
- __flush_tlb_one(va);
- else
- leave_mm(smp_processor_id());
- }
-
- if (!cpus_empty(cpu_mask))
- flush_tlb_others(cpu_mask, mm, va);
-
- preempt_enable();
-}
-
-static void do_flush_tlb_all(void* info)
-{
- unsigned long cpu = smp_processor_id();
-
- __flush_tlb_all();
- if (read_pda(mmu_state) == TLBSTATE_LAZY)
- leave_mm(cpu);
-}
-
-void flush_tlb_all(void)
-{
- on_each_cpu(do_flush_tlb_all, NULL, 1, 1);
-}
diff --git a/arch/x86/kernel/tlb_32.c b/arch/x86/kernel/tlb_32.c
new file mode 100644
index 0000000..9bb2363
--- /dev/null
+++ b/arch/x86/kernel/tlb_32.c
@@ -0,0 +1,243 @@
+#include <linux/spinlock.h>
+#include <linux/cpu.h>
+#include <linux/interrupt.h>
+
+#include <asm/tlbflush.h>
+
+DEFINE_PER_CPU(struct tlb_state, cpu_tlbstate)
+ ____cacheline_aligned = { &init_mm, 0, };
+
+/* must come after the send_IPI functions above for inlining */
+#include <mach_ipi.h>
+
+/*
+ * Smarter SMP flushing macros.
+ * c/o Linus Torvalds.
+ *
+ * These mean you can really definitely utterly forget about
+ * writing to user space from interrupts. (Its not allowed anyway).
+ *
+ * Optimizations Manfred Spraul <[email protected]>
+ */
+
+static cpumask_t flush_cpumask;
+static struct mm_struct *flush_mm;
+static unsigned long flush_va;
+static DEFINE_SPINLOCK(tlbstate_lock);
+
+/*
+ * We cannot call mmdrop() because we are in interrupt context,
+ * instead update mm->cpu_vm_mask.
+ *
+ * We need to reload %cr3 since the page tables may be going
+ * away from under us..
+ */
+void leave_mm(int cpu)
+{
+ if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_OK)
+ BUG();
+ cpu_clear(cpu, per_cpu(cpu_tlbstate, cpu).active_mm->cpu_vm_mask);
+ load_cr3(swapper_pg_dir);
+}
+EXPORT_SYMBOL_GPL(leave_mm);
+
+/*
+ *
+ * The flush IPI assumes that a thread switch happens in this order:
+ * [cpu0: the cpu that switches]
+ * 1) switch_mm() either 1a) or 1b)
+ * 1a) thread switch to a different mm
+ * 1a1) cpu_clear(cpu, old_mm->cpu_vm_mask);
+ * Stop ipi delivery for the old mm. This is not synchronized with
+ * the other cpus, but smp_invalidate_interrupt ignore flush ipis
+ * for the wrong mm, and in the worst case we perform a superfluous
+ * tlb flush.
+ * 1a2) set cpu_tlbstate to TLBSTATE_OK
+ * Now the smp_invalidate_interrupt won't call leave_mm if cpu0
+ * was in lazy tlb mode.
+ * 1a3) update cpu_tlbstate[].active_mm
+ * Now cpu0 accepts tlb flushes for the new mm.
+ * 1a4) cpu_set(cpu, new_mm->cpu_vm_mask);
+ * Now the other cpus will send tlb flush ipis.
+ * 1a4) change cr3.
+ * 1b) thread switch without mm change
+ * cpu_tlbstate[].active_mm is correct, cpu0 already handles
+ * flush ipis.
+ * 1b1) set cpu_tlbstate to TLBSTATE_OK
+ * 1b2) test_and_set the cpu bit in cpu_vm_mask.
+ * Atomically set the bit [other cpus will start sending flush ipis],
+ * and test the bit.
+ * 1b3) if the bit was 0: leave_mm was called, flush the tlb.
+ * 2) switch %%esp, ie current
+ *
+ * The interrupt must handle 2 special cases:
+ * - cr3 is changed before %%esp, ie. it cannot use current->{active_,}mm.
+ * - the cpu performs speculative tlb reads, i.e. even if the cpu only
+ * runs in kernel space, the cpu could load tlb entries for user space
+ * pages.
+ *
+ * The good news is that cpu_tlbstate is local to each cpu, no
+ * write/read ordering problems.
+ */
+
+/*
+ * TLB flush IPI:
+ *
+ * 1) Flush the tlb entries if the cpu uses the mm that's being flushed.
+ * 2) Leave the mm if we are in the lazy tlb mode.
+ */
+
+void smp_invalidate_interrupt(struct pt_regs *regs)
+{
+ unsigned long cpu;
+
+ cpu = get_cpu();
+
+ if (!cpu_isset(cpu, flush_cpumask))
+ goto out;
+ /*
+ * This was a BUG() but until someone can quote me the
+ * line from the intel manual that guarantees an IPI to
+ * multiple CPUs is retried _only_ on the erroring CPUs
+ * its staying as a return
+ *
+ * BUG();
+ */
+
+ if (flush_mm == per_cpu(cpu_tlbstate, cpu).active_mm) {
+ if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_OK) {
+ if (flush_va == TLB_FLUSH_ALL)
+ local_flush_tlb();
+ else
+ __flush_tlb_one(flush_va);
+ } else
+ leave_mm(cpu);
+ }
+ ack_APIC_irq();
+ smp_mb__before_clear_bit();
+ cpu_clear(cpu, flush_cpumask);
+ smp_mb__after_clear_bit();
+out:
+ put_cpu_no_resched();
+ __get_cpu_var(irq_stat).irq_tlb_count++;
+}
+
+void native_flush_tlb_others(const cpumask_t *cpumaskp, struct mm_struct *mm,
+ unsigned long va)
+{
+ cpumask_t cpumask = *cpumaskp;
+
+ /*
+ * A couple of (to be removed) sanity checks:
+ *
+ * - current CPU must not be in mask
+ * - mask must exist :)
+ */
+ BUG_ON(cpus_empty(cpumask));
+ BUG_ON(cpu_isset(smp_processor_id(), cpumask));
+ BUG_ON(!mm);
+
+#ifdef CONFIG_HOTPLUG_CPU
+ /* If a CPU which we ran on has gone down, OK. */
+ cpus_and(cpumask, cpumask, cpu_online_map);
+ if (unlikely(cpus_empty(cpumask)))
+ return;
+#endif
+
+ /*
+ * i'm not happy about this global shared spinlock in the
+ * MM hot path, but we'll see how contended it is.
+ * AK: x86-64 has a faster method that could be ported.
+ */
+ spin_lock(&tlbstate_lock);
+
+ flush_mm = mm;
+ flush_va = va;
+ cpus_or(flush_cpumask, cpumask, flush_cpumask);
+ /*
+ * We have to send the IPI only to
+ * CPUs affected.
+ */
+ send_IPI_mask(cpumask, INVALIDATE_TLB_VECTOR);
+
+ while (!cpus_empty(flush_cpumask))
+ /* nothing. lockup detection does not belong here */
+ cpu_relax();
+
+ flush_mm = NULL;
+ flush_va = 0;
+ spin_unlock(&tlbstate_lock);
+}
+
+void flush_tlb_current_task(void)
+{
+ struct mm_struct *mm = current->mm;
+ cpumask_t cpu_mask;
+
+ preempt_disable();
+ cpu_mask = mm->cpu_vm_mask;
+ cpu_clear(smp_processor_id(), cpu_mask);
+
+ local_flush_tlb();
+ if (!cpus_empty(cpu_mask))
+ flush_tlb_others(cpu_mask, mm, TLB_FLUSH_ALL);
+ preempt_enable();
+}
+
+void flush_tlb_mm(struct mm_struct *mm)
+{
+ cpumask_t cpu_mask;
+
+ preempt_disable();
+ cpu_mask = mm->cpu_vm_mask;
+ cpu_clear(smp_processor_id(), cpu_mask);
+
+ if (current->active_mm == mm) {
+ if (current->mm)
+ local_flush_tlb();
+ else
+ leave_mm(smp_processor_id());
+ }
+ if (!cpus_empty(cpu_mask))
+ flush_tlb_others(cpu_mask, mm, TLB_FLUSH_ALL);
+
+ preempt_enable();
+}
+
+void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
+{
+ struct mm_struct *mm = vma->vm_mm;
+ cpumask_t cpu_mask;
+
+ preempt_disable();
+ cpu_mask = mm->cpu_vm_mask;
+ cpu_clear(smp_processor_id(), cpu_mask);
+
+ if (current->active_mm == mm) {
+ if (current->mm)
+ __flush_tlb_one(va);
+ else
+ leave_mm(smp_processor_id());
+ }
+
+ if (!cpus_empty(cpu_mask))
+ flush_tlb_others(cpu_mask, mm, va);
+
+ preempt_enable();
+}
+EXPORT_SYMBOL(flush_tlb_page);
+
+static void do_flush_tlb_all(void *info)
+{
+ unsigned long cpu = smp_processor_id();
+
+ __flush_tlb_all();
+ if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_LAZY)
+ leave_mm(cpu);
+}
+
+void flush_tlb_all(void)
+{
+ on_each_cpu(do_flush_tlb_all, NULL, 1, 1);
+}
+
diff --git a/arch/x86/kernel/tlb_64.c b/arch/x86/kernel/tlb_64.c
new file mode 100644
index 0000000..615d848
--- /dev/null
+++ b/arch/x86/kernel/tlb_64.c
@@ -0,0 +1,273 @@
+#include <linux/init.h>
+
+#include <linux/mm.h>
+#include <linux/delay.h>
+#include <linux/spinlock.h>
+#include <linux/smp.h>
+#include <linux/kernel_stat.h>
+#include <linux/mc146818rtc.h>
+#include <linux/interrupt.h>
+
+#include <asm/mtrr.h>
+#include <asm/pgalloc.h>
+#include <asm/tlbflush.h>
+#include <asm/mach_apic.h>
+#include <asm/mmu_context.h>
+#include <asm/proto.h>
+#include <asm/apicdef.h>
+#include <asm/idle.h>
+/*
+ * Smarter SMP flushing macros.
+ * c/o Linus Torvalds.
+ *
+ * These mean you can really definitely utterly forget about
+ * writing to user space from interrupts. (Its not allowed anyway).
+ *
+ * Optimizations Manfred Spraul <[email protected]>
+ *
+ * More scalable flush, from Andi Kleen
+ *
+ * To avoid global state use 8 different call vectors.
+ * Each CPU uses a specific vector to trigger flushes on other
+ * CPUs. Depending on the received vector the target CPUs look into
+ * the right per cpu variable for the flush data.
+ *
+ * With more than 8 CPUs they are hashed to the 8 available
+ * vectors. The limited global vector space forces us to this right now.
+ * In future when interrupts are split into per CPU domains this could be
+ * fixed, at the cost of triggering multiple IPIs in some cases.
+ */
+
+union smp_flush_state {
+ struct {
+ cpumask_t flush_cpumask;
+ struct mm_struct *flush_mm;
+ unsigned long flush_va;
+ spinlock_t tlbstate_lock;
+ };
+ char pad[SMP_CACHE_BYTES];
+} ____cacheline_aligned;
+
+/* State is put into the per CPU data section, but padded
+ to a full cache line because other CPUs can access it and we don't
+ want false sharing in the per cpu data segment. */
+static DEFINE_PER_CPU(union smp_flush_state, flush_state);
+
+/*
+ * We cannot call mmdrop() because we are in interrupt context,
+ * instead update mm->cpu_vm_mask.
+ */
+void leave_mm(int cpu)
+{
+ if (read_pda(mmu_state) == TLBSTATE_OK)
+ BUG();
+ cpu_clear(cpu, read_pda(active_mm)->cpu_vm_mask);
+ load_cr3(swapper_pg_dir);
+}
+EXPORT_SYMBOL_GPL(leave_mm);
+
+/*
+ *
+ * The flush IPI assumes that a thread switch happens in this order:
+ * [cpu0: the cpu that switches]
+ * 1) switch_mm() either 1a) or 1b)
+ * 1a) thread switch to a different mm
+ * 1a1) cpu_clear(cpu, old_mm->cpu_vm_mask);
+ * Stop ipi delivery for the old mm. This is not synchronized with
+ * the other cpus, but smp_invalidate_interrupt ignore flush ipis
+ * for the wrong mm, and in the worst case we perform a superfluous
+ * tlb flush.
+ * 1a2) set cpu mmu_state to TLBSTATE_OK
+ * Now the smp_invalidate_interrupt won't call leave_mm if cpu0
+ * was in lazy tlb mode.
+ * 1a3) update cpu active_mm
+ * Now cpu0 accepts tlb flushes for the new mm.
+ * 1a4) cpu_set(cpu, new_mm->cpu_vm_mask);
+ * Now the other cpus will send tlb flush ipis.
+ * 1a4) change cr3.
+ * 1b) thread switch without mm change
+ * cpu active_mm is correct, cpu0 already handles
+ * flush ipis.
+ * 1b1) set cpu mmu_state to TLBSTATE_OK
+ * 1b2) test_and_set the cpu bit in cpu_vm_mask.
+ * Atomically set the bit [other cpus will start sending flush ipis],
+ * and test the bit.
+ * 1b3) if the bit was 0: leave_mm was called, flush the tlb.
+ * 2) switch %%esp, ie current
+ *
+ * The interrupt must handle 2 special cases:
+ * - cr3 is changed before %%esp, ie. it cannot use current->{active_,}mm.
+ * - the cpu performs speculative tlb reads, i.e. even if the cpu only
+ * runs in kernel space, the cpu could load tlb entries for user space
+ * pages.
+ *
+ * The good news is that cpu mmu_state is local to each cpu, no
+ * write/read ordering problems.
+ */
+
+/*
+ * TLB flush IPI:
+ *
+ * 1) Flush the tlb entries if the cpu uses the mm that's being flushed.
+ * 2) Leave the mm if we are in the lazy tlb mode.
+ *
+ * Interrupts are disabled.
+ */
+
+asmlinkage void smp_invalidate_interrupt(struct pt_regs *regs)
+{
+ int cpu;
+ int sender;
+ union smp_flush_state *f;
+
+ cpu = smp_processor_id();
+ /*
+ * orig_rax contains the negated interrupt vector.
+ * Use that to determine where the sender put the data.
+ */
+ sender = ~regs->orig_ax - INVALIDATE_TLB_VECTOR_START;
+ f = &per_cpu(flush_state, sender);
+
+ if (!cpu_isset(cpu, f->flush_cpumask))
+ goto out;
+ /*
+ * This was a BUG() but until someone can quote me the
+ * line from the intel manual that guarantees an IPI to
+ * multiple CPUs is retried _only_ on the erroring CPUs
+ * its staying as a return
+ *
+ * BUG();
+ */
+
+ if (f->flush_mm == read_pda(active_mm)) {
+ if (read_pda(mmu_state) == TLBSTATE_OK) {
+ if (f->flush_va == TLB_FLUSH_ALL)
+ local_flush_tlb();
+ else
+ __flush_tlb_one(f->flush_va);
+ } else
+ leave_mm(cpu);
+ }
+out:
+ ack_APIC_irq();
+ cpu_clear(cpu, f->flush_cpumask);
+ add_pda(irq_tlb_count, 1);
+}
+
+void native_flush_tlb_others(const cpumask_t *cpumaskp, struct mm_struct *mm,
+ unsigned long va)
+{
+ int sender;
+ union smp_flush_state *f;
+ cpumask_t cpumask = *cpumaskp;
+
+ /* Caller has disabled preemption */
+ sender = smp_processor_id() % NUM_INVALIDATE_TLB_VECTORS;
+ f = &per_cpu(flush_state, sender);
+
+ /*
+ * Could avoid this lock when
+ * num_online_cpus() <= NUM_INVALIDATE_TLB_VECTORS, but it is
+ * probably not worth checking this for a cache-hot lock.
+ */
+ spin_lock(&f->tlbstate_lock);
+
+ f->flush_mm = mm;
+ f->flush_va = va;
+ cpus_or(f->flush_cpumask, cpumask, f->flush_cpumask);
+
+ /*
+ * We have to send the IPI only to
+ * CPUs affected.
+ */
+ send_IPI_mask(cpumask, INVALIDATE_TLB_VECTOR_START + sender);
+
+ while (!cpus_empty(f->flush_cpumask))
+ cpu_relax();
+
+ f->flush_mm = NULL;
+ f->flush_va = 0;
+ spin_unlock(&f->tlbstate_lock);
+}
+
+int __cpuinit init_smp_flush(void)
+{
+ int i;
+
+ for_each_cpu_mask(i, cpu_possible_map) {
+ spin_lock_init(&per_cpu(flush_state, i).tlbstate_lock);
+ }
+ return 0;
+}
+core_initcall(init_smp_flush);
+
+void flush_tlb_current_task(void)
+{
+ struct mm_struct *mm = current->mm;
+ cpumask_t cpu_mask;
+
+ preempt_disable();
+ cpu_mask = mm->cpu_vm_mask;
+ cpu_clear(smp_processor_id(), cpu_mask);
+
+ local_flush_tlb();
+ if (!cpus_empty(cpu_mask))
+ flush_tlb_others(cpu_mask, mm, TLB_FLUSH_ALL);
+ preempt_enable();
+}
+
+void flush_tlb_mm(struct mm_struct *mm)
+{
+ cpumask_t cpu_mask;
+
+ preempt_disable();
+ cpu_mask = mm->cpu_vm_mask;
+ cpu_clear(smp_processor_id(), cpu_mask);
+
+ if (current->active_mm == mm) {
+ if (current->mm)
+ local_flush_tlb();
+ else
+ leave_mm(smp_processor_id());
+ }
+ if (!cpus_empty(cpu_mask))
+ flush_tlb_others(cpu_mask, mm, TLB_FLUSH_ALL);
+
+ preempt_enable();
+}
+
+void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
+{
+ struct mm_struct *mm = vma->vm_mm;
+ cpumask_t cpu_mask;
+
+ preempt_disable();
+ cpu_mask = mm->cpu_vm_mask;
+ cpu_clear(smp_processor_id(), cpu_mask);
+
+ if (current->active_mm == mm) {
+ if (current->mm)
+ __flush_tlb_one(va);
+ else
+ leave_mm(smp_processor_id());
+ }
+
+ if (!cpus_empty(cpu_mask))
+ flush_tlb_others(cpu_mask, mm, va);
+
+ preempt_enable();
+}
+
+static void do_flush_tlb_all(void *info)
+{
+ unsigned long cpu = smp_processor_id();
+
+ __flush_tlb_all();
+ if (read_pda(mmu_state) == TLBSTATE_LAZY)
+ leave_mm(cpu);
+}
+
+void flush_tlb_all(void)
+{
+ on_each_cpu(do_flush_tlb_all, NULL, 1, 1);
+}
--
1.5.0.6
This is done by introducing the empty smp_callin_clear_local_apic()
and map_cpu_to_logical_apicid()
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smpboot_64.c | 4 ++++
1 files changed, 4 insertions(+), 0 deletions(-)
diff --git a/arch/x86/kernel/smpboot_64.c b/arch/x86/kernel/smpboot_64.c
index 4f6d976..b6a5da5 100644
--- a/arch/x86/kernel/smpboot_64.c
+++ b/arch/x86/kernel/smpboot_64.c
@@ -106,6 +106,8 @@ static inline void wait_for_init_deassert(atomic_t *deassert)
cpu_relax();
return;
}
+#define smp_callin_clear_local_apic() do {} while (0)
+#define map_cpu_to_logical_apicid() do {} while (0)
static atomic_t init_deasserted __cpuinitdata;
@@ -171,8 +173,10 @@ void __cpuinit smp_callin(void)
*/
Dprintk("CALLIN, before setup_local_APIC().\n");
+ smp_callin_clear_local_apic();
setup_local_APIC();
end_local_APIC_setup();
+ map_cpu_to_logical_apicid();
/*
* Get our bogomips.
--
1.5.0.6
although i386 appearantly does not have problems with the nmi,
we leave irqs enabled, as it is common ground between it and x86_64.
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smpboot_32.c | 5 +++++
1 files changed, 5 insertions(+), 0 deletions(-)
diff --git a/arch/x86/kernel/smpboot_32.c b/arch/x86/kernel/smpboot_32.c
index 2dd95ba..8c39369 100644
--- a/arch/x86/kernel/smpboot_32.c
+++ b/arch/x86/kernel/smpboot_32.c
@@ -208,8 +208,13 @@ static void __cpuinit smp_callin(void)
/*
* Get our bogomips.
+ *
+ * Need to enable IRQs because it can take longer and then
+ * the NMI watchdog might kill us.
*/
+ local_irq_enable();
calibrate_delay();
+ local_irq_disable();
Dprintk("Stack at about %p\n",&cpuid);
/*
--
1.5.0.6
it is a little bit more complicated than x86_64 due to erratas and
other stuff, but its existance will ease integration
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/apic_32.c | 73 ++++++++++++++++++++++++--------------------
1 files changed, 40 insertions(+), 33 deletions(-)
diff --git a/arch/x86/kernel/apic_32.c b/arch/x86/kernel/apic_32.c
index 6aa93db..4a754ee 100644
--- a/arch/x86/kernel/apic_32.c
+++ b/arch/x86/kernel/apic_32.c
@@ -897,12 +897,50 @@ void __init init_bsp_APIC(void)
apic_write_around(APIC_LVT1, value);
}
+void __cpuinit lapic_setup_esr(void)
+{
+ unsigned long oldvalue, value, maxlvt;
+ if (lapic_is_integrated() && !esr_disable) {
+ /* !82489DX */
+ maxlvt = lapic_get_maxlvt();
+ if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
+ apic_write(APIC_ESR, 0);
+ oldvalue = apic_read(APIC_ESR);
+
+ /* enables sending errors */
+ value = ERROR_APIC_VECTOR;
+ apic_write_around(APIC_LVTERR, value);
+ /*
+ * spec says clear errors after enabling vector.
+ */
+ if (maxlvt > 3)
+ apic_write(APIC_ESR, 0);
+ value = apic_read(APIC_ESR);
+ if (value != oldvalue)
+ apic_printk(APIC_VERBOSE, "ESR value before enabling "
+ "vector: 0x%08lx after: 0x%08lx\n",
+ oldvalue, value);
+ } else {
+ if (esr_disable)
+ /*
+ * Something untraceable is creating bad interrupts on
+ * secondary quads ... for the moment, just leave the
+ * ESR disabled - we can't do anything useful with the
+ * errors anyway - mbligh
+ */
+ printk(KERN_INFO "Leaving ESR disabled.\n");
+ else
+ printk(KERN_INFO "No ESR for 82489DX.\n");
+ }
+}
+
+
/**
* setup_local_APIC - setup the local APIC
*/
void __cpuinit setup_local_APIC(void)
{
- unsigned long oldvalue, value, maxlvt, integrated;
+ unsigned long value, integrated;
int i, j;
/* Pound the ESR really hard over the head with a big hammer - mbligh */
@@ -1027,38 +1065,7 @@ void __cpuinit setup_local_APIC(void)
value |= APIC_LVT_LEVEL_TRIGGER;
apic_write_around(APIC_LVT1, value);
- if (integrated && !esr_disable) {
- /* !82489DX */
- maxlvt = lapic_get_maxlvt();
- if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
- apic_write(APIC_ESR, 0);
- oldvalue = apic_read(APIC_ESR);
-
- /* enables sending errors */
- value = ERROR_APIC_VECTOR;
- apic_write_around(APIC_LVTERR, value);
- /*
- * spec says clear errors after enabling vector.
- */
- if (maxlvt > 3)
- apic_write(APIC_ESR, 0);
- value = apic_read(APIC_ESR);
- if (value != oldvalue)
- apic_printk(APIC_VERBOSE, "ESR value before enabling "
- "vector: 0x%08lx after: 0x%08lx\n",
- oldvalue, value);
- } else {
- if (esr_disable)
- /*
- * Something untraceable is creating bad interrupts on
- * secondary quads ... for the moment, just leave the
- * ESR disabled - we can't do anything useful with the
- * errors anyway - mbligh
- */
- printk(KERN_INFO "Leaving ESR disabled.\n");
- else
- printk(KERN_INFO "No ESR for 82489DX.\n");
- }
+ lapic_setup_esr();
/* Disable the local apic timer */
value = apic_read(APIC_LVTT);
--
1.5.0.6
it is used by smp_callin, that is about to be removed from
this file.
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smpboot_32.c | 4 ++--
1 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/arch/x86/kernel/smpboot_32.c b/arch/x86/kernel/smpboot_32.c
index 4a08822..adad1ad 100644
--- a/arch/x86/kernel/smpboot_32.c
+++ b/arch/x86/kernel/smpboot_32.c
@@ -73,7 +73,7 @@ EXPORT_PER_CPU_SYMBOL(x86_cpu_to_apicid);
u8 apicid_2_node[MAX_APICID];
-static void map_cpu_to_logical_apicid(void);
+void map_cpu_to_logical_apicid(void);
/* State of each CPU. */
DEFINE_PER_CPU(int, cpu_state) = { 0 };
@@ -353,7 +353,7 @@ static inline void unmap_cpu_to_node(int cpu)
u8 cpu_2_logical_apicid[NR_CPUS] __read_mostly = { [0 ... NR_CPUS-1] = BAD_APICID };
-static void map_cpu_to_logical_apicid(void)
+void map_cpu_to_logical_apicid(void)
{
int cpu = smp_processor_id();
int apicid = logical_smp_processor_id();
--
1.5.0.6
smp_callin() looks like the same between i386 and x86_64,
and it is merged in a common file
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smpboot.c | 110 ++++++++++++++++++++++++++++++++++++++++++
arch/x86/kernel/smpboot_32.c | 89 ----------------------------------
arch/x86/kernel/smpboot_64.c | 102 +--------------------------------------
include/asm-x86/smp.h | 3 +
4 files changed, 114 insertions(+), 190 deletions(-)
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index b13b9d5..e0830bb 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -11,6 +11,11 @@
#include <asm/cpu.h>
#include <asm/numa.h>
+#ifdef CONFIG_X86_32
+#include <mach_apic.h>
+#include <mach_wakecpu.h>
+#endif
+
/* Number of siblings per CPU package */
int smp_num_siblings = 1;
EXPORT_SYMBOL(smp_num_siblings);
@@ -151,6 +156,111 @@ void __init smp_alloc_memory(void)
}
#endif
+#ifdef CONFIG_X86_64
+static inline void wait_for_init_deassert(atomic_t *deassert)
+{
+ while (!atomic_read(deassert))
+ cpu_relax();
+ return;
+}
+#define smp_callin_clear_local_apic() do {} while (0)
+#define map_cpu_to_logical_apicid() do {} while (0)
+#else
+extern void map_cpu_to_logical_apicid(void);
+#endif
+
+atomic_t init_deasserted __cpuinitdata;
+extern void smp_store_cpu_info(int cpuid);
+
+/*
+ * Report back to the Boot Processor.
+ * Running on AP.
+ */
+void __cpuinit smp_callin(void)
+{
+ int cpuid, phys_id;
+ unsigned long timeout;
+
+ /*
+ * If waken up by an INIT in an 82489DX configuration
+ * we may get here before an INIT-deassert IPI reaches
+ * our local APIC. We have to wait for the IPI or we'll
+ * lock up on an APIC access.
+ */
+ wait_for_init_deassert(&init_deasserted);
+
+ /*
+ * (This works even if the APIC is not enabled.)
+ */
+ phys_id = GET_APIC_ID(apic_read(APIC_ID));
+ cpuid = smp_processor_id();
+ if (cpu_isset(cpuid, cpu_callin_map)) {
+ panic("smp_callin: phys CPU#%d, CPU#%d already present??\n",
+ phys_id, cpuid);
+ }
+ Dprintk("CPU#%d (phys ID: %d) waiting for CALLOUT\n", cpuid, phys_id);
+
+ /*
+ * STARTUP IPIs are fragile beasts as they might sometimes
+ * trigger some glue motherboard logic. Complete APIC bus
+ * silence for 1 second, this overestimates the time the
+ * boot CPU is spending to send the up to 2 STARTUP IPIs
+ * by a factor of two. This should be enough.
+ */
+
+ /*
+ * Waiting 2s total for startup (udelay is not yet working)
+ */
+ timeout = jiffies + 2*HZ;
+ while (time_before(jiffies, timeout)) {
+ /*
+ * Has the boot CPU finished it's STARTUP sequence?
+ */
+ if (cpu_isset(cpuid, cpu_callout_map))
+ break;
+ cpu_relax();
+ }
+
+ if (!time_before(jiffies, timeout)) {
+ panic("%s: CPU%d started up but did not get a callout!\n",
+ __func__, cpuid);
+ }
+
+ /*
+ * the boot CPU has finished the init stage and is spinning
+ * on callin_map until we finish. We are free to set up this
+ * CPU, first the APIC. (this is probably redundant on most
+ * boards)
+ */
+
+ Dprintk("CALLIN, before setup_local_APIC().\n");
+ smp_callin_clear_local_apic();
+ setup_local_APIC();
+ end_local_APIC_setup();
+ map_cpu_to_logical_apicid();
+
+ /*
+ * Get our bogomips.
+ *
+ * Need to enable IRQs because it can take longer and then
+ * the NMI watchdog might kill us.
+ */
+ local_irq_enable();
+ calibrate_delay();
+ local_irq_disable();
+ Dprintk("Stack at about %p\n", &cpuid);
+
+ /*
+ * Save our processor parameters
+ */
+ smp_store_cpu_info(cpuid);
+
+ /*
+ * Allow the master to continue.
+ */
+ cpu_set(cpuid, cpu_callin_map);
+}
+
#ifdef CONFIG_HOTPLUG_CPU
void remove_siblinginfo(int cpu)
{
diff --git a/arch/x86/kernel/smpboot_32.c b/arch/x86/kernel/smpboot_32.c
index adad1ad..0bc78a5 100644
--- a/arch/x86/kernel/smpboot_32.c
+++ b/arch/x86/kernel/smpboot_32.c
@@ -140,95 +140,6 @@ valid_k7:
;
}
-static atomic_t init_deasserted;
-
-static void __cpuinit smp_callin(void)
-{
- int cpuid, phys_id;
- unsigned long timeout;
-
- /*
- * If waken up by an INIT in an 82489DX configuration
- * we may get here before an INIT-deassert IPI reaches
- * our local APIC. We have to wait for the IPI or we'll
- * lock up on an APIC access.
- */
- wait_for_init_deassert(&init_deasserted);
-
- /*
- * (This works even if the APIC is not enabled.)
- */
- phys_id = GET_APIC_ID(apic_read(APIC_ID));
- cpuid = smp_processor_id();
- if (cpu_isset(cpuid, cpu_callin_map)) {
- printk("huh, phys CPU#%d, CPU#%d already present??\n",
- phys_id, cpuid);
- BUG();
- }
- Dprintk("CPU#%d (phys ID: %d) waiting for CALLOUT\n", cpuid, phys_id);
-
- /*
- * STARTUP IPIs are fragile beasts as they might sometimes
- * trigger some glue motherboard logic. Complete APIC bus
- * silence for 1 second, this overestimates the time the
- * boot CPU is spending to send the up to 2 STARTUP IPIs
- * by a factor of two. This should be enough.
- */
-
- /*
- * Waiting 2s total for startup (udelay is not yet working)
- */
- timeout = jiffies + 2*HZ;
- while (time_before(jiffies, timeout)) {
- /*
- * Has the boot CPU finished it's STARTUP sequence?
- */
- if (cpu_isset(cpuid, cpu_callout_map))
- break;
- cpu_relax();
- }
-
- if (!time_before(jiffies, timeout)) {
- printk("BUG: CPU%d started up but did not get a callout!\n",
- cpuid);
- BUG();
- }
-
- /*
- * the boot CPU has finished the init stage and is spinning
- * on callin_map until we finish. We are free to set up this
- * CPU, first the APIC. (this is probably redundant on most
- * boards)
- */
-
- Dprintk("CALLIN, before setup_local_APIC().\n");
- smp_callin_clear_local_apic();
- setup_local_APIC();
- end_local_APIC_setup();
- map_cpu_to_logical_apicid();
-
- /*
- * Get our bogomips.
- *
- * Need to enable IRQs because it can take longer and then
- * the NMI watchdog might kill us.
- */
- local_irq_enable();
- calibrate_delay();
- local_irq_disable();
- Dprintk("Stack at about %p\n",&cpuid);
-
- /*
- * Save our processor parameters
- */
- smp_store_cpu_info(cpuid);
-
- /*
- * Allow the master to continue.
- */
- cpu_set(cpuid, cpu_callin_map);
-}
-
static int cpucount;
/*
diff --git a/arch/x86/kernel/smpboot_64.c b/arch/x86/kernel/smpboot_64.c
index b6a5da5..b59da62 100644
--- a/arch/x86/kernel/smpboot_64.c
+++ b/arch/x86/kernel/smpboot_64.c
@@ -90,7 +90,7 @@ struct task_struct *idle_thread_array[NR_CPUS] __cpuinitdata ;
* a given CPU
*/
-static void __cpuinit smp_store_cpu_info(int id)
+void __cpuinit smp_store_cpu_info(int id)
{
struct cpuinfo_x86 *c = &cpu_data(id);
@@ -100,106 +100,6 @@ static void __cpuinit smp_store_cpu_info(int id)
print_cpu_info(c);
}
-static inline void wait_for_init_deassert(atomic_t *deassert)
-{
- while (!atomic_read(deassert))
- cpu_relax();
- return;
-}
-#define smp_callin_clear_local_apic() do {} while (0)
-#define map_cpu_to_logical_apicid() do {} while (0)
-
-static atomic_t init_deasserted __cpuinitdata;
-
-/*
- * Report back to the Boot Processor.
- * Running on AP.
- */
-void __cpuinit smp_callin(void)
-{
- int cpuid, phys_id;
- unsigned long timeout;
-
- /*
- * If waken up by an INIT in an 82489DX configuration
- * we may get here before an INIT-deassert IPI reaches
- * our local APIC. We have to wait for the IPI or we'll
- * lock up on an APIC access.
- */
- wait_for_init_deassert(&init_deasserted);
-
- /*
- * (This works even if the APIC is not enabled.)
- */
- phys_id = GET_APIC_ID(apic_read(APIC_ID));
- cpuid = smp_processor_id();
- if (cpu_isset(cpuid, cpu_callin_map)) {
- panic("smp_callin: phys CPU#%d, CPU#%d already present??\n",
- phys_id, cpuid);
- }
- Dprintk("CPU#%d (phys ID: %d) waiting for CALLOUT\n", cpuid, phys_id);
-
- /*
- * STARTUP IPIs are fragile beasts as they might sometimes
- * trigger some glue motherboard logic. Complete APIC bus
- * silence for 1 second, this overestimates the time the
- * boot CPU is spending to send the up to 2 STARTUP IPIs
- * by a factor of two. This should be enough.
- */
-
- /*
- * Waiting 2s total for startup (udelay is not yet working)
- */
- timeout = jiffies + 2*HZ;
- while (time_before(jiffies, timeout)) {
- /*
- * Has the boot CPU finished it's STARTUP sequence?
- */
- if (cpu_isset(cpuid, cpu_callout_map))
- break;
- cpu_relax();
- }
-
- if (!time_before(jiffies, timeout)) {
- panic("smp_callin: CPU%d started up but did not get a callout!\n",
- cpuid);
- }
-
- /*
- * the boot CPU has finished the init stage and is spinning
- * on callin_map until we finish. We are free to set up this
- * CPU, first the APIC. (this is probably redundant on most
- * boards)
- */
-
- Dprintk("CALLIN, before setup_local_APIC().\n");
- smp_callin_clear_local_apic();
- setup_local_APIC();
- end_local_APIC_setup();
- map_cpu_to_logical_apicid();
-
- /*
- * Get our bogomips.
- *
- * Need to enable IRQs because it can take longer and then
- * the NMI watchdog might kill us.
- */
- local_irq_enable();
- calibrate_delay();
- local_irq_disable();
- Dprintk("Stack at about %p\n",&cpuid);
-
- /*
- * Save our processor parameters
- */
- smp_store_cpu_info(cpuid);
-
- /*
- * Allow the master to continue.
- */
- cpu_set(cpuid, cpu_callin_map);
-}
-
/*
* Setup code on secondary processor (after comming out of the trampoline)
*/
diff --git a/include/asm-x86/smp.h b/include/asm-x86/smp.h
index 513c857..821716c 100644
--- a/include/asm-x86/smp.h
+++ b/include/asm-x86/smp.h
@@ -3,11 +3,13 @@
#ifndef __ASSEMBLY__
#include <linux/cpumask.h>
#include <linux/init.h>
+#include <asm/atomic.h>
extern cpumask_t cpu_callout_map;
extern int smp_num_siblings;
extern unsigned int num_processors;
+extern atomic_t init_deasserted;
/*
* Trampoline 80x86 program as an array.
@@ -99,6 +101,7 @@ extern void cpu_uninit(void);
extern void remove_siblinginfo(int cpu);
#endif
+extern void smp_callin(void);
extern void smp_alloc_memory(void);
extern void lock_ipi_call_lock(void);
extern void unlock_ipi_call_lock(void);
--
1.5.0.6
It splits setup_local_APIC in two, providing a function corresponding
to the ending part of it. As a side effect, smp_callin looks the same
between i386 and x86_64.
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/apic_32.c | 7 ++++++-
arch/x86/kernel/smpboot_32.c | 1 +
2 files changed, 7 insertions(+), 1 deletions(-)
diff --git a/arch/x86/kernel/apic_32.c b/arch/x86/kernel/apic_32.c
index 4a754ee..6d132a9 100644
--- a/arch/x86/kernel/apic_32.c
+++ b/arch/x86/kernel/apic_32.c
@@ -1064,9 +1064,13 @@ void __cpuinit setup_local_APIC(void)
if (!integrated) /* 82489DX */
value |= APIC_LVT_LEVEL_TRIGGER;
apic_write_around(APIC_LVT1, value);
+}
- lapic_setup_esr();
+void __cpuinit end_local_APIC_setup(void)
+{
+ unsigned long value;
+ lapic_setup_esr();
/* Disable the local apic timer */
value = apic_read(APIC_LVTT);
value |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
@@ -1256,6 +1260,7 @@ int __init APIC_init_uniprocessor(void)
setup_local_APIC();
+ end_local_APIC_setup();
#ifdef CONFIG_X86_IO_APIC
if (smp_found_config)
if (!skip_ioapic_setup && nr_ioapics)
diff --git a/arch/x86/kernel/smpboot_32.c b/arch/x86/kernel/smpboot_32.c
index 8c39369..4a08822 100644
--- a/arch/x86/kernel/smpboot_32.c
+++ b/arch/x86/kernel/smpboot_32.c
@@ -204,6 +204,7 @@ static void __cpuinit smp_callin(void)
Dprintk("CALLIN, before setup_local_APIC().\n");
smp_callin_clear_local_apic();
setup_local_APIC();
+ end_local_APIC_setup();
map_cpu_to_logical_apicid();
/*
--
1.5.0.6
set_cpu_sibling_map() and remove_sibling_info() are
equal between architectures, and are now moved to common
file
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smpboot.c | 88 +++++++++++++++++++++++++++++++++++++++++
arch/x86/kernel/smpboot_32.c | 88 -----------------------------------------
arch/x86/kernel/smpboot_64.c | 89 ------------------------------------------
3 files changed, 88 insertions(+), 177 deletions(-)
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 40a3b56..d774520 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -29,7 +29,95 @@ EXPORT_PER_CPU_SYMBOL(cpu_core_map);
/* Per CPU bogomips and other parameters */
DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);
EXPORT_PER_CPU_SYMBOL(cpu_info);
+
+/* representing cpus for which sibling maps can be computed */
+static cpumask_t cpu_sibling_setup_map;
+
+void __cpuinit set_cpu_sibling_map(int cpu)
+{
+ int i;
+ struct cpuinfo_x86 *c = &cpu_data(cpu);
+
+ cpu_set(cpu, cpu_sibling_setup_map);
+
+ if (smp_num_siblings > 1) {
+ for_each_cpu_mask(i, cpu_sibling_setup_map) {
+ if (c->phys_proc_id == cpu_data(i).phys_proc_id &&
+ c->cpu_core_id == cpu_data(i).cpu_core_id) {
+ cpu_set(i, per_cpu(cpu_sibling_map, cpu));
+ cpu_set(cpu, per_cpu(cpu_sibling_map, i));
+ cpu_set(i, per_cpu(cpu_core_map, cpu));
+ cpu_set(cpu, per_cpu(cpu_core_map, i));
+ cpu_set(i, c->llc_shared_map);
+ cpu_set(cpu, cpu_data(i).llc_shared_map);
+ }
+ }
+ } else {
+ cpu_set(cpu, per_cpu(cpu_sibling_map, cpu));
+ }
+
+ cpu_set(cpu, c->llc_shared_map);
+
+ if (current_cpu_data.x86_max_cores == 1) {
+ per_cpu(cpu_core_map, cpu) = per_cpu(cpu_sibling_map, cpu);
+ c->booted_cores = 1;
+ return;
+ }
+
+ for_each_cpu_mask(i, cpu_sibling_setup_map) {
+ if (per_cpu(cpu_llc_id, cpu) != BAD_APICID &&
+ per_cpu(cpu_llc_id, cpu) == per_cpu(cpu_llc_id, i)) {
+ cpu_set(i, c->llc_shared_map);
+ cpu_set(cpu, cpu_data(i).llc_shared_map);
+ }
+ if (c->phys_proc_id == cpu_data(i).phys_proc_id) {
+ cpu_set(i, per_cpu(cpu_core_map, cpu));
+ cpu_set(cpu, per_cpu(cpu_core_map, i));
+ /*
+ * Does this new cpu bringup a new core?
+ */
+ if (cpus_weight(per_cpu(cpu_sibling_map, cpu)) == 1) {
+ /*
+ * for each core in package, increment
+ * the booted_cores for this new cpu
+ */
+ if (first_cpu(per_cpu(cpu_sibling_map, i)) == i)
+ c->booted_cores++;
+ /*
+ * increment the core count for all
+ * the other cpus in this package
+ */
+ if (i != cpu)
+ cpu_data(i).booted_cores++;
+ } else if (i != cpu && !c->booted_cores)
+ c->booted_cores = cpu_data(i).booted_cores;
+ }
+ }
+}
+
#ifdef CONFIG_HOTPLUG_CPU
+void remove_siblinginfo(int cpu)
+{
+ int sibling;
+ struct cpuinfo_x86 *c = &cpu_data(cpu);
+
+ for_each_cpu_mask(sibling, per_cpu(cpu_core_map, cpu)) {
+ cpu_clear(cpu, per_cpu(cpu_core_map, sibling));
+ /*/
+ * last thread sibling in this cpu core going down
+ */
+ if (cpus_weight(per_cpu(cpu_sibling_map, cpu)) == 1)
+ cpu_data(sibling).booted_cores--;
+ }
+
+ for_each_cpu_mask(sibling, per_cpu(cpu_sibling_map, cpu))
+ cpu_clear(cpu, per_cpu(cpu_sibling_map, sibling));
+ cpus_clear(per_cpu(cpu_sibling_map, cpu));
+ cpus_clear(per_cpu(cpu_core_map, cpu));
+ c->phys_proc_id = 0;
+ c->cpu_core_id = 0;
+ cpu_clear(cpu, cpu_sibling_setup_map);
+}
int additional_cpus __initdata = -1;
diff --git a/arch/x86/kernel/smpboot_32.c b/arch/x86/kernel/smpboot_32.c
index 0fbc981..322f466 100644
--- a/arch/x86/kernel/smpboot_32.c
+++ b/arch/x86/kernel/smpboot_32.c
@@ -274,71 +274,6 @@ cpumask_t cpu_coregroup_map(int cpu)
return c->llc_shared_map;
}
-/* representing cpus for which sibling maps can be computed */
-static cpumask_t cpu_sibling_setup_map;
-
-void __cpuinit set_cpu_sibling_map(int cpu)
-{
- int i;
- struct cpuinfo_x86 *c = &cpu_data(cpu);
-
- cpu_set(cpu, cpu_sibling_setup_map);
-
- if (smp_num_siblings > 1) {
- for_each_cpu_mask(i, cpu_sibling_setup_map) {
- if (c->phys_proc_id == cpu_data(i).phys_proc_id &&
- c->cpu_core_id == cpu_data(i).cpu_core_id) {
- cpu_set(i, per_cpu(cpu_sibling_map, cpu));
- cpu_set(cpu, per_cpu(cpu_sibling_map, i));
- cpu_set(i, per_cpu(cpu_core_map, cpu));
- cpu_set(cpu, per_cpu(cpu_core_map, i));
- cpu_set(i, c->llc_shared_map);
- cpu_set(cpu, cpu_data(i).llc_shared_map);
- }
- }
- } else {
- cpu_set(cpu, per_cpu(cpu_sibling_map, cpu));
- }
-
- cpu_set(cpu, c->llc_shared_map);
-
- if (current_cpu_data.x86_max_cores == 1) {
- per_cpu(cpu_core_map, cpu) = per_cpu(cpu_sibling_map, cpu);
- c->booted_cores = 1;
- return;
- }
-
- for_each_cpu_mask(i, cpu_sibling_setup_map) {
- if (per_cpu(cpu_llc_id, cpu) != BAD_APICID &&
- per_cpu(cpu_llc_id, cpu) == per_cpu(cpu_llc_id, i)) {
- cpu_set(i, c->llc_shared_map);
- cpu_set(cpu, cpu_data(i).llc_shared_map);
- }
- if (c->phys_proc_id == cpu_data(i).phys_proc_id) {
- cpu_set(i, per_cpu(cpu_core_map, cpu));
- cpu_set(cpu, per_cpu(cpu_core_map, i));
- /*
- * Does this new cpu bringup a new core?
- */
- if (cpus_weight(per_cpu(cpu_sibling_map, cpu)) == 1) {
- /*
- * for each core in package, increment
- * the booted_cores for this new cpu
- */
- if (first_cpu(per_cpu(cpu_sibling_map, i)) == i)
- c->booted_cores++;
- /*
- * increment the core count for all
- * the other cpus in this package
- */
- if (i != cpu)
- cpu_data(i).booted_cores++;
- } else if (i != cpu && !c->booted_cores)
- c->booted_cores = cpu_data(i).booted_cores;
- }
- }
-}
-
/*
* Activate a secondary processor.
*/
@@ -1120,29 +1055,6 @@ void __init native_smp_prepare_boot_cpu(void)
}
#ifdef CONFIG_HOTPLUG_CPU
-void remove_siblinginfo(int cpu)
-{
- int sibling;
- struct cpuinfo_x86 *c = &cpu_data(cpu);
-
- for_each_cpu_mask(sibling, per_cpu(cpu_core_map, cpu)) {
- cpu_clear(cpu, per_cpu(cpu_core_map, sibling));
- /*/
- * last thread sibling in this cpu core going down
- */
- if (cpus_weight(per_cpu(cpu_sibling_map, cpu)) == 1)
- cpu_data(sibling).booted_cores--;
- }
-
- for_each_cpu_mask(sibling, per_cpu(cpu_sibling_map, cpu))
- cpu_clear(cpu, per_cpu(cpu_sibling_map, sibling));
- cpus_clear(per_cpu(cpu_sibling_map, cpu));
- cpus_clear(per_cpu(cpu_core_map, cpu));
- c->phys_proc_id = 0;
- c->cpu_core_id = 0;
- cpu_clear(cpu, cpu_sibling_setup_map);
-}
-
int __cpu_disable(void)
{
cpumask_t map = cpu_online_map;
diff --git a/arch/x86/kernel/smpboot_64.c b/arch/x86/kernel/smpboot_64.c
index 20f1c7d..329f9c5 100644
--- a/arch/x86/kernel/smpboot_64.c
+++ b/arch/x86/kernel/smpboot_64.c
@@ -225,71 +225,6 @@ cpumask_t cpu_coregroup_map(int cpu)
return c->llc_shared_map;
}
-/* representing cpus for which sibling maps can be computed */
-static cpumask_t cpu_sibling_setup_map;
-
-void __cpuinit set_cpu_sibling_map(int cpu)
-{
- int i;
- struct cpuinfo_x86 *c = &cpu_data(cpu);
-
- cpu_set(cpu, cpu_sibling_setup_map);
-
- if (smp_num_siblings > 1) {
- for_each_cpu_mask(i, cpu_sibling_setup_map) {
- if (c->phys_proc_id == cpu_data(i).phys_proc_id &&
- c->cpu_core_id == cpu_data(i).cpu_core_id) {
- cpu_set(i, per_cpu(cpu_sibling_map, cpu));
- cpu_set(cpu, per_cpu(cpu_sibling_map, i));
- cpu_set(i, per_cpu(cpu_core_map, cpu));
- cpu_set(cpu, per_cpu(cpu_core_map, i));
- cpu_set(i, c->llc_shared_map);
- cpu_set(cpu, cpu_data(i).llc_shared_map);
- }
- }
- } else {
- cpu_set(cpu, per_cpu(cpu_sibling_map, cpu));
- }
-
- cpu_set(cpu, c->llc_shared_map);
-
- if (current_cpu_data.x86_max_cores == 1) {
- per_cpu(cpu_core_map, cpu) = per_cpu(cpu_sibling_map, cpu);
- c->booted_cores = 1;
- return;
- }
-
- for_each_cpu_mask(i, cpu_sibling_setup_map) {
- if (per_cpu(cpu_llc_id, cpu) != BAD_APICID &&
- per_cpu(cpu_llc_id, cpu) == per_cpu(cpu_llc_id, i)) {
- cpu_set(i, c->llc_shared_map);
- cpu_set(cpu, cpu_data(i).llc_shared_map);
- }
- if (c->phys_proc_id == cpu_data(i).phys_proc_id) {
- cpu_set(i, per_cpu(cpu_core_map, cpu));
- cpu_set(cpu, per_cpu(cpu_core_map, i));
- /*
- * Does this new cpu bringup a new core?
- */
- if (cpus_weight(per_cpu(cpu_sibling_map, cpu)) == 1) {
- /*
- * for each core in package, increment
- * the booted_cores for this new cpu
- */
- if (first_cpu(per_cpu(cpu_sibling_map, i)) == i)
- c->booted_cores++;
- /*
- * increment the core count for all
- * the other cpus in this package
- */
- if (i != cpu)
- cpu_data(i).booted_cores++;
- } else if (i != cpu && !c->booted_cores)
- c->booted_cores = cpu_data(i).booted_cores;
- }
- }
-}
-
/*
* Setup code on secondary processor (after comming out of the trampoline)
*/
@@ -917,30 +852,6 @@ void __init native_smp_cpus_done(unsigned int max_cpus)
}
#ifdef CONFIG_HOTPLUG_CPU
-
-void remove_siblinginfo(int cpu)
-{
- int sibling;
- struct cpuinfo_x86 *c = &cpu_data(cpu);
-
- for_each_cpu_mask(sibling, per_cpu(cpu_core_map, cpu)) {
- cpu_clear(cpu, per_cpu(cpu_core_map, sibling));
- /*
- * last thread sibling in this cpu core going down
- */
- if (cpus_weight(per_cpu(cpu_sibling_map, cpu)) == 1)
- cpu_data(sibling).booted_cores--;
- }
-
- for_each_cpu_mask(sibling, per_cpu(cpu_sibling_map, cpu))
- cpu_clear(cpu, per_cpu(cpu_sibling_map, sibling));
- cpus_clear(per_cpu(cpu_sibling_map, cpu));
- cpus_clear(per_cpu(cpu_core_map, cpu));
- c->phys_proc_id = 0;
- c->cpu_core_id = 0;
- cpu_clear(cpu, cpu_sibling_setup_map);
-}
-
static void __ref remove_cpu_from_maps(void)
{
int cpu = smp_processor_id();
--
1.5.0.6
it is already used in x86_64. In i386, it only
removes from cpu_online_map
Signed-off-by: Glauber Costa <[email protected]>
---
arch/x86/kernel/smpboot_32.c | 7 ++++++-
arch/x86/kernel/smpboot_64.c | 8 +++-----
2 files changed, 9 insertions(+), 6 deletions(-)
diff --git a/arch/x86/kernel/smpboot_32.c b/arch/x86/kernel/smpboot_32.c
index a58ca7f..4939b3a 100644
--- a/arch/x86/kernel/smpboot_32.c
+++ b/arch/x86/kernel/smpboot_32.c
@@ -1041,6 +1041,11 @@ void __init native_smp_prepare_boot_cpu(void)
}
#ifdef CONFIG_HOTPLUG_CPU
+static void __ref remove_cpu_from_maps(int cpu)
+{
+ cpu_clear(cpu, cpu_online_map);
+}
+
int __cpu_disable(void)
{
cpumask_t map = cpu_online_map;
@@ -1066,7 +1071,7 @@ int __cpu_disable(void)
remove_siblinginfo(cpu);
- cpu_clear(cpu, map);
+ remove_cpu_from_maps(cpu);
fixup_irqs(map);
/* It's now safe to remove this processor from the online map */
cpu_clear(cpu, cpu_online_map);
diff --git a/arch/x86/kernel/smpboot_64.c b/arch/x86/kernel/smpboot_64.c
index ca3a3c5..6509d3c 100644
--- a/arch/x86/kernel/smpboot_64.c
+++ b/arch/x86/kernel/smpboot_64.c
@@ -838,10 +838,9 @@ void __init native_smp_cpus_done(unsigned int max_cpus)
}
#ifdef CONFIG_HOTPLUG_CPU
-static void __ref remove_cpu_from_maps(void)
+static void __ref remove_cpu_from_maps(int cpu)
{
- int cpu = smp_processor_id();
-
+ cpu_clear(cpu, cpu_online_map);
cpu_clear(cpu, cpu_callout_map);
cpu_clear(cpu, cpu_callin_map);
clear_bit(cpu, (unsigned long *)&cpu_initialized); /* was set by cpu_init() */
@@ -880,8 +879,7 @@ int __cpu_disable(void)
remove_siblinginfo(cpu);
/* It's now safe to remove this processor from the online map */
- cpu_clear(cpu, cpu_online_map);
- remove_cpu_from_maps();
+ remove_cpu_from_maps(cpu);
fixup_irqs(cpu_online_map);
return 0;
}
--
1.5.0.6
Glauber Costa <[email protected]> writes:
>
> Comments are welcome.
The main difference between i386 and x86-64 smp boot up (among a lot
of quirks and workarounds in i386 that x86-64 doesn't have/need and
some trivialities) is that x86-64 follows the standard hotplug state
machine and i386 doesn't.
I would suggest that if you want to unify you convert i386 over to the new
state machine first because that is the key difference. Everything else
after that is relatively simple. You can probably pattern that after
the original changeset who did this for x86-64.
The reason I never attempted this myself is that i was too worried about
regressions on old machines for i386 (this code is partly very fragile)
You'll likely encounter that problem. Just make sure to keep a stable
email address for some years to handle the fallout.
-Andi
* Glauber Costa <[email protected]> wrote:
> Hi,
>
> This series of patches does a first attempt at smp integration (I call
> it first, although I don't believe it's too far away from a real one).
> The series comprises 52 patches, and the full diffstat says in the
> end:
>
> 34 files changed, 1858 insertions(+), 2344 deletions(-)
>
> smp_32.c and smp_64.c are completely gone. smp.h and smpboot_{32,64}.c
> are not, although substantially reduced. I know that ideally, no
> leftovers are wanted, but the series were already big enough (I tried
> to split the patches as much as I could), so I considered this a good
> cutting point. If this is all agreed upon and merged, I'll start
> looking at the other bits that are left.
>
> All patches are tested is various random x86 subarchitectures, and all
> of them works. The result also boots fine in mine x86_64 and i386
> boxes.
wow, really nice work and the result looks awesome! I'll put this into
x86.git and see how it goes.
Ingo
On Mon, Mar 3, 2008 at 2:22 PM, Ingo Molnar <[email protected]> wrote:
>
> * Glauber Costa <[email protected]> wrote:
>
> > Hi,
> >
> > This series of patches does a first attempt at smp integration (I call
> > it first, although I don't believe it's too far away from a real one).
> > The series comprises 52 patches, and the full diffstat says in the
> > end:
> >
> > 34 files changed, 1858 insertions(+), 2344 deletions(-)
> >
> > smp_32.c and smp_64.c are completely gone. smp.h and smpboot_{32,64}.c
> > are not, although substantially reduced. I know that ideally, no
> > leftovers are wanted, but the series were already big enough (I tried
> > to split the patches as much as I could), so I considered this a good
> > cutting point. If this is all agreed upon and merged, I'll start
> > looking at the other bits that are left.
> >
> > All patches are tested is various random x86 subarchitectures, and all
> > of them works. The result also boots fine in mine x86_64 and i386
> > boxes.
>
> wow, really nice work and the result looks awesome! I'll put this into
> x86.git and see how it goes.
>
> Ingo
>
Thanks ingo. I'll hook up to the next part as soon as I can.
--
Glauber Costa.
"Free as in Freedom"
http://glommer.net
"The less confident you are, the more serious you have to act."
On Mon, Mar 3, 2008 at 2:23 PM, Andi Kleen <[email protected]> wrote:
> Glauber Costa <[email protected]> writes:
> >
> > Comments are welcome.
>
> The main difference between i386 and x86-64 smp boot up (among a lot
> of quirks and workarounds in i386 that x86-64 doesn't have/need and
> some trivialities) is that x86-64 follows the standard hotplug state
> machine and i386 doesn't.
>
> I would suggest that if you want to unify you convert i386 over to the new
> state machine first because that is the key difference. Everything else
> after that is relatively simple. You can probably pattern that after
> the original changeset who did this for x86-64.
>
> The reason I never attempted this myself is that i was too worried about
> regressions on old machines for i386 (this code is partly very fragile)
> You'll likely encounter that problem. Just make sure to keep a stable
> email address for some years to handle the fallout.
Thanks Andi, all very helpful.
I'll try to handle the problems in my configuration, but I'm too very
sure they'll appear, and other people testing it will surely add up.
As for the e-mail address, I can list my personal address in the top
of the file, or make sure red hat won't fire me. Maybe I can do both.
--
Glauber Costa.
"Free as in Freedom"
http://glommer.net
"The less confident you are, the more serious you have to act."