The following patch is to remove the pseries_notify_add_cpu() call
and replace it by a hot plug notifier.
This would prevent cpuidle resources being
released and allocated each time cpu comes online on pseries.
The earlier design was causing a lockdep problem
in start_secondary as reported on this thread
-https://lkml.org/lkml/2012/5/17/2
This applies on 3.4-rc7
Signed-off-by: Deepthi Dharwar <[email protected]>
---
arch/powerpc/include/asm/processor.h | 2 --
arch/powerpc/platforms/pseries/processor_idle.c | 25
+++++++++++++++++------
arch/powerpc/platforms/pseries/smp.c | 1 -
3 files changed, 19 insertions(+), 9 deletions(-)
diff --git a/arch/powerpc/include/asm/processor.h
b/arch/powerpc/include/asm/processor.h
index 8e2d037..c6bc22b 100644
--- a/arch/powerpc/include/asm/processor.h
+++ b/arch/powerpc/include/asm/processor.h
@@ -390,10 +390,8 @@ void cpu_idle_wait(void);
#ifdef CONFIG_PSERIES_IDLE
extern void update_smt_snooze_delay(int snooze);
-extern int pseries_notify_cpuidle_add_cpu(int cpu);
#else
static inline void update_smt_snooze_delay(int snooze) {}
-static inline int pseries_notify_cpuidle_add_cpu(int cpu) { return 0; }
#endif
extern void flush_instruction_cache(void);
diff --git a/arch/powerpc/platforms/pseries/processor_idle.c
b/arch/powerpc/platforms/pseries/processor_idle.c
index 41a34bc..d1a7dc0 100644
--- a/arch/powerpc/platforms/pseries/processor_idle.c
+++ b/arch/powerpc/platforms/pseries/processor_idle.c
@@ -11,6 +11,7 @@
#include <linux/moduleparam.h>
#include <linux/cpuidle.h>
#include <linux/cpu.h>
+#include <linux/notifier.h>
#include <asm/paca.h>
#include <asm/reg.h>
@@ -186,17 +187,28 @@ static struct cpuidle_state
shared_states[MAX_IDLE_STATE_COUNT] = {
.enter = &shared_cede_loop },
};
-int pseries_notify_cpuidle_add_cpu(int cpu)
+static int pseries_cpuidle_add_cpu_notifier(struct notifier_block *n,
+ unsigned long action, void *hcpu)
{
+ int hotcpu = (unsigned long)hcpu;
struct cpuidle_device *dev =
- per_cpu_ptr(pseries_cpuidle_devices, cpu);
- if (dev && cpuidle_get_driver()) {
- cpuidle_disable_device(dev);
- cpuidle_enable_device(dev);
+ per_cpu_ptr(pseries_cpuidle_devices, hotcpu);
+
+ switch (action & 0xf) {
+ case CPU_ONLINE:
+ if (dev && cpuidle_get_driver()) {
+ cpuidle_disable_device(dev);
+ cpuidle_enable_device(dev);
+ }
+ break;
}
- return 0;
+ return NOTIFY_OK;
}
+static struct notifier_block setup_hotplug_notifier = {
+ .notifier_call = pseries_cpuidle_add_cpu_notifier,
+};
+
/*
* pseries_cpuidle_driver_init()
*/
@@ -321,6 +333,7 @@ static int __init pseries_processor_idle_init(void)
return retval;
}
+ register_cpu_notifier(&setup_hotplug_notifier);
printk(KERN_DEBUG "pseries_idle_driver registered\n");
return 0;
diff --git a/arch/powerpc/platforms/pseries/smp.c
b/arch/powerpc/platforms/pseries/smp.c
index e16bb8d..71706bc 100644
--- a/arch/powerpc/platforms/pseries/smp.c
+++ b/arch/powerpc/platforms/pseries/smp.c
@@ -147,7 +147,6 @@ static void __devinit smp_xics_setup_cpu(int cpu)
set_cpu_current_state(cpu, CPU_STATE_ONLINE);
set_default_offline_state(cpu);
#endif
- pseries_notify_cpuidle_add_cpu(cpu);
}
static int __devinit smp_pSeries_kick_cpu(int nr)
On Fri, 2012-05-18 at 18:58 +0530, Deepthi Dharwar wrote:
> The following patch is to remove the pseries_notify_add_cpu() call
> and replace it by a hot plug notifier.
> This would prevent cpuidle resources being
> released and allocated each time cpu comes online on pseries.
> The earlier design was causing a lockdep problem
> in start_secondary as reported on this thread
> -https://lkml.org/lkml/2012/5/17/2
>
> This applies on 3.4-rc7
>
> Signed-off-by: Deepthi Dharwar <[email protected]>
> ---
Any reason why you don't do cpuidle_disable_device() when the
CPU is going offline and cpuidle_enable_device() when it's coming
back ?
I'm applying the patch for now since it fixes a real problem but
if the above makes sense, please send a followup fix.
Cheers,
Ben.
> arch/powerpc/include/asm/processor.h | 2 --
> arch/powerpc/platforms/pseries/processor_idle.c | 25
> +++++++++++++++++------
> arch/powerpc/platforms/pseries/smp.c | 1 -
> 3 files changed, 19 insertions(+), 9 deletions(-)
>
> diff --git a/arch/powerpc/include/asm/processor.h
> b/arch/powerpc/include/asm/processor.h
> index 8e2d037..c6bc22b 100644
> --- a/arch/powerpc/include/asm/processor.h
> +++ b/arch/powerpc/include/asm/processor.h
> @@ -390,10 +390,8 @@ void cpu_idle_wait(void);
>
> #ifdef CONFIG_PSERIES_IDLE
> extern void update_smt_snooze_delay(int snooze);
> -extern int pseries_notify_cpuidle_add_cpu(int cpu);
> #else
> static inline void update_smt_snooze_delay(int snooze) {}
> -static inline int pseries_notify_cpuidle_add_cpu(int cpu) { return 0; }
> #endif
>
> extern void flush_instruction_cache(void);
> diff --git a/arch/powerpc/platforms/pseries/processor_idle.c
> b/arch/powerpc/platforms/pseries/processor_idle.c
> index 41a34bc..d1a7dc0 100644
> --- a/arch/powerpc/platforms/pseries/processor_idle.c
> +++ b/arch/powerpc/platforms/pseries/processor_idle.c
> @@ -11,6 +11,7 @@
> #include <linux/moduleparam.h>
> #include <linux/cpuidle.h>
> #include <linux/cpu.h>
> +#include <linux/notifier.h>
>
> #include <asm/paca.h>
> #include <asm/reg.h>
> @@ -186,17 +187,28 @@ static struct cpuidle_state
> shared_states[MAX_IDLE_STATE_COUNT] = {
> .enter = &shared_cede_loop },
> };
>
> -int pseries_notify_cpuidle_add_cpu(int cpu)
> +static int pseries_cpuidle_add_cpu_notifier(struct notifier_block *n,
> + unsigned long action, void *hcpu)
> {
> + int hotcpu = (unsigned long)hcpu;
> struct cpuidle_device *dev =
> - per_cpu_ptr(pseries_cpuidle_devices, cpu);
> - if (dev && cpuidle_get_driver()) {
> - cpuidle_disable_device(dev);
> - cpuidle_enable_device(dev);
> + per_cpu_ptr(pseries_cpuidle_devices, hotcpu);
> +
> + switch (action & 0xf) {
> + case CPU_ONLINE:
> + if (dev && cpuidle_get_driver()) {
> + cpuidle_disable_device(dev);
> + cpuidle_enable_device(dev);
> + }
> + break;
> }
> - return 0;
> + return NOTIFY_OK;
> }
>
> +static struct notifier_block setup_hotplug_notifier = {
> + .notifier_call = pseries_cpuidle_add_cpu_notifier,
> +};
> +
> /*
> * pseries_cpuidle_driver_init()
> */
> @@ -321,6 +333,7 @@ static int __init pseries_processor_idle_init(void)
> return retval;
> }
>
> + register_cpu_notifier(&setup_hotplug_notifier);
> printk(KERN_DEBUG "pseries_idle_driver registered\n");
>
> return 0;
> diff --git a/arch/powerpc/platforms/pseries/smp.c
> b/arch/powerpc/platforms/pseries/smp.c
> index e16bb8d..71706bc 100644
> --- a/arch/powerpc/platforms/pseries/smp.c
> +++ b/arch/powerpc/platforms/pseries/smp.c
> @@ -147,7 +147,6 @@ static void __devinit smp_xics_setup_cpu(int cpu)
> set_cpu_current_state(cpu, CPU_STATE_ONLINE);
> set_default_offline_state(cpu);
> #endif
> - pseries_notify_cpuidle_add_cpu(cpu);
> }
>
> static int __devinit smp_pSeries_kick_cpu(int nr)
On Mon, 2012-05-21 at 10:47 +1000, Benjamin Herrenschmidt wrote:
> Any reason why you don't do cpuidle_disable_device() when the
> CPU is going offline and cpuidle_enable_device() when it's coming
> back ?
>
> I'm applying the patch for now since it fixes a real problem but
> if the above makes sense, please send a followup fix.
No I'm not ... it's damaged in some odd ways (some whitespace
missing here or there among others). Please resend a fixed one.
Cheers,
Ben.
The following patch is to remove the pseries_notify_add_cpu() call
and replace it by a hot plug notifier.
This would prevent cpuidle resources being
released and allocated each time cpu comes online on pseries.
The earlier design was causing a lockdep problem
in start_secondary as reported on this thread
-https://lkml.org/lkml/2012/5/17/2
This applies on 3.4-rc7
Signed-off-by: Deepthi Dharwar <[email protected]>
---
arch/powerpc/include/asm/processor.h | 2 --
arch/powerpc/platforms/pseries/processor_idle.c | 25 +++++++++++++++++------
arch/powerpc/platforms/pseries/smp.c | 1 -
3 files changed, 19 insertions(+), 9 deletions(-)
diff --git a/arch/powerpc/include/asm/processor.h b/arch/powerpc/include/asm/processor.h
index 8e2d037..c6bc22b 100644
--- a/arch/powerpc/include/asm/processor.h
+++ b/arch/powerpc/include/asm/processor.h
@@ -390,10 +390,8 @@ void cpu_idle_wait(void);
#ifdef CONFIG_PSERIES_IDLE
extern void update_smt_snooze_delay(int snooze);
-extern int pseries_notify_cpuidle_add_cpu(int cpu);
#else
static inline void update_smt_snooze_delay(int snooze) {}
-static inline int pseries_notify_cpuidle_add_cpu(int cpu) { return 0; }
#endif
extern void flush_instruction_cache(void);
diff --git a/arch/powerpc/platforms/pseries/processor_idle.c b/arch/powerpc/platforms/pseries/processor_idle.c
index 41a34bc..d1a7dc0 100644
--- a/arch/powerpc/platforms/pseries/processor_idle.c
+++ b/arch/powerpc/platforms/pseries/processor_idle.c
@@ -11,6 +11,7 @@
#include <linux/moduleparam.h>
#include <linux/cpuidle.h>
#include <linux/cpu.h>
+#include <linux/notifier.h>
#include <asm/paca.h>
#include <asm/reg.h>
@@ -186,17 +187,28 @@ static struct cpuidle_state shared_states[MAX_IDLE_STATE_COUNT] = {
.enter = &shared_cede_loop },
};
-int pseries_notify_cpuidle_add_cpu(int cpu)
+static int pseries_cpuidle_add_cpu_notifier(struct notifier_block *n,
+ unsigned long action, void *hcpu)
{
+ int hotcpu = (unsigned long)hcpu;
struct cpuidle_device *dev =
- per_cpu_ptr(pseries_cpuidle_devices, cpu);
- if (dev && cpuidle_get_driver()) {
- cpuidle_disable_device(dev);
- cpuidle_enable_device(dev);
+ per_cpu_ptr(pseries_cpuidle_devices, hotcpu);
+
+ switch (action & 0xf) {
+ case CPU_ONLINE:
+ if (dev && cpuidle_get_driver()) {
+ cpuidle_disable_device(dev);
+ cpuidle_enable_device(dev);
+ }
+ break;
}
- return 0;
+ return NOTIFY_OK;
}
+static struct notifier_block setup_hotplug_notifier = {
+ .notifier_call = pseries_cpuidle_add_cpu_notifier,
+};
+
/*
* pseries_cpuidle_driver_init()
*/
@@ -321,6 +333,7 @@ static int __init pseries_processor_idle_init(void)
return retval;
}
+ register_cpu_notifier(&setup_hotplug_notifier);
printk(KERN_DEBUG "pseries_idle_driver registered\n");
return 0;
diff --git a/arch/powerpc/platforms/pseries/smp.c b/arch/powerpc/platforms/pseries/smp.c
index e16bb8d..71706bc 100644
--- a/arch/powerpc/platforms/pseries/smp.c
+++ b/arch/powerpc/platforms/pseries/smp.c
@@ -147,7 +147,6 @@ static void __devinit smp_xics_setup_cpu(int cpu)
set_cpu_current_state(cpu, CPU_STATE_ONLINE);
set_default_offline_state(cpu);
#endif
- pseries_notify_cpuidle_add_cpu(cpu);
}
static int __devinit smp_pSeries_kick_cpu(int nr)
Hi Ben,
On 05/21/2012 06:17 AM, Benjamin Herrenschmidt wrote:
> On Fri, 2012-05-18 at 18:58 +0530, Deepthi Dharwar wrote:
>> The following patch is to remove the pseries_notify_add_cpu() call
>> and replace it by a hot plug notifier.
>> This would prevent cpuidle resources being
>> released and allocated each time cpu comes online on pseries.
>> The earlier design was causing a lockdep problem
>> in start_secondary as reported on this thread
>> -https://lkml.org/lkml/2012/5/17/2
>>
>> This applies on 3.4-rc7
>>
>> Signed-off-by: Deepthi Dharwar <[email protected]>
>> ---
>
> Any reason why you don't do cpuidle_disable_device() when the
> CPU is going offline and cpuidle_enable_device() when it's coming
> back ?
In the current design disable and enable device are called
when the cpu comes online. This is to make sure that we clean up and
re-register again. All the counters are reset.
Not calling cpu disable when cpu goes offline currently, would only
retain the counters right now.
But I could add a offline check and disable the device there, if that
results in cleaner design. I will test and send across the patch with
couple more pseries-idle fixes soon.
Thanks for your review comments !
> I'm applying the patch for now since it fixes a real problem but
> if the above makes sense, please send a followup fix.
>
> Cheers,
> Ben.
>
>> arch/powerpc/include/asm/processor.h | 2 --
>> arch/powerpc/platforms/pseries/processor_idle.c | 25
>> +++++++++++++++++------
>> arch/powerpc/platforms/pseries/smp.c | 1 -
>> 3 files changed, 19 insertions(+), 9 deletions(-)
>>
>> diff --git a/arch/powerpc/include/asm/processor.h
>> b/arch/powerpc/include/asm/processor.h
>> index 8e2d037..c6bc22b 100644
>> --- a/arch/powerpc/include/asm/processor.h
>> +++ b/arch/powerpc/include/asm/processor.h
>> @@ -390,10 +390,8 @@ void cpu_idle_wait(void);
>>
>> #ifdef CONFIG_PSERIES_IDLE
>> extern void update_smt_snooze_delay(int snooze);
>> -extern int pseries_notify_cpuidle_add_cpu(int cpu);
>> #else
>> static inline void update_smt_snooze_delay(int snooze) {}
>> -static inline int pseries_notify_cpuidle_add_cpu(int cpu) { return 0; }
>> #endif
>>
>> extern void flush_instruction_cache(void);
>> diff --git a/arch/powerpc/platforms/pseries/processor_idle.c
>> b/arch/powerpc/platforms/pseries/processor_idle.c
>> index 41a34bc..d1a7dc0 100644
>> --- a/arch/powerpc/platforms/pseries/processor_idle.c
>> +++ b/arch/powerpc/platforms/pseries/processor_idle.c
>> @@ -11,6 +11,7 @@
>> #include <linux/moduleparam.h>
>> #include <linux/cpuidle.h>
>> #include <linux/cpu.h>
>> +#include <linux/notifier.h>
>>
>> #include <asm/paca.h>
>> #include <asm/reg.h>
>> @@ -186,17 +187,28 @@ static struct cpuidle_state
>> shared_states[MAX_IDLE_STATE_COUNT] = {
>> .enter = &shared_cede_loop },
>> };
>>
>> -int pseries_notify_cpuidle_add_cpu(int cpu)
>> +static int pseries_cpuidle_add_cpu_notifier(struct notifier_block *n,
>> + unsigned long action, void *hcpu)
>> {
>> + int hotcpu = (unsigned long)hcpu;
>> struct cpuidle_device *dev =
>> - per_cpu_ptr(pseries_cpuidle_devices, cpu);
>> - if (dev && cpuidle_get_driver()) {
>> - cpuidle_disable_device(dev);
>> - cpuidle_enable_device(dev);
>> + per_cpu_ptr(pseries_cpuidle_devices, hotcpu);
>> +
>> + switch (action & 0xf) {
>> + case CPU_ONLINE:
>> + if (dev && cpuidle_get_driver()) {
>> + cpuidle_disable_device(dev);
>> + cpuidle_enable_device(dev);
>> + }
>> + break;
>> }
>> - return 0;
>> + return NOTIFY_OK;
>> }
>>
>> +static struct notifier_block setup_hotplug_notifier = {
>> + .notifier_call = pseries_cpuidle_add_cpu_notifier,
>> +};
>> +
>> /*
>> * pseries_cpuidle_driver_init()
>> */
>> @@ -321,6 +333,7 @@ static int __init pseries_processor_idle_init(void)
>> return retval;
>> }
>>
>> + register_cpu_notifier(&setup_hotplug_notifier);
>> printk(KERN_DEBUG "pseries_idle_driver registered\n");
>>
>> return 0;
>> diff --git a/arch/powerpc/platforms/pseries/smp.c
>> b/arch/powerpc/platforms/pseries/smp.c
>> index e16bb8d..71706bc 100644
>> --- a/arch/powerpc/platforms/pseries/smp.c
>> +++ b/arch/powerpc/platforms/pseries/smp.c
>> @@ -147,7 +147,6 @@ static void __devinit smp_xics_setup_cpu(int cpu)
>> set_cpu_current_state(cpu, CPU_STATE_ONLINE);
>> set_default_offline_state(cpu);
>> #endif
>> - pseries_notify_cpuidle_add_cpu(cpu);
>> }
>>
>> static int __devinit smp_pSeries_kick_cpu(int nr)
>
>
> _______________________________________________
> Linuxppc-dev mailing list
> [email protected]
> https://lists.ozlabs.org/listinfo/linuxppc-dev
>