Zhang Liguang report a bug as bellow:
1) system detected cmci storm on current cpu
2) disable cmci interrupt on banks ownd by current cpu, then swiching to poll mode
3) a few minites later, system swiching to interrupt mode on current cpu
4) we expect system to reenable cmci interrupt on banks ownd by current cpu
mce_intel_adjust_timer
|-> cmci_reenable
|-> cmci_discover # but, ownd banks is ignore here
> static void cmci_discover(int banks)
> ...
> for (i = 0; i < banks; i++) {
> ...
> if (test_bit(i, owned)) # ownd banks is ignore here
> continue;
In this patch, we add a func cmci_storm_set_cmci(), just to enable or
disable banks which ownd by current cpu without clean the ownd flags.
Reported-by: Zhang Liguang <[email protected]>
Cc: [email protected] # v3.15+
Signed-off-by: Xie XiuQi <[email protected]>
---
arch/x86/kernel/cpu/mcheck/mce_intel.c | 41 +++++++++++++++++++---------------
1 file changed, 23 insertions(+), 18 deletions(-)
diff --git a/arch/x86/kernel/cpu/mcheck/mce_intel.c b/arch/x86/kernel/cpu/mcheck/mce_intel.c
index 844f56c..a20e18b 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_intel.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_intel.c
@@ -146,6 +146,27 @@ void mce_intel_hcpu_update(unsigned long cpu)
per_cpu(cmci_storm_state, cpu) = CMCI_STORM_NONE;
}
+static void cmci_storm_set_cmci(bool on)
+{
+ unsigned long flags, *owned;
+ int bank;
+ u64 val;
+
+ raw_spin_lock_irqsave(&cmci_discover_lock, flags);
+ owned = this_cpu_ptr(mce_banks_owned);
+ for_each_set_bit(bank, owned, MAX_NR_BANKS) {
+ rdmsrl(MSR_IA32_MCx_CTL2(bank), val);
+
+ if (on)
+ val |= MCI_CTL2_CMCI_EN;
+ else
+ val &= ~MCI_CTL2_CMCI_EN;
+
+ wrmsrl(MSR_IA32_MCx_CTL2(bank), val);
+ }
+ raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
+}
+
unsigned long cmci_intel_adjust_timer(unsigned long interval)
{
if ((this_cpu_read(cmci_backoff_cnt) > 0) &&
@@ -175,7 +196,7 @@ unsigned long cmci_intel_adjust_timer(unsigned long interval)
*/
if (!atomic_read(&cmci_storm_on_cpus)) {
__this_cpu_write(cmci_storm_state, CMCI_STORM_NONE);
- cmci_reenable();
+ cmci_storm_set_cmci(true);
cmci_recheck();
}
return CMCI_POLL_INTERVAL;
@@ -186,22 +207,6 @@ unsigned long cmci_intel_adjust_timer(unsigned long interval)
}
}
-static void cmci_storm_disable_banks(void)
-{
- unsigned long flags, *owned;
- int bank;
- u64 val;
-
- raw_spin_lock_irqsave(&cmci_discover_lock, flags);
- owned = this_cpu_ptr(mce_banks_owned);
- for_each_set_bit(bank, owned, MAX_NR_BANKS) {
- rdmsrl(MSR_IA32_MCx_CTL2(bank), val);
- val &= ~MCI_CTL2_CMCI_EN;
- wrmsrl(MSR_IA32_MCx_CTL2(bank), val);
- }
- raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
-}
-
static bool cmci_storm_detect(void)
{
unsigned int cnt = __this_cpu_read(cmci_storm_cnt);
@@ -223,7 +228,7 @@ static bool cmci_storm_detect(void)
if (cnt <= CMCI_STORM_THRESHOLD)
return false;
- cmci_storm_disable_banks();
+ cmci_storm_set_cmci(false);
__this_cpu_write(cmci_storm_state, CMCI_STORM_ACTIVE);
r = atomic_add_return(1, &cmci_storm_on_cpus);
mce_timer_kick(CMCI_STORM_INTERVAL);
--
2.0.0
On Wed, Aug 12, 2015 at 10:51:11AM +0800, Xie XiuQi wrote:
> Zhang Liguang report a bug as bellow:
> 1) system detected cmci storm on current cpu
> 2) disable cmci interrupt on banks ownd by current cpu, then swiching to poll mode
> 3) a few minites later, system swiching to interrupt mode on current cpu
> 4) we expect system to reenable cmci interrupt on banks ownd by current cpu
> mce_intel_adjust_timer
> |-> cmci_reenable
> |-> cmci_discover # but, ownd banks is ignore here
>
> > static void cmci_discover(int banks)
> > ...
> > for (i = 0; i < banks; i++) {
> > ...
> > if (test_bit(i, owned)) # ownd banks is ignore here
> > continue;
>
> In this patch, we add a func cmci_storm_set_cmci(), just to enable or
Yeah, that's too many "cmci"'s in the name. Here's what I committed:
---
From: Xie XiuQi <[email protected]>
Date: Wed, 12 Aug 2015 10:51:11 +0800
Subject: [PATCH] x86/mce: Reenable CMCI banks when swiching back to interrupt mode
Zhang Liguang reported the following issue:
1) System detects a CMCI storm on the current CPU.
2) Kernel disables the CMCI interrupt on banks owned by the current CPU and
switches to poll mode
3) After the CMCI storm subsides, kernel switches back to interrupt mode
4) We expect the system to reenable the CMCI interrupt on banks owned by
the current CPU
mce_intel_adjust_timer
|-> cmci_reenable
|-> cmci_discover # owned banks are ignored here
static void cmci_discover(int banks)
...
for (i = 0; i < banks; i++) {
...
if (test_bit(i, owned)) # ownd banks is ignore here
continue;
So convert cmci_storm_disable_banks() to cmci_toggle_interrupt_mode()
which controls whether to enable or disable CMCI interrupts with its
argument.
NB: We cannot clear the owned bit because the banks won't be polled,
otherwise. See
27f6c573e0f7 ("x86, CMCI: Add proper detection of end of CMCI storms")
for more info.
Reported-by: Zhang Liguang <[email protected]>
Signed-off-by: Xie XiuQi <[email protected]>
Cc: <[email protected]> # v3.15+
Cc: "H. Peter Anvin" <[email protected]>
Cc: [email protected]
Cc: Ingo Molnar <[email protected]>
Cc: linux-edac <[email protected]>
Cc: [email protected]
Cc: Thomas Gleixner <[email protected]>
Cc: Tony Luck <[email protected]>
Cc: x86-ml <[email protected]>
Link: http://lkml.kernel.org/r/[email protected]
Signed-off-by: Borislav Petkov <[email protected]>
---
arch/x86/kernel/cpu/mcheck/mce_intel.c | 41 +++++++++++++++++++---------------
1 file changed, 23 insertions(+), 18 deletions(-)
diff --git a/arch/x86/kernel/cpu/mcheck/mce_intel.c b/arch/x86/kernel/cpu/mcheck/mce_intel.c
index c5c003291861..1e8bb6c94f14 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_intel.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_intel.c
@@ -146,6 +146,27 @@ void mce_intel_hcpu_update(unsigned long cpu)
per_cpu(cmci_storm_state, cpu) = CMCI_STORM_NONE;
}
+static void cmci_toggle_interrupt_mode(bool on)
+{
+ unsigned long flags, *owned;
+ int bank;
+ u64 val;
+
+ raw_spin_lock_irqsave(&cmci_discover_lock, flags);
+ owned = this_cpu_ptr(mce_banks_owned);
+ for_each_set_bit(bank, owned, MAX_NR_BANKS) {
+ rdmsrl(MSR_IA32_MCx_CTL2(bank), val);
+
+ if (on)
+ val |= MCI_CTL2_CMCI_EN;
+ else
+ val &= ~MCI_CTL2_CMCI_EN;
+
+ wrmsrl(MSR_IA32_MCx_CTL2(bank), val);
+ }
+ raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
+}
+
unsigned long cmci_intel_adjust_timer(unsigned long interval)
{
if ((this_cpu_read(cmci_backoff_cnt) > 0) &&
@@ -175,7 +196,7 @@ unsigned long cmci_intel_adjust_timer(unsigned long interval)
*/
if (!atomic_read(&cmci_storm_on_cpus)) {
__this_cpu_write(cmci_storm_state, CMCI_STORM_NONE);
- cmci_reenable();
+ cmci_toggle_interrupt_mode(true);
cmci_recheck();
}
return CMCI_POLL_INTERVAL;
@@ -186,22 +207,6 @@ unsigned long cmci_intel_adjust_timer(unsigned long interval)
}
}
-static void cmci_storm_disable_banks(void)
-{
- unsigned long flags, *owned;
- int bank;
- u64 val;
-
- raw_spin_lock_irqsave(&cmci_discover_lock, flags);
- owned = this_cpu_ptr(mce_banks_owned);
- for_each_set_bit(bank, owned, MAX_NR_BANKS) {
- rdmsrl(MSR_IA32_MCx_CTL2(bank), val);
- val &= ~MCI_CTL2_CMCI_EN;
- wrmsrl(MSR_IA32_MCx_CTL2(bank), val);
- }
- raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
-}
-
static bool cmci_storm_detect(void)
{
unsigned int cnt = __this_cpu_read(cmci_storm_cnt);
@@ -223,7 +228,7 @@ static bool cmci_storm_detect(void)
if (cnt <= CMCI_STORM_THRESHOLD)
return false;
- cmci_storm_disable_banks();
+ cmci_toggle_interrupt_mode(false);
__this_cpu_write(cmci_storm_state, CMCI_STORM_ACTIVE);
r = atomic_add_return(1, &cmci_storm_on_cpus);
mce_timer_kick(CMCI_STORM_INTERVAL);
--
2.5.0.rc2.28.g6003e7f
--
Regards/Gruss,
Boris.
ECO tip #101: Trim your mails when you reply.
--
On 2015/8/12 17:54, Borislav Petkov wrote:
> On Wed, Aug 12, 2015 at 10:51:11AM +0800, Xie XiuQi wrote:
>> Zhang Liguang report a bug as bellow:
>> 1) system detected cmci storm on current cpu
>> 2) disable cmci interrupt on banks ownd by current cpu, then swiching to poll mode
>> 3) a few minites later, system swiching to interrupt mode on current cpu
>> 4) we expect system to reenable cmci interrupt on banks ownd by current cpu
>> mce_intel_adjust_timer
>> |-> cmci_reenable
>> |-> cmci_discover # but, ownd banks is ignore here
>>
>>> static void cmci_discover(int banks)
>>> ...
>>> for (i = 0; i < banks; i++) {
>>> ...
>>> if (test_bit(i, owned)) # ownd banks is ignore here
>>> continue;
>>
>> In this patch, we add a func cmci_storm_set_cmci(), just to enable or
>
> Yeah, that's too many "cmci"'s in the name. Here's what I committed:
It looks much better than me.
Thanks.
>
> ---
> From: Xie XiuQi <[email protected]>
> Date: Wed, 12 Aug 2015 10:51:11 +0800
> Subject: [PATCH] x86/mce: Reenable CMCI banks when swiching back to interrupt mode
>
> Zhang Liguang reported the following issue:
>
> 1) System detects a CMCI storm on the current CPU.
>
> 2) Kernel disables the CMCI interrupt on banks owned by the current CPU and
> switches to poll mode
>
> 3) After the CMCI storm subsides, kernel switches back to interrupt mode
>
> 4) We expect the system to reenable the CMCI interrupt on banks owned by
> the current CPU
>
> mce_intel_adjust_timer
> |-> cmci_reenable
> |-> cmci_discover # owned banks are ignored here
>
> static void cmci_discover(int banks)
> ...
> for (i = 0; i < banks; i++) {
> ...
> if (test_bit(i, owned)) # ownd banks is ignore here
> continue;
>
> So convert cmci_storm_disable_banks() to cmci_toggle_interrupt_mode()
> which controls whether to enable or disable CMCI interrupts with its
> argument.
>
> NB: We cannot clear the owned bit because the banks won't be polled,
> otherwise. See
>
> 27f6c573e0f7 ("x86, CMCI: Add proper detection of end of CMCI storms")
>
> for more info.
>
> Reported-by: Zhang Liguang <[email protected]>
> Signed-off-by: Xie XiuQi <[email protected]>
> Cc: <[email protected]> # v3.15+
> Cc: "H. Peter Anvin" <[email protected]>
> Cc: [email protected]
> Cc: Ingo Molnar <[email protected]>
> Cc: linux-edac <[email protected]>
> Cc: [email protected]
> Cc: Thomas Gleixner <[email protected]>
> Cc: Tony Luck <[email protected]>
> Cc: x86-ml <[email protected]>
> Link: http://lkml.kernel.org/r/[email protected]
> Signed-off-by: Borislav Petkov <[email protected]>
> ---
> arch/x86/kernel/cpu/mcheck/mce_intel.c | 41 +++++++++++++++++++---------------
> 1 file changed, 23 insertions(+), 18 deletions(-)
>
> diff --git a/arch/x86/kernel/cpu/mcheck/mce_intel.c b/arch/x86/kernel/cpu/mcheck/mce_intel.c
> index c5c003291861..1e8bb6c94f14 100644
> --- a/arch/x86/kernel/cpu/mcheck/mce_intel.c
> +++ b/arch/x86/kernel/cpu/mcheck/mce_intel.c
> @@ -146,6 +146,27 @@ void mce_intel_hcpu_update(unsigned long cpu)
> per_cpu(cmci_storm_state, cpu) = CMCI_STORM_NONE;
> }
>
> +static void cmci_toggle_interrupt_mode(bool on)
> +{
> + unsigned long flags, *owned;
> + int bank;
> + u64 val;
> +
> + raw_spin_lock_irqsave(&cmci_discover_lock, flags);
> + owned = this_cpu_ptr(mce_banks_owned);
> + for_each_set_bit(bank, owned, MAX_NR_BANKS) {
> + rdmsrl(MSR_IA32_MCx_CTL2(bank), val);
> +
> + if (on)
> + val |= MCI_CTL2_CMCI_EN;
> + else
> + val &= ~MCI_CTL2_CMCI_EN;
> +
> + wrmsrl(MSR_IA32_MCx_CTL2(bank), val);
> + }
> + raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
> +}
> +
> unsigned long cmci_intel_adjust_timer(unsigned long interval)
> {
> if ((this_cpu_read(cmci_backoff_cnt) > 0) &&
> @@ -175,7 +196,7 @@ unsigned long cmci_intel_adjust_timer(unsigned long interval)
> */
> if (!atomic_read(&cmci_storm_on_cpus)) {
> __this_cpu_write(cmci_storm_state, CMCI_STORM_NONE);
> - cmci_reenable();
> + cmci_toggle_interrupt_mode(true);
> cmci_recheck();
> }
> return CMCI_POLL_INTERVAL;
> @@ -186,22 +207,6 @@ unsigned long cmci_intel_adjust_timer(unsigned long interval)
> }
> }
>
> -static void cmci_storm_disable_banks(void)
> -{
> - unsigned long flags, *owned;
> - int bank;
> - u64 val;
> -
> - raw_spin_lock_irqsave(&cmci_discover_lock, flags);
> - owned = this_cpu_ptr(mce_banks_owned);
> - for_each_set_bit(bank, owned, MAX_NR_BANKS) {
> - rdmsrl(MSR_IA32_MCx_CTL2(bank), val);
> - val &= ~MCI_CTL2_CMCI_EN;
> - wrmsrl(MSR_IA32_MCx_CTL2(bank), val);
> - }
> - raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
> -}
> -
> static bool cmci_storm_detect(void)
> {
> unsigned int cnt = __this_cpu_read(cmci_storm_cnt);
> @@ -223,7 +228,7 @@ static bool cmci_storm_detect(void)
> if (cnt <= CMCI_STORM_THRESHOLD)
> return false;
>
> - cmci_storm_disable_banks();
> + cmci_toggle_interrupt_mode(false);
> __this_cpu_write(cmci_storm_state, CMCI_STORM_ACTIVE);
> r = atomic_add_return(1, &cmci_storm_on_cpus);
> mce_timer_kick(CMCI_STORM_INTERVAL);
>