My Del Latitude D600 (single 32-bit CPU) shows the following locking message
when booted:
============================================
WARNING: possible recursive locking detected
4.12.0-rc2-00029-gfc8dffd #159 Not tainted
--------------------------------------------
systemd-udevd/153 is trying to acquire lock:
(cpu_hotplug_lock.rw_sem){.+.+.+}, at: [<c030fc26>] stop_machine+0x16/0x30
but task is already holding lock:
(cpu_hotplug_lock.rw_sem){.+.+.+}, at: [<c0234353>] mtrr_add_page+0x83/0x470
other info that might help us debug this:
Possible unsafe locking scenario:
CPU0
----
lock(cpu_hotplug_lock.rw_sem);
lock(cpu_hotplug_lock.rw_sem);
*** DEADLOCK ***
May be due to missing lock nesting notation
4 locks held by systemd-udevd/153:
#0: (&dev->mutex){......}, at: [<c06c8c23>] __driver_attach+0x53/0x100
#1: (&dev->mutex){......}, at: [<c06c8c2f>] __driver_attach+0x5f/0x100
#2: (cpu_hotplug_lock.rw_sem){.+.+.+}, at: [<c0234353>] mtrr_add_page+0x83/0x470
#3: (mtrr_mutex){+.+...}, at: [<c023435f>] mtrr_add_page+0x8f/0x470
stack backtrace:
CPU: 0 PID: 153 Comm: systemd-udevd Not tainted 4.12.0-rc2-00029-gfc8dffd #159
Hardware name: Dell Computer Corporation Latitude D600
/03U652, BIOS A05 05/29/2003
Call Trace:
dump_stack+0x76/0xb2
__lock_acquire+0xecc/0x1070
? mtrr_restore+0x80/0x80
lock_acquire+0xa6/0x1e0
? stop_machine+0x16/0x30
? mtrr_restore+0x80/0x80
cpus_read_lock+0x48/0x90
? stop_machine+0x16/0x30
stop_machine+0x16/0x30
mtrr_add_page+0x18b/0x470
mtrr_add+0x3e/0x70
arch_phys_wc_add+0x4d/0x80
drm_get_pci_dev+0xef/0x140 [drm]
radeon_pci_probe+0x99/0xc0 [radeon]
pci_device_probe+0xb7/0x130
? devices_kset_move_last+0x56/0x90
driver_probe_device+0x279/0x450
? _raw_spin_unlock+0x22/0x30
? pci_match_device+0xd2/0x100
__driver_attach+0xf9/0x100
? _raw_spin_unlock+0x22/0x30
? klist_next+0x60/0xb0
? driver_probe_device+0x450/0x450
bus_for_each_dev+0x4f/0x80
driver_attach+0x1e/0x20
? driver_probe_device+0x450/0x450
bus_add_driver+0x147/0x260
driver_register+0x59/0xe0
__pci_register_driver+0x4a/0x50
? 0xf829d000
drm_pci_init+0xe5/0xf0 [drm]
? 0xf829d000
radeon_init+0x93/0xb8 [radeon]
? 0xf829d000
do_one_initcall+0x46/0x190
? kmem_cache_alloc_trace+0xa9/0x2a0
? rcu_read_lock_sched_held+0x9f/0xb0
? kmem_cache_alloc_trace+0x215/0x2a0
? do_init_module+0x24/0x1e4
do_init_module+0x53/0x1e4
load_module+0x129f/0x19c0
SyS_finit_module+0x82/0xd0
? find_held_lock+0x2a/0xa0
do_fast_syscall_32+0xa0/0x1d0
entry_SYSENTER_32+0x53/0x86
EIP: 0xb7786da9
EFLAGS: 00000246 CPU: 0
EAX: ffffffda EBX: 00000013 ECX: b770dc35 EDX: 00000000
ESI: 09f897ac EDI: 09f8a503 EBP: b75e8e02 ESP: bf832c1c
DS: 007b ES: 007b FS: 0000 GS: 0033 SS: 007b
This problem was bisected to commit fc8dffd379ca ("cpu/hotplug: Convert hotplug
locking to percpu rwsem"). I am very confident in the correctness of the bisection.
Although the box has only a single CPU core, an SMP kernel is being generated.
The configuration file is attached. As usual, I will test any suggested fixes.
Larry
On Mon, 14 Aug 2017, Larry Finger wrote:
> My Del Latitude D600 (single 32-bit CPU) shows the following locking message
> when booted:
>
> ============================================
> WARNING: possible recursive locking detected
> 4.12.0-rc2-00029-gfc8dffd #159 Not tainted
> --------------------------------------------
> systemd-udevd/153 is trying to acquire lock:
> (cpu_hotplug_lock.rw_sem){.+.+.+}, at: [<c030fc26>] stop_machine+0x16/0x30
>
> but task is already holding lock:
> (cpu_hotplug_lock.rw_sem){.+.+.+}, at: [<c0234353>] mtrr_add_page+0x83/0x470
....
> cpus_read_lock+0x48/0x90
> ? stop_machine+0x16/0x30
> stop_machine+0x16/0x30
> mtrr_add_page+0x18b/0x470
> mtrr_add+0x3e/0x70
> This problem was bisected to commit fc8dffd379ca ("cpu/hotplug: Convert
> hotplug locking to percpu rwsem"). I am very confident in the correctness of
> the bisection.
Fix below.
Thanks,
tglx
8<--------------------
diff --git a/arch/x86/kernel/cpu/mtrr/main.c b/arch/x86/kernel/cpu/mtrr/main.c
index c5bb63be4ba1..40d5a8a75212 100644
--- a/arch/x86/kernel/cpu/mtrr/main.c
+++ b/arch/x86/kernel/cpu/mtrr/main.c
@@ -237,6 +237,18 @@ set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type typ
stop_machine(mtrr_rendezvous_handler, &data, cpu_online_mask);
}
+static void set_mtrr_cpuslocked(unsigned int reg, unsigned long base,
+ unsigned long size, mtrr_type type)
+{
+ struct set_mtrr_data data = { .smp_reg = reg,
+ .smp_base = base,
+ .smp_size = size,
+ .smp_type = type
+ };
+
+ stop_machine_cpuslocked(mtrr_rendezvous_handler, &data, cpu_online_mask);
+}
+
static void set_mtrr_from_inactive_cpu(unsigned int reg, unsigned long base,
unsigned long size, mtrr_type type)
{
@@ -370,7 +382,7 @@ int mtrr_add_page(unsigned long base, unsigned long size,
/* Search for an empty MTRR */
i = mtrr_if->get_free_region(base, size, replace);
if (i >= 0) {
- set_mtrr(i, base, size, type);
+ set_mtrr_cpuslocked(i, base, size, type);
if (likely(replace < 0)) {
mtrr_usage_table[i] = 1;
} else {
@@ -378,7 +390,7 @@ int mtrr_add_page(unsigned long base, unsigned long size,
if (increment)
mtrr_usage_table[i]++;
if (unlikely(replace != i)) {
- set_mtrr(replace, 0, 0, 0);
+ set_mtrr_cpuslocked(replace, 0, 0, 0);
mtrr_usage_table[replace] = 0;
}
}
@@ -506,7 +518,7 @@ int mtrr_del_page(int reg, unsigned long base, unsigned long size)
goto out;
}
if (--mtrr_usage_table[reg] < 1)
- set_mtrr(reg, 0, 0, 0);
+ set_mtrr_cpuslocked(reg, 0, 0, 0);
error = reg;
out:
mutex_unlock(&mtrr_mutex);
On 08/14/2017 02:21 AM, Thomas Gleixner wrote:
> On Mon, 14 Aug 2017, Larry Finger wrote:
>
>> My Del Latitude D600 (single 32-bit CPU) shows the following locking message
>> when booted:
>>
>> ============================================
>> WARNING: possible recursive locking detected
>> 4.12.0-rc2-00029-gfc8dffd #159 Not tainted
>> --------------------------------------------
>> systemd-udevd/153 is trying to acquire lock:
>> (cpu_hotplug_lock.rw_sem){.+.+.+}, at: [<c030fc26>] stop_machine+0x16/0x30
>>
>> but task is already holding lock:
>> (cpu_hotplug_lock.rw_sem){.+.+.+}, at: [<c0234353>] mtrr_add_page+0x83/0x470
>
> ....
>
>> cpus_read_lock+0x48/0x90
>> ? stop_machine+0x16/0x30
>> stop_machine+0x16/0x30
>> mtrr_add_page+0x18b/0x470
>> mtrr_add+0x3e/0x70
>
>> This problem was bisected to commit fc8dffd379ca ("cpu/hotplug: Convert
>> hotplug locking to percpu rwsem"). I am very confident in the correctness of
>> the bisection.
>
> Fix below.
>
> Thanks,
>
> tglx
>
> 8<--------------------
>
> diff --git a/arch/x86/kernel/cpu/mtrr/main.c b/arch/x86/kernel/cpu/mtrr/main.c
> index c5bb63be4ba1..40d5a8a75212 100644
> --- a/arch/x86/kernel/cpu/mtrr/main.c
> +++ b/arch/x86/kernel/cpu/mtrr/main.c
> @@ -237,6 +237,18 @@ set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type typ
> stop_machine(mtrr_rendezvous_handler, &data, cpu_online_mask);
> }
>
> +static void set_mtrr_cpuslocked(unsigned int reg, unsigned long base,
> + unsigned long size, mtrr_type type)
> +{
> + struct set_mtrr_data data = { .smp_reg = reg,
> + .smp_base = base,
> + .smp_size = size,
> + .smp_type = type
> + };
> +
> + stop_machine_cpuslocked(mtrr_rendezvous_handler, &data, cpu_online_mask);
> +}
> +
> static void set_mtrr_from_inactive_cpu(unsigned int reg, unsigned long base,
> unsigned long size, mtrr_type type)
> {
> @@ -370,7 +382,7 @@ int mtrr_add_page(unsigned long base, unsigned long size,
> /* Search for an empty MTRR */
> i = mtrr_if->get_free_region(base, size, replace);
> if (i >= 0) {
> - set_mtrr(i, base, size, type);
> + set_mtrr_cpuslocked(i, base, size, type);
> if (likely(replace < 0)) {
> mtrr_usage_table[i] = 1;
> } else {
> @@ -378,7 +390,7 @@ int mtrr_add_page(unsigned long base, unsigned long size,
> if (increment)
> mtrr_usage_table[i]++;
> if (unlikely(replace != i)) {
> - set_mtrr(replace, 0, 0, 0);
> + set_mtrr_cpuslocked(replace, 0, 0, 0);
> mtrr_usage_table[replace] = 0;
> }
> }
> @@ -506,7 +518,7 @@ int mtrr_del_page(int reg, unsigned long base, unsigned long size)
> goto out;
> }
> if (--mtrr_usage_table[reg] < 1)
> - set_mtrr(reg, 0, 0, 0);
> + set_mtrr_cpuslocked(reg, 0, 0, 0);
> error = reg;
> out:
> mutex_unlock(&mtrr_mutex);
>
That fix works for me.
Thanks,
Larry
Commit-ID: 84393817db09bb436e934f8f8cc981cbca9ea4dc
Gitweb: http://git.kernel.org/tip/84393817db09bb436e934f8f8cc981cbca9ea4dc
Author: Thomas Gleixner <[email protected]>
AuthorDate: Tue, 15 Aug 2017 13:03:47 +0200
Committer: Thomas Gleixner <[email protected]>
CommitDate: Tue, 15 Aug 2017 13:03:47 +0200
x86/mtrr: Prevent CPU hotplug lock recursion
Larry reported a CPU hotplug lock recursion in the MTRR code.
============================================
WARNING: possible recursive locking detected
systemd-udevd/153 is trying to acquire lock:
(cpu_hotplug_lock.rw_sem){.+.+.+}, at: [<c030fc26>] stop_machine+0x16/0x30
but task is already holding lock:
(cpu_hotplug_lock.rw_sem){.+.+.+}, at: [<c0234353>] mtrr_add_page+0x83/0x470
....
cpus_read_lock+0x48/0x90
stop_machine+0x16/0x30
mtrr_add_page+0x18b/0x470
mtrr_add+0x3e/0x70
mtrr_add_page() holds the hotplug rwsem already and calls stop_machine()
which acquires it again.
Call stop_machine_cpuslocked() instead.
Reported-and-tested-by: Larry Finger <[email protected]>
Reported-by: Dmitry Vyukov <[email protected]>
Signed-off-by: Thomas Gleixner <[email protected]>
Link: http://lkml.kernel.org/r/alpine.DEB.2.20.1708140920250.1865@nanos
Cc: "Paul E. McKenney" <[email protected]>
Cc: Borislav Petkov <[email protected]>
---
arch/x86/kernel/cpu/mtrr/main.c | 18 +++++++++++++++---
1 file changed, 15 insertions(+), 3 deletions(-)
diff --git a/arch/x86/kernel/cpu/mtrr/main.c b/arch/x86/kernel/cpu/mtrr/main.c
index c5bb63b..40d5a8a 100644
--- a/arch/x86/kernel/cpu/mtrr/main.c
+++ b/arch/x86/kernel/cpu/mtrr/main.c
@@ -237,6 +237,18 @@ set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type typ
stop_machine(mtrr_rendezvous_handler, &data, cpu_online_mask);
}
+static void set_mtrr_cpuslocked(unsigned int reg, unsigned long base,
+ unsigned long size, mtrr_type type)
+{
+ struct set_mtrr_data data = { .smp_reg = reg,
+ .smp_base = base,
+ .smp_size = size,
+ .smp_type = type
+ };
+
+ stop_machine_cpuslocked(mtrr_rendezvous_handler, &data, cpu_online_mask);
+}
+
static void set_mtrr_from_inactive_cpu(unsigned int reg, unsigned long base,
unsigned long size, mtrr_type type)
{
@@ -370,7 +382,7 @@ int mtrr_add_page(unsigned long base, unsigned long size,
/* Search for an empty MTRR */
i = mtrr_if->get_free_region(base, size, replace);
if (i >= 0) {
- set_mtrr(i, base, size, type);
+ set_mtrr_cpuslocked(i, base, size, type);
if (likely(replace < 0)) {
mtrr_usage_table[i] = 1;
} else {
@@ -378,7 +390,7 @@ int mtrr_add_page(unsigned long base, unsigned long size,
if (increment)
mtrr_usage_table[i]++;
if (unlikely(replace != i)) {
- set_mtrr(replace, 0, 0, 0);
+ set_mtrr_cpuslocked(replace, 0, 0, 0);
mtrr_usage_table[replace] = 0;
}
}
@@ -506,7 +518,7 @@ int mtrr_del_page(int reg, unsigned long base, unsigned long size)
goto out;
}
if (--mtrr_usage_table[reg] < 1)
- set_mtrr(reg, 0, 0, 0);
+ set_mtrr_cpuslocked(reg, 0, 0, 0);
error = reg;
out:
mutex_unlock(&mtrr_mutex);