2013-08-25 19:59:09

by Rafael J. Wysocki

[permalink] [raw]
Subject: [PATCH] driver core / ACPI: Avoid device removal locking problems

From: Rafael J. Wysocki <[email protected]>

There are two mutexes, device_hotplug_lock and acpi_scan_lock, held
around the acpi_bus_trim() call in acpi_scan_hot_remove() which
generally removes devices (it removes ACPI device objects at least,
but it may also remove "physical" device objects through .detach()
callbacks of ACPI scan handlers). Thus, potentially, device sysfs
attributes are removed under these locks and to remove those
attributes it is necessary to hold the s_active references of their
directory entries for writing.

On the other hand, the execution of a .show() or .store() callback
from a sysfs attribute is carried out with that attribute's s_active
reference held for reading. Consequently, if any device sysfs
attribute that may be removed from within acpi_scan_hot_remove()
through acpi_bus_trim() has a .store() or .show() callback which
acquires either acpi_scan_lock or device_hotplug_lock, the execution
of that callback may deadlock with the removal of the attribute.
[Unfortunately, the "online" device attribute of CPUs and memory
blocks and the "eject" attribute of ACPI device objects are affected
by this issue.]

To avoid those deadlocks introduce a new protection mechanism that
can be used by the device sysfs attributes in question. Namely,
if a device sysfs attribute's .store() or .show() callback routine
is about to acquire device_hotplug_lock or acpi_scan_lock, it can
first execute read_lock_device_remove() and return an error code if
that function returns false. If true is returned, the lock in
question may be acquired and read_unlock_device_remove() must be
called. [This mechanism is implemented by means of an additional
rwsem in drivers/base/core.c.]

Make the affected sysfs attributes in the driver core and ACPI core
use read_lock_device_remove() and read_unlock_device_remove() as
described above.

Signed-off-by: Rafael J. Wysocki <[email protected]>
Reported-by: Gu Zheng <[email protected]>
---

For 3.12, applies on top of linux-pm.git/linux-next.

Thanks,
Rafael

---
drivers/acpi/scan.c | 8 ++++++++
drivers/base/core.c | 29 +++++++++++++++++++++++++++++
include/linux/device.h | 4 ++++
3 files changed, 41 insertions(+)

Index: linux-pm/drivers/base/core.c
===================================================================
--- linux-pm.orig/drivers/base/core.c
+++ linux-pm/drivers/base/core.c
@@ -408,7 +408,11 @@ static ssize_t show_online(struct device
{
bool val;

+ if (!read_lock_device_remove())
+ return -EBUSY;
+
lock_device_hotplug();
+ read_unlock_device_remove();
val = !dev->offline;
unlock_device_hotplug();
return sprintf(buf, "%u\n", val);
@@ -424,7 +428,11 @@ static ssize_t store_online(struct devic
if (ret < 0)
return ret;

+ if (!read_lock_device_remove())
+ return -EBUSY;
+
lock_device_hotplug();
+ read_unlock_device_remove();
ret = val ? device_online(dev) : device_offline(dev);
unlock_device_hotplug();
return ret < 0 ? ret : count;
@@ -1479,8 +1487,29 @@ EXPORT_SYMBOL_GPL(put_device);
EXPORT_SYMBOL_GPL(device_create_file);
EXPORT_SYMBOL_GPL(device_remove_file);

+static DECLARE_RWSEM(device_remove_rwsem);
static DEFINE_MUTEX(device_hotplug_lock);

+bool __must_check read_lock_device_remove(void)
+{
+ return down_read_trylock(&device_remove_rwsem);
+}
+
+void read_unlock_device_remove(void)
+{
+ up_read(&device_remove_rwsem);
+}
+
+void device_remove_begin(void)
+{
+ down_write(&device_remove_rwsem);
+}
+
+void device_remove_end(void)
+{
+ up_write(&device_remove_rwsem);
+}
+
void lock_device_hotplug(void)
{
mutex_lock(&device_hotplug_lock);
Index: linux-pm/include/linux/device.h
===================================================================
--- linux-pm.orig/include/linux/device.h
+++ linux-pm/include/linux/device.h
@@ -893,6 +893,10 @@ static inline bool device_supports_offli
return dev->bus && dev->bus->offline && dev->bus->online;
}

+extern bool __must_check read_lock_device_remove(void);
+extern void read_unlock_device_remove(void);
+extern void device_remove_begin(void);
+extern void device_remove_end(void);
extern void lock_device_hotplug(void);
extern void unlock_device_hotplug(void);
extern int device_offline(struct device *dev);
Index: linux-pm/drivers/acpi/scan.c
===================================================================
--- linux-pm.orig/drivers/acpi/scan.c
+++ linux-pm/drivers/acpi/scan.c
@@ -288,6 +288,7 @@ static void acpi_bus_device_eject(void *
struct acpi_scan_handler *handler;
u32 ost_code = ACPI_OST_SC_NON_SPECIFIC_FAILURE;

+ device_remove_begin();
mutex_lock(&acpi_scan_lock);

acpi_bus_get_device(handle, &device);
@@ -315,6 +316,7 @@ static void acpi_bus_device_eject(void *

out:
mutex_unlock(&acpi_scan_lock);
+ device_remove_end();
return;

err_out:
@@ -449,6 +451,7 @@ void acpi_bus_hot_remove_device(void *co
acpi_handle handle = device->handle;
int error;

+ device_remove_begin();
mutex_lock(&acpi_scan_lock);

error = acpi_scan_hot_remove(device);
@@ -458,6 +461,7 @@ void acpi_bus_hot_remove_device(void *co
NULL);

mutex_unlock(&acpi_scan_lock);
+ device_remove_end();
kfree(context);
}
EXPORT_SYMBOL(acpi_bus_hot_remove_device);
@@ -510,7 +514,11 @@ acpi_eject_store(struct device *d, struc
if (ACPI_FAILURE(status) || !acpi_device->flags.ejectable)
return -ENODEV;

+ if (!read_lock_device_remove())
+ return -EBUSY;
+
mutex_lock(&acpi_scan_lock);
+ read_unlock_device_remove();

if (acpi_device->flags.eject_pending) {
/* ACPI eject notification event. */


2013-08-25 21:52:11

by Greg Kroah-Hartman

[permalink] [raw]
Subject: Re: [PATCH] driver core / ACPI: Avoid device removal locking problems

On Sun, Aug 25, 2013 at 10:09:47PM +0200, Rafael J. Wysocki wrote:
> From: Rafael J. Wysocki <[email protected]>
>
> There are two mutexes, device_hotplug_lock and acpi_scan_lock, held
> around the acpi_bus_trim() call in acpi_scan_hot_remove() which
> generally removes devices (it removes ACPI device objects at least,
> but it may also remove "physical" device objects through .detach()
> callbacks of ACPI scan handlers). Thus, potentially, device sysfs
> attributes are removed under these locks and to remove those
> attributes it is necessary to hold the s_active references of their
> directory entries for writing.
>
> On the other hand, the execution of a .show() or .store() callback
> from a sysfs attribute is carried out with that attribute's s_active
> reference held for reading. Consequently, if any device sysfs
> attribute that may be removed from within acpi_scan_hot_remove()
> through acpi_bus_trim() has a .store() or .show() callback which
> acquires either acpi_scan_lock or device_hotplug_lock, the execution
> of that callback may deadlock with the removal of the attribute.
> [Unfortunately, the "online" device attribute of CPUs and memory
> blocks and the "eject" attribute of ACPI device objects are affected
> by this issue.]
>
> To avoid those deadlocks introduce a new protection mechanism that
> can be used by the device sysfs attributes in question. Namely,
> if a device sysfs attribute's .store() or .show() callback routine
> is about to acquire device_hotplug_lock or acpi_scan_lock, it can
> first execute read_lock_device_remove() and return an error code if
> that function returns false. If true is returned, the lock in
> question may be acquired and read_unlock_device_remove() must be
> called. [This mechanism is implemented by means of an additional
> rwsem in drivers/base/core.c.]
>
> Make the affected sysfs attributes in the driver core and ACPI core
> use read_lock_device_remove() and read_unlock_device_remove() as
> described above.
>
> Signed-off-by: Rafael J. Wysocki <[email protected]>
> Reported-by: Gu Zheng <[email protected]>

Acked-by: Greg Kroah-Hartman <[email protected]>

2013-08-26 03:17:37

by Gu Zheng

[permalink] [raw]
Subject: Re: [PATCH] driver core / ACPI: Avoid device removal locking problems

Hi Rafael,

On 08/26/2013 04:09 AM, Rafael J. Wysocki wrote:

> From: Rafael J. Wysocki <[email protected]>
>
> There are two mutexes, device_hotplug_lock and acpi_scan_lock, held
> around the acpi_bus_trim() call in acpi_scan_hot_remove() which
> generally removes devices (it removes ACPI device objects at least,
> but it may also remove "physical" device objects through .detach()
> callbacks of ACPI scan handlers). Thus, potentially, device sysfs
> attributes are removed under these locks and to remove those
> attributes it is necessary to hold the s_active references of their
> directory entries for writing.
>
> On the other hand, the execution of a .show() or .store() callback
> from a sysfs attribute is carried out with that attribute's s_active
> reference held for reading. Consequently, if any device sysfs
> attribute that may be removed from within acpi_scan_hot_remove()
> through acpi_bus_trim() has a .store() or .show() callback which
> acquires either acpi_scan_lock or device_hotplug_lock, the execution
> of that callback may deadlock with the removal of the attribute.
> [Unfortunately, the "online" device attribute of CPUs and memory
> blocks and the "eject" attribute of ACPI device objects are affected
> by this issue.]
>
> To avoid those deadlocks introduce a new protection mechanism that
> can be used by the device sysfs attributes in question. Namely,
> if a device sysfs attribute's .store() or .show() callback routine
> is about to acquire device_hotplug_lock or acpi_scan_lock, it can
> first execute read_lock_device_remove() and return an error code if
> that function returns false. If true is returned, the lock in
> question may be acquired and read_unlock_device_remove() must be
> called. [This mechanism is implemented by means of an additional
> rwsem in drivers/base/core.c.]
>
> Make the affected sysfs attributes in the driver core and ACPI core
> use read_lock_device_remove() and read_unlock_device_remove() as
> described above.
>
> Signed-off-by: Rafael J. Wysocki <[email protected]>
> Reported-by: Gu Zheng <[email protected]>

I'm sorry to forget to mention that the original reporter is
Yasuaki Ishimatsu <[email protected]>. I continued
the investigation and found more issues.

We tested this patch on kernel 3.11-rc6, but it seems that the
issue is still there. Detail info as following.

Thanks,
Gu

======================================================
[ INFO: possible circular locking dependency detected ]
3.11.0-rc6-lockdebug-refea+ #162 Tainted: GF
-------------------------------------------------------
kworker/0:2/754 is trying to acquire lock:
(s_active#73){++++.+}, at: [<ffffffff8121062b>] sysfs_addrm_finish+0x3b/0x70

but task is already holding lock:
(mem_sysfs_mutex){+.+.+.}, at: [<ffffffff813b949d>] remove_memory_block+0x1d/0xa0

which lock already depends on the new lock.


the existing dependency chain (in reverse order) is:

-> #4 (mem_sysfs_mutex){+.+.+.}:
[<ffffffff810ba88c>] validate_chain+0x70c/0x870
[<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
[<ffffffff810bb080>] lock_acquire+0xa0/0x130
[<ffffffff8159781b>] mutex_lock_nested+0x7b/0x3b0
[<ffffffff813b9361>] add_memory_section+0x51/0x150
[<ffffffff813b947b>] register_new_memory+0x1b/0x20
[<ffffffff81590d21>] __add_pages+0x111/0x120
[<ffffffff81041224>] arch_add_memory+0x64/0xf0
[<ffffffff81590e07>] add_memory+0xd7/0x1e0
[<ffffffff81347db4>] acpi_memory_enable_device+0x77/0x12b
[<ffffffff8134801e>] acpi_memory_device_add+0x18f/0x198
[<ffffffff8131aa5a>] acpi_bus_device_attach+0x9a/0x100
[<ffffffff8133666d>] acpi_ns_walk_namespace+0xbe/0x17d
[<ffffffff81336b03>] acpi_walk_namespace+0x8a/0xc4
[<ffffffff8131c126>] acpi_bus_scan+0x91/0x9c
[<ffffffff8131c1af>] acpi_scan_bus_device_check+0x7e/0x10f
[<ffffffff8131c253>] acpi_scan_device_check+0x13/0x15
[<ffffffff81315dac>] acpi_os_execute_deferred+0x27/0x34
[<ffffffff8106bec8>] process_one_work+0x1e8/0x560
[<ffffffff8106d0a0>] worker_thread+0x120/0x3a0
[<ffffffff81073b5e>] kthread+0xee/0x100
[<ffffffff815a605c>] ret_from_fork+0x7c/0xb0

-> #3 (pm_mutex){+.+.+.}:
[<ffffffff810ba88c>] validate_chain+0x70c/0x870
[<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
[<ffffffff810bb080>] lock_acquire+0xa0/0x130
[<ffffffff8159781b>] mutex_lock_nested+0x7b/0x3b0
[<ffffffff81188795>] lock_memory_hotplug+0x35/0x40
[<ffffffff81590d5f>] add_memory+0x2f/0x1e0
[<ffffffff81347db4>] acpi_memory_enable_device+0x77/0x12b
[<ffffffff8134801e>] acpi_memory_device_add+0x18f/0x198
[<ffffffff8131aa5a>] acpi_bus_device_attach+0x9a/0x100
[<ffffffff8133666d>] acpi_ns_walk_namespace+0xbe/0x17d
[<ffffffff81336b03>] acpi_walk_namespace+0x8a/0xc4
[<ffffffff8131c126>] acpi_bus_scan+0x91/0x9c
[<ffffffff81d39862>] acpi_scan_init+0x66/0x15a
[<ffffffff81d397a0>] acpi_init+0xa1/0xbb
[<ffffffff810002c2>] do_one_initcall+0xf2/0x1a0
[<ffffffff81d018da>] do_basic_setup+0x9d/0xbb
[<ffffffff81d01b02>] kernel_init_freeable+0x20a/0x28a
[<ffffffff8158f20e>] kernel_init+0xe/0xf0
[<ffffffff815a605c>] ret_from_fork+0x7c/0xb0

-> #2 (mem_hotplug_mutex){+.+.+.}:
[<ffffffff810ba88c>] validate_chain+0x70c/0x870
[<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
[<ffffffff810bb080>] lock_acquire+0xa0/0x130
[<ffffffff8159781b>] mutex_lock_nested+0x7b/0x3b0
[<ffffffff81188777>] lock_memory_hotplug+0x17/0x40
[<ffffffff81590d5f>] add_memory+0x2f/0x1e0
[<ffffffff81347db4>] acpi_memory_enable_device+0x77/0x12b
[<ffffffff8134801e>] acpi_memory_device_add+0x18f/0x198
[<ffffffff8131aa5a>] acpi_bus_device_attach+0x9a/0x100
[<ffffffff8133666d>] acpi_ns_walk_namespace+0xbe/0x17d
[<ffffffff81336b03>] acpi_walk_namespace+0x8a/0xc4
[<ffffffff8131c126>] acpi_bus_scan+0x91/0x9c
[<ffffffff8131c1af>] acpi_scan_bus_device_check+0x7e/0x10f
[<ffffffff8131c253>] acpi_scan_device_check+0x13/0x15
[<ffffffff81315dac>] acpi_os_execute_deferred+0x27/0x34
[<ffffffff8106bec8>] process_one_work+0x1e8/0x560
[<ffffffff8106d0a0>] worker_thread+0x120/0x3a0
[<ffffffff81073b5e>] kthread+0xee/0x100
[<ffffffff815a605c>] ret_from_fork+0x7c/0xb0

-> #1 (device_hotplug_lock){+.+.+.}:
[<ffffffff810ba88c>] validate_chain+0x70c/0x870
[<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
[<ffffffff810bb080>] lock_acquire+0xa0/0x130
[<ffffffff8159781b>] mutex_lock_nested+0x7b/0x3b0
[<ffffffff813a16e7>] lock_device_hotplug+0x17/0x20
[<ffffffff813a2553>] show_online+0x33/0x80
[<ffffffff813a1ce7>] dev_attr_show+0x27/0x50
[<ffffffff8120ee94>] fill_read_buffer+0x74/0x100
[<ffffffff8120efcc>] sysfs_read_file+0xac/0xe0
[<ffffffff81195d21>] vfs_read+0xb1/0x130
[<ffffffff81196232>] SyS_read+0x62/0xb0
[<ffffffff815a6102>] system_call_fastpath+0x16/0x1b

-> #0 (s_active#73){++++.+}:
[<ffffffff810ba148>] check_prev_add+0x598/0x5d0
[<ffffffff810ba88c>] validate_chain+0x70c/0x870
[<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
[<ffffffff810bb080>] lock_acquire+0xa0/0x130
[<ffffffff8120fed7>] sysfs_deactivate+0x157/0x1c0
[<ffffffff8121062b>] sysfs_addrm_finish+0x3b/0x70
[<ffffffff8120e280>] sysfs_hash_and_remove+0x60/0xb0
[<ffffffff8120f2a9>] sysfs_remove_file+0x39/0x50
[<ffffffff813a2977>] device_remove_file+0x17/0x20
[<ffffffff813a29aa>] device_remove_attrs+0x2a/0xe0
[<ffffffff813a2b8b>] device_del+0x12b/0x1f0
[<ffffffff813a2c72>] device_unregister+0x22/0x60
[<ffffffff813b94ed>] remove_memory_block+0x6d/0xa0
[<ffffffff813b953f>] unregister_memory_section+0x1f/0x30
[<ffffffff81189468>] __remove_pages+0xc8/0x150
[<ffffffff8158f994>] arch_remove_memory+0x94/0xd0
[<ffffffff81590bef>] remove_memory+0x6f/0x90
[<ffffffff81347ccc>] acpi_memory_remove_memory+0x7e/0xa3
[<ffffffff81347d18>] acpi_memory_device_remove+0x27/0x33
[<ffffffff8131a8a2>] acpi_bus_device_detach+0x3d/0x5e
[<ffffffff81336685>] acpi_ns_walk_namespace+0xd6/0x17d
[<ffffffff81336b03>] acpi_walk_namespace+0x8a/0xc4
[<ffffffff8131a8f6>] acpi_bus_trim+0x33/0x7a
[<ffffffff8131b4c0>] acpi_scan_hot_remove+0x160/0x281
[<ffffffff8131b6fc>] acpi_bus_hot_remove_device+0x37/0x73
[<ffffffff81315dac>] acpi_os_execute_deferred+0x27/0x34
[<ffffffff8106bec8>] process_one_work+0x1e8/0x560
[<ffffffff8106d0a0>] worker_thread+0x120/0x3a0
[<ffffffff81073b5e>] kthread+0xee/0x100
[<ffffffff815a605c>] ret_from_fork+0x7c/0xb0

other info that might help us debug this:

Chain exists of:
s_active#73 --> pm_mutex --> mem_sysfs_mutex

Possible unsafe locking scenario:

CPU0 CPU1
---- ----
lock(mem_sysfs_mutex);
lock(pm_mutex);
lock(mem_sysfs_mutex);
lock(s_active#73);

*** DEADLOCK ***

8 locks held by kworker/0:2/754:
#0: (kacpi_hotplug){.+.+.+}, at: [<ffffffff8106be53>] process_one_work+0x173/0x560
#1: ((&dpc->work)#3){+.+.+.}, at: [<ffffffff8106be53>] process_one_work+0x173/0x560
#2: (device_remove_rwsem){+++++.}, at: [<ffffffff813a1c35>] device_remove_begin+0x15/0x20
#3: (acpi_scan_lock){+.+.+.}, at: [<ffffffff8131b6f4>] acpi_bus_hot_remove_device+0x2f/0x73
#4: (device_hotplug_lock){+.+.+.}, at: [<ffffffff813a16e7>] lock_device_hotplug+0x17/0x20
#5: (mem_hotplug_mutex){+.+.+.}, at: [<ffffffff81188777>] lock_memory_hotplug+0x17/0x40
#6: (pm_mutex){+.+.+.}, at: [<ffffffff81188795>] lock_memory_hotplug+0x35/0x40
#7: (mem_sysfs_mutex){+.+.+.}, at: [<ffffffff813b949d>] remove_memory_block+0x1d/0xa0

stack backtrace:
CPU: 0 PID: 754 Comm: kworker/0:2 Tainted: GF 3.11.0-rc6-lockdebug-refea+ #162
Hardware name: FUJITSU-SV PRIMEQUEST 1800E/SB, BIOS PRIMEQUEST 1000 Series BIOS Version 89.32 DP Proto 08/16/2012
Workqueue: kacpi_hotplug acpi_os_execute_deferred
ffff8807b953cb20 ffff8807b90b3558 ffffffff81596d87 0000000000000002
0000000000000000 ffff8807b90b35a8 ffffffff810b7ec9 ffff8807b90b35c8
ffffffff82104e60 ffff8807b90b35a8 ffff8807b953cae8 ffff8807b953cb20
Call Trace:
[<ffffffff81596d87>] dump_stack+0x59/0x82
[<ffffffff810b7ec9>] print_circular_bug+0x109/0x110
[<ffffffff810ba148>] check_prev_add+0x598/0x5d0
[<ffffffff810ba88c>] validate_chain+0x70c/0x870
[<ffffffff810b9151>] ? mark_lock+0x161/0x240
[<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
[<ffffffff8121062b>] ? sysfs_addrm_finish+0x3b/0x70
[<ffffffff810bb080>] lock_acquire+0xa0/0x130
[<ffffffff8121062b>] ? sysfs_addrm_finish+0x3b/0x70
[<ffffffff810b64e7>] ? lockdep_init_map+0x57/0x170
[<ffffffff8120fed7>] sysfs_deactivate+0x157/0x1c0
[<ffffffff8121062b>] ? sysfs_addrm_finish+0x3b/0x70
[<ffffffff8121062b>] sysfs_addrm_finish+0x3b/0x70
[<ffffffff8120e280>] sysfs_hash_and_remove+0x60/0xb0
[<ffffffff8120f2a9>] sysfs_remove_file+0x39/0x50
[<ffffffff813a2977>] device_remove_file+0x17/0x20
[<ffffffff813a29aa>] device_remove_attrs+0x2a/0xe0
[<ffffffff813a2b8b>] device_del+0x12b/0x1f0
[<ffffffff813a2c72>] device_unregister+0x22/0x60
[<ffffffff813b94ed>] remove_memory_block+0x6d/0xa0
[<ffffffff813b953f>] unregister_memory_section+0x1f/0x30
[<ffffffff81189468>] __remove_pages+0xc8/0x150
[<ffffffff8131a865>] ? power_state_show+0x36/0x36
[<ffffffff8158f994>] arch_remove_memory+0x94/0xd0
[<ffffffff81590bef>] remove_memory+0x6f/0x90
[<ffffffff81347ccc>] acpi_memory_remove_memory+0x7e/0xa3
[<ffffffff81347d18>] acpi_memory_device_remove+0x27/0x33
[<ffffffff8131a8a2>] acpi_bus_device_detach+0x3d/0x5e
[<ffffffff81336685>] acpi_ns_walk_namespace+0xd6/0x17d
[<ffffffff8131a865>] ? power_state_show+0x36/0x36
[<ffffffff81336b03>] acpi_walk_namespace+0x8a/0xc4
[<ffffffff8131a8f6>] acpi_bus_trim+0x33/0x7a
[<ffffffff8131b4c0>] acpi_scan_hot_remove+0x160/0x281
[<ffffffff8131b6f4>] ? acpi_bus_hot_remove_device+0x2f/0x73
[<ffffffff8131b6fc>] acpi_bus_hot_remove_device+0x37/0x73
[<ffffffff81315dac>] acpi_os_execute_deferred+0x27/0x34
[<ffffffff8106bec8>] process_one_work+0x1e8/0x560
[<ffffffff8106be53>] ? process_one_work+0x173/0x560
[<ffffffff8106d0a0>] worker_thread+0x120/0x3a0
[<ffffffff8106cf80>] ? manage_workers+0x170/0x170
[<ffffffff81073b5e>] kthread+0xee/0x100
[<ffffffff810bb59c>] ? __lock_release+0x9c/0x200
[<ffffffff81073a70>] ? __init_kthread_worker+0x70/0x70
[<ffffffff815a605c>] ret_from_fork+0x7c/0xb0
[<ffffffff81073a70>] ? __init_kthread_worker+0x70/0x70


> ---
>
> For 3.12, applies on top of linux-pm.git/linux-next.
>
> Thanks,
> Rafael
>
> ---
> drivers/acpi/scan.c | 8 ++++++++
> drivers/base/core.c | 29 +++++++++++++++++++++++++++++
> include/linux/device.h | 4 ++++
> 3 files changed, 41 insertions(+)
>
> Index: linux-pm/drivers/base/core.c
> ===================================================================
> --- linux-pm.orig/drivers/base/core.c
> +++ linux-pm/drivers/base/core.c
> @@ -408,7 +408,11 @@ static ssize_t show_online(struct device
> {
> bool val;
>
> + if (!read_lock_device_remove())
> + return -EBUSY;
> +
> lock_device_hotplug();
> + read_unlock_device_remove();
> val = !dev->offline;
> unlock_device_hotplug();
> return sprintf(buf, "%u\n", val);
> @@ -424,7 +428,11 @@ static ssize_t store_online(struct devic
> if (ret < 0)
> return ret;
>
> + if (!read_lock_device_remove())
> + return -EBUSY;
> +
> lock_device_hotplug();
> + read_unlock_device_remove();
> ret = val ? device_online(dev) : device_offline(dev);
> unlock_device_hotplug();
> return ret < 0 ? ret : count;
> @@ -1479,8 +1487,29 @@ EXPORT_SYMBOL_GPL(put_device);
> EXPORT_SYMBOL_GPL(device_create_file);
> EXPORT_SYMBOL_GPL(device_remove_file);
>
> +static DECLARE_RWSEM(device_remove_rwsem);
> static DEFINE_MUTEX(device_hotplug_lock);
>
> +bool __must_check read_lock_device_remove(void)
> +{
> + return down_read_trylock(&device_remove_rwsem);
> +}
> +
> +void read_unlock_device_remove(void)
> +{
> + up_read(&device_remove_rwsem);
> +}
> +
> +void device_remove_begin(void)
> +{
> + down_write(&device_remove_rwsem);
> +}
> +
> +void device_remove_end(void)
> +{
> + up_write(&device_remove_rwsem);
> +}
> +
> void lock_device_hotplug(void)
> {
> mutex_lock(&device_hotplug_lock);
> Index: linux-pm/include/linux/device.h
> ===================================================================
> --- linux-pm.orig/include/linux/device.h
> +++ linux-pm/include/linux/device.h
> @@ -893,6 +893,10 @@ static inline bool device_supports_offli
> return dev->bus && dev->bus->offline && dev->bus->online;
> }
>
> +extern bool __must_check read_lock_device_remove(void);
> +extern void read_unlock_device_remove(void);
> +extern void device_remove_begin(void);
> +extern void device_remove_end(void);
> extern void lock_device_hotplug(void);
> extern void unlock_device_hotplug(void);
> extern int device_offline(struct device *dev);
> Index: linux-pm/drivers/acpi/scan.c
> ===================================================================
> --- linux-pm.orig/drivers/acpi/scan.c
> +++ linux-pm/drivers/acpi/scan.c
> @@ -288,6 +288,7 @@ static void acpi_bus_device_eject(void *
> struct acpi_scan_handler *handler;
> u32 ost_code = ACPI_OST_SC_NON_SPECIFIC_FAILURE;
>
> + device_remove_begin();
> mutex_lock(&acpi_scan_lock);
>
> acpi_bus_get_device(handle, &device);
> @@ -315,6 +316,7 @@ static void acpi_bus_device_eject(void *
>
> out:
> mutex_unlock(&acpi_scan_lock);
> + device_remove_end();
> return;
>
> err_out:
> @@ -449,6 +451,7 @@ void acpi_bus_hot_remove_device(void *co
> acpi_handle handle = device->handle;
> int error;
>
> + device_remove_begin();
> mutex_lock(&acpi_scan_lock);
>
> error = acpi_scan_hot_remove(device);
> @@ -458,6 +461,7 @@ void acpi_bus_hot_remove_device(void *co
> NULL);
>
> mutex_unlock(&acpi_scan_lock);
> + device_remove_end();
> kfree(context);
> }
> EXPORT_SYMBOL(acpi_bus_hot_remove_device);
> @@ -510,7 +514,11 @@ acpi_eject_store(struct device *d, struc
> if (ACPI_FAILURE(status) || !acpi_device->flags.ejectable)
> return -ENODEV;
>
> + if (!read_lock_device_remove())
> + return -EBUSY;
> +
> mutex_lock(&acpi_scan_lock);
> + read_unlock_device_remove();
>
> if (acpi_device->flags.eject_pending) {
> /* ACPI eject notification event. */
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to [email protected]
> More majordomo info at http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at http://www.tux.org/lkml/
>

2013-08-26 12:31:32

by Rafael J. Wysocki

[permalink] [raw]
Subject: Re: [PATCH] driver core / ACPI: Avoid device removal locking problems

On Monday, August 26, 2013 11:13:13 AM Gu Zheng wrote:
> Hi Rafael,

Hi,

> On 08/26/2013 04:09 AM, Rafael J. Wysocki wrote:
>
> > From: Rafael J. Wysocki <[email protected]>
> >
> > There are two mutexes, device_hotplug_lock and acpi_scan_lock, held
> > around the acpi_bus_trim() call in acpi_scan_hot_remove() which
> > generally removes devices (it removes ACPI device objects at least,
> > but it may also remove "physical" device objects through .detach()
> > callbacks of ACPI scan handlers). Thus, potentially, device sysfs
> > attributes are removed under these locks and to remove those
> > attributes it is necessary to hold the s_active references of their
> > directory entries for writing.
> >
> > On the other hand, the execution of a .show() or .store() callback
> > from a sysfs attribute is carried out with that attribute's s_active
> > reference held for reading. Consequently, if any device sysfs
> > attribute that may be removed from within acpi_scan_hot_remove()
> > through acpi_bus_trim() has a .store() or .show() callback which
> > acquires either acpi_scan_lock or device_hotplug_lock, the execution
> > of that callback may deadlock with the removal of the attribute.
> > [Unfortunately, the "online" device attribute of CPUs and memory
> > blocks and the "eject" attribute of ACPI device objects are affected
> > by this issue.]
> >
> > To avoid those deadlocks introduce a new protection mechanism that
> > can be used by the device sysfs attributes in question. Namely,
> > if a device sysfs attribute's .store() or .show() callback routine
> > is about to acquire device_hotplug_lock or acpi_scan_lock, it can
> > first execute read_lock_device_remove() and return an error code if
> > that function returns false. If true is returned, the lock in
> > question may be acquired and read_unlock_device_remove() must be
> > called. [This mechanism is implemented by means of an additional
> > rwsem in drivers/base/core.c.]
> >
> > Make the affected sysfs attributes in the driver core and ACPI core
> > use read_lock_device_remove() and read_unlock_device_remove() as
> > described above.
> >
> > Signed-off-by: Rafael J. Wysocki <[email protected]>
> > Reported-by: Gu Zheng <[email protected]>
>
> I'm sorry to forget to mention that the original reporter is
> Yasuaki Ishimatsu <[email protected]>. I continued
> the investigation and found more issues.
>
> We tested this patch on kernel 3.11-rc6, but it seems that the
> issue is still there. Detail info as following.

Well, taking pm_mutex under acpi_scan_lock (trace #2) is a bad idea anyway,
because we'll need to take acpi_scan_lock during system suspend for PCI hot
remove to work and that's under pm_mutex. So I wonder if we can simply
drop the system sleep locking from lock/unlock_memory_hotplug(). But that's
a side note, because dropping it won't help here.

Now ->

> ======================================================
> [ INFO: possible circular locking dependency detected ]
> 3.11.0-rc6-lockdebug-refea+ #162 Tainted: GF
> -------------------------------------------------------
> kworker/0:2/754 is trying to acquire lock:
> (s_active#73){++++.+}, at: [<ffffffff8121062b>] sysfs_addrm_finish+0x3b/0x70
>
> but task is already holding lock:
> (mem_sysfs_mutex){+.+.+.}, at: [<ffffffff813b949d>] remove_memory_block+0x1d/0xa0
>
> which lock already depends on the new lock.
>
>
> the existing dependency chain (in reverse order) is:
>
> -> #4 (mem_sysfs_mutex){+.+.+.}:
> [<ffffffff810ba88c>] validate_chain+0x70c/0x870
> [<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
> [<ffffffff810bb080>] lock_acquire+0xa0/0x130
> [<ffffffff8159781b>] mutex_lock_nested+0x7b/0x3b0
> [<ffffffff813b9361>] add_memory_section+0x51/0x150
> [<ffffffff813b947b>] register_new_memory+0x1b/0x20
> [<ffffffff81590d21>] __add_pages+0x111/0x120
> [<ffffffff81041224>] arch_add_memory+0x64/0xf0
> [<ffffffff81590e07>] add_memory+0xd7/0x1e0
> [<ffffffff81347db4>] acpi_memory_enable_device+0x77/0x12b
> [<ffffffff8134801e>] acpi_memory_device_add+0x18f/0x198
> [<ffffffff8131aa5a>] acpi_bus_device_attach+0x9a/0x100
> [<ffffffff8133666d>] acpi_ns_walk_namespace+0xbe/0x17d
> [<ffffffff81336b03>] acpi_walk_namespace+0x8a/0xc4
> [<ffffffff8131c126>] acpi_bus_scan+0x91/0x9c
> [<ffffffff8131c1af>] acpi_scan_bus_device_check+0x7e/0x10f
> [<ffffffff8131c253>] acpi_scan_device_check+0x13/0x15
> [<ffffffff81315dac>] acpi_os_execute_deferred+0x27/0x34
> [<ffffffff8106bec8>] process_one_work+0x1e8/0x560
> [<ffffffff8106d0a0>] worker_thread+0x120/0x3a0
> [<ffffffff81073b5e>] kthread+0xee/0x100
> [<ffffffff815a605c>] ret_from_fork+0x7c/0xb0
>
> -> #3 (pm_mutex){+.+.+.}:
> [<ffffffff810ba88c>] validate_chain+0x70c/0x870
> [<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
> [<ffffffff810bb080>] lock_acquire+0xa0/0x130
> [<ffffffff8159781b>] mutex_lock_nested+0x7b/0x3b0
> [<ffffffff81188795>] lock_memory_hotplug+0x35/0x40
> [<ffffffff81590d5f>] add_memory+0x2f/0x1e0
> [<ffffffff81347db4>] acpi_memory_enable_device+0x77/0x12b
> [<ffffffff8134801e>] acpi_memory_device_add+0x18f/0x198
> [<ffffffff8131aa5a>] acpi_bus_device_attach+0x9a/0x100
> [<ffffffff8133666d>] acpi_ns_walk_namespace+0xbe/0x17d
> [<ffffffff81336b03>] acpi_walk_namespace+0x8a/0xc4
> [<ffffffff8131c126>] acpi_bus_scan+0x91/0x9c
> [<ffffffff81d39862>] acpi_scan_init+0x66/0x15a
> [<ffffffff81d397a0>] acpi_init+0xa1/0xbb
> [<ffffffff810002c2>] do_one_initcall+0xf2/0x1a0
> [<ffffffff81d018da>] do_basic_setup+0x9d/0xbb
> [<ffffffff81d01b02>] kernel_init_freeable+0x20a/0x28a
> [<ffffffff8158f20e>] kernel_init+0xe/0xf0
> [<ffffffff815a605c>] ret_from_fork+0x7c/0xb0
>
> -> #2 (mem_hotplug_mutex){+.+.+.}:
> [<ffffffff810ba88c>] validate_chain+0x70c/0x870
> [<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
> [<ffffffff810bb080>] lock_acquire+0xa0/0x130
> [<ffffffff8159781b>] mutex_lock_nested+0x7b/0x3b0
> [<ffffffff81188777>] lock_memory_hotplug+0x17/0x40
> [<ffffffff81590d5f>] add_memory+0x2f/0x1e0
> [<ffffffff81347db4>] acpi_memory_enable_device+0x77/0x12b
> [<ffffffff8134801e>] acpi_memory_device_add+0x18f/0x198
> [<ffffffff8131aa5a>] acpi_bus_device_attach+0x9a/0x100
> [<ffffffff8133666d>] acpi_ns_walk_namespace+0xbe/0x17d
> [<ffffffff81336b03>] acpi_walk_namespace+0x8a/0xc4
> [<ffffffff8131c126>] acpi_bus_scan+0x91/0x9c
> [<ffffffff8131c1af>] acpi_scan_bus_device_check+0x7e/0x10f
> [<ffffffff8131c253>] acpi_scan_device_check+0x13/0x15
> [<ffffffff81315dac>] acpi_os_execute_deferred+0x27/0x34
> [<ffffffff8106bec8>] process_one_work+0x1e8/0x560
> [<ffffffff8106d0a0>] worker_thread+0x120/0x3a0
> [<ffffffff81073b5e>] kthread+0xee/0x100
> [<ffffffff815a605c>] ret_from_fork+0x7c/0xb0
>
> -> #1 (device_hotplug_lock){+.+.+.}:
> [<ffffffff810ba88c>] validate_chain+0x70c/0x870
> [<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
> [<ffffffff810bb080>] lock_acquire+0xa0/0x130
> [<ffffffff8159781b>] mutex_lock_nested+0x7b/0x3b0
> [<ffffffff813a16e7>] lock_device_hotplug+0x17/0x20

-> we should have grabbed device_remove_rwsem for reading here with the patch
applied, which means that -->

> [<ffffffff813a2553>] show_online+0x33/0x80
> [<ffffffff813a1ce7>] dev_attr_show+0x27/0x50
> [<ffffffff8120ee94>] fill_read_buffer+0x74/0x100
> [<ffffffff8120efcc>] sysfs_read_file+0xac/0xe0
> [<ffffffff81195d21>] vfs_read+0xb1/0x130
> [<ffffffff81196232>] SyS_read+0x62/0xb0
> [<ffffffff815a6102>] system_call_fastpath+0x16/0x1b
>
> -> #0 (s_active#73){++++.+}:
> [<ffffffff810ba148>] check_prev_add+0x598/0x5d0
> [<ffffffff810ba88c>] validate_chain+0x70c/0x870
> [<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
> [<ffffffff810bb080>] lock_acquire+0xa0/0x130
> [<ffffffff8120fed7>] sysfs_deactivate+0x157/0x1c0
> [<ffffffff8121062b>] sysfs_addrm_finish+0x3b/0x70
> [<ffffffff8120e280>] sysfs_hash_and_remove+0x60/0xb0
> [<ffffffff8120f2a9>] sysfs_remove_file+0x39/0x50
> [<ffffffff813a2977>] device_remove_file+0x17/0x20
> [<ffffffff813a29aa>] device_remove_attrs+0x2a/0xe0
> [<ffffffff813a2b8b>] device_del+0x12b/0x1f0
> [<ffffffff813a2c72>] device_unregister+0x22/0x60
> [<ffffffff813b94ed>] remove_memory_block+0x6d/0xa0
> [<ffffffff813b953f>] unregister_memory_section+0x1f/0x30
> [<ffffffff81189468>] __remove_pages+0xc8/0x150
> [<ffffffff8158f994>] arch_remove_memory+0x94/0xd0
> [<ffffffff81590bef>] remove_memory+0x6f/0x90
> [<ffffffff81347ccc>] acpi_memory_remove_memory+0x7e/0xa3
> [<ffffffff81347d18>] acpi_memory_device_remove+0x27/0x33
> [<ffffffff8131a8a2>] acpi_bus_device_detach+0x3d/0x5e
> [<ffffffff81336685>] acpi_ns_walk_namespace+0xd6/0x17d
> [<ffffffff81336b03>] acpi_walk_namespace+0x8a/0xc4
> [<ffffffff8131a8f6>] acpi_bus_trim+0x33/0x7a
> [<ffffffff8131b4c0>] acpi_scan_hot_remove+0x160/0x281

--> device_hotplug_lock is already held by show_online() at this point (or if
it is not held, that function will return -EBUSY after attempting to grab
device_remove_rwsem for reading), so acpi_scan_hot_remove() will wait for it
to be released before calling acpi_bus_trim().

So the situation in which one thread holds s_active for reading and blocks on
device_hotplug_lock while another thread is holding it over device removal is
clearly impossible to me.

So I'm not sure how device_hotplug_lock is still involved?

> [<ffffffff8131b6fc>] acpi_bus_hot_remove_device+0x37/0x73
> [<ffffffff81315dac>] acpi_os_execute_deferred+0x27/0x34
> [<ffffffff8106bec8>] process_one_work+0x1e8/0x560
> [<ffffffff8106d0a0>] worker_thread+0x120/0x3a0
> [<ffffffff81073b5e>] kthread+0xee/0x100
> [<ffffffff815a605c>] ret_from_fork+0x7c/0xb0
>
> other info that might help us debug this:
>
> Chain exists of:
> s_active#73 --> pm_mutex --> mem_sysfs_mutex
>
> Possible unsafe locking scenario:
>
> CPU0 CPU1
> ---- ----
> lock(mem_sysfs_mutex);
> lock(pm_mutex);
> lock(mem_sysfs_mutex);
> lock(s_active#73);
>
> *** DEADLOCK ***

Thanks,
Rafael

2013-08-26 14:32:48

by Rafael J. Wysocki

[permalink] [raw]
Subject: Re: [PATCH] driver core / ACPI: Avoid device removal locking problems

On Monday, August 26, 2013 02:42:09 PM Rafael J. Wysocki wrote:
> On Monday, August 26, 2013 11:13:13 AM Gu Zheng wrote:
> > Hi Rafael,
>
> Hi,
>
> > On 08/26/2013 04:09 AM, Rafael J. Wysocki wrote:
> >
> > > From: Rafael J. Wysocki <[email protected]>
> > >
> > > There are two mutexes, device_hotplug_lock and acpi_scan_lock, held
> > > around the acpi_bus_trim() call in acpi_scan_hot_remove() which
> > > generally removes devices (it removes ACPI device objects at least,
> > > but it may also remove "physical" device objects through .detach()
> > > callbacks of ACPI scan handlers). Thus, potentially, device sysfs
> > > attributes are removed under these locks and to remove those
> > > attributes it is necessary to hold the s_active references of their
> > > directory entries for writing.
> > >
> > > On the other hand, the execution of a .show() or .store() callback
> > > from a sysfs attribute is carried out with that attribute's s_active
> > > reference held for reading. Consequently, if any device sysfs
> > > attribute that may be removed from within acpi_scan_hot_remove()
> > > through acpi_bus_trim() has a .store() or .show() callback which
> > > acquires either acpi_scan_lock or device_hotplug_lock, the execution
> > > of that callback may deadlock with the removal of the attribute.
> > > [Unfortunately, the "online" device attribute of CPUs and memory
> > > blocks and the "eject" attribute of ACPI device objects are affected
> > > by this issue.]
> > >
> > > To avoid those deadlocks introduce a new protection mechanism that
> > > can be used by the device sysfs attributes in question. Namely,
> > > if a device sysfs attribute's .store() or .show() callback routine
> > > is about to acquire device_hotplug_lock or acpi_scan_lock, it can
> > > first execute read_lock_device_remove() and return an error code if
> > > that function returns false. If true is returned, the lock in
> > > question may be acquired and read_unlock_device_remove() must be
> > > called. [This mechanism is implemented by means of an additional
> > > rwsem in drivers/base/core.c.]
> > >
> > > Make the affected sysfs attributes in the driver core and ACPI core
> > > use read_lock_device_remove() and read_unlock_device_remove() as
> > > described above.
> > >
> > > Signed-off-by: Rafael J. Wysocki <[email protected]>
> > > Reported-by: Gu Zheng <[email protected]>
> >
> > I'm sorry to forget to mention that the original reporter is
> > Yasuaki Ishimatsu <[email protected]>. I continued
> > the investigation and found more issues.
> >
> > We tested this patch on kernel 3.11-rc6, but it seems that the
> > issue is still there. Detail info as following.
>
> Well, taking pm_mutex under acpi_scan_lock (trace #2) is a bad idea anyway,
> because we'll need to take acpi_scan_lock during system suspend for PCI hot
> remove to work and that's under pm_mutex. So I wonder if we can simply
> drop the system sleep locking from lock/unlock_memory_hotplug(). But that's
> a side note, because dropping it won't help here.
>
> Now ->
>
> > ======================================================
> > [ INFO: possible circular locking dependency detected ]
> > 3.11.0-rc6-lockdebug-refea+ #162 Tainted: GF
> > -------------------------------------------------------
> > kworker/0:2/754 is trying to acquire lock:
> > (s_active#73){++++.+}, at: [<ffffffff8121062b>] sysfs_addrm_finish+0x3b/0x70
> >
> > but task is already holding lock:
> > (mem_sysfs_mutex){+.+.+.}, at: [<ffffffff813b949d>] remove_memory_block+0x1d/0xa0
> >
> > which lock already depends on the new lock.
> >
> >
> > the existing dependency chain (in reverse order) is:
> >
> > -> #4 (mem_sysfs_mutex){+.+.+.}:
> > [<ffffffff810ba88c>] validate_chain+0x70c/0x870
> > [<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
> > [<ffffffff810bb080>] lock_acquire+0xa0/0x130
> > [<ffffffff8159781b>] mutex_lock_nested+0x7b/0x3b0
> > [<ffffffff813b9361>] add_memory_section+0x51/0x150
> > [<ffffffff813b947b>] register_new_memory+0x1b/0x20
> > [<ffffffff81590d21>] __add_pages+0x111/0x120
> > [<ffffffff81041224>] arch_add_memory+0x64/0xf0
> > [<ffffffff81590e07>] add_memory+0xd7/0x1e0
> > [<ffffffff81347db4>] acpi_memory_enable_device+0x77/0x12b
> > [<ffffffff8134801e>] acpi_memory_device_add+0x18f/0x198
> > [<ffffffff8131aa5a>] acpi_bus_device_attach+0x9a/0x100
> > [<ffffffff8133666d>] acpi_ns_walk_namespace+0xbe/0x17d
> > [<ffffffff81336b03>] acpi_walk_namespace+0x8a/0xc4
> > [<ffffffff8131c126>] acpi_bus_scan+0x91/0x9c
> > [<ffffffff8131c1af>] acpi_scan_bus_device_check+0x7e/0x10f
> > [<ffffffff8131c253>] acpi_scan_device_check+0x13/0x15
> > [<ffffffff81315dac>] acpi_os_execute_deferred+0x27/0x34
> > [<ffffffff8106bec8>] process_one_work+0x1e8/0x560
> > [<ffffffff8106d0a0>] worker_thread+0x120/0x3a0
> > [<ffffffff81073b5e>] kthread+0xee/0x100
> > [<ffffffff815a605c>] ret_from_fork+0x7c/0xb0
> >
> > -> #3 (pm_mutex){+.+.+.}:
> > [<ffffffff810ba88c>] validate_chain+0x70c/0x870
> > [<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
> > [<ffffffff810bb080>] lock_acquire+0xa0/0x130
> > [<ffffffff8159781b>] mutex_lock_nested+0x7b/0x3b0
> > [<ffffffff81188795>] lock_memory_hotplug+0x35/0x40
> > [<ffffffff81590d5f>] add_memory+0x2f/0x1e0
> > [<ffffffff81347db4>] acpi_memory_enable_device+0x77/0x12b
> > [<ffffffff8134801e>] acpi_memory_device_add+0x18f/0x198
> > [<ffffffff8131aa5a>] acpi_bus_device_attach+0x9a/0x100
> > [<ffffffff8133666d>] acpi_ns_walk_namespace+0xbe/0x17d
> > [<ffffffff81336b03>] acpi_walk_namespace+0x8a/0xc4
> > [<ffffffff8131c126>] acpi_bus_scan+0x91/0x9c
> > [<ffffffff81d39862>] acpi_scan_init+0x66/0x15a
> > [<ffffffff81d397a0>] acpi_init+0xa1/0xbb
> > [<ffffffff810002c2>] do_one_initcall+0xf2/0x1a0
> > [<ffffffff81d018da>] do_basic_setup+0x9d/0xbb
> > [<ffffffff81d01b02>] kernel_init_freeable+0x20a/0x28a
> > [<ffffffff8158f20e>] kernel_init+0xe/0xf0
> > [<ffffffff815a605c>] ret_from_fork+0x7c/0xb0
> >
> > -> #2 (mem_hotplug_mutex){+.+.+.}:
> > [<ffffffff810ba88c>] validate_chain+0x70c/0x870
> > [<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
> > [<ffffffff810bb080>] lock_acquire+0xa0/0x130
> > [<ffffffff8159781b>] mutex_lock_nested+0x7b/0x3b0
> > [<ffffffff81188777>] lock_memory_hotplug+0x17/0x40
> > [<ffffffff81590d5f>] add_memory+0x2f/0x1e0
> > [<ffffffff81347db4>] acpi_memory_enable_device+0x77/0x12b
> > [<ffffffff8134801e>] acpi_memory_device_add+0x18f/0x198
> > [<ffffffff8131aa5a>] acpi_bus_device_attach+0x9a/0x100
> > [<ffffffff8133666d>] acpi_ns_walk_namespace+0xbe/0x17d
> > [<ffffffff81336b03>] acpi_walk_namespace+0x8a/0xc4
> > [<ffffffff8131c126>] acpi_bus_scan+0x91/0x9c
> > [<ffffffff8131c1af>] acpi_scan_bus_device_check+0x7e/0x10f
> > [<ffffffff8131c253>] acpi_scan_device_check+0x13/0x15
> > [<ffffffff81315dac>] acpi_os_execute_deferred+0x27/0x34
> > [<ffffffff8106bec8>] process_one_work+0x1e8/0x560
> > [<ffffffff8106d0a0>] worker_thread+0x120/0x3a0
> > [<ffffffff81073b5e>] kthread+0xee/0x100
> > [<ffffffff815a605c>] ret_from_fork+0x7c/0xb0
> >
> > -> #1 (device_hotplug_lock){+.+.+.}:
> > [<ffffffff810ba88c>] validate_chain+0x70c/0x870
> > [<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
> > [<ffffffff810bb080>] lock_acquire+0xa0/0x130
> > [<ffffffff8159781b>] mutex_lock_nested+0x7b/0x3b0
> > [<ffffffff813a16e7>] lock_device_hotplug+0x17/0x20
>
> -> we should have grabbed device_remove_rwsem for reading here with the patch
> applied, which means that -->
>
> > [<ffffffff813a2553>] show_online+0x33/0x80
> > [<ffffffff813a1ce7>] dev_attr_show+0x27/0x50
> > [<ffffffff8120ee94>] fill_read_buffer+0x74/0x100
> > [<ffffffff8120efcc>] sysfs_read_file+0xac/0xe0
> > [<ffffffff81195d21>] vfs_read+0xb1/0x130
> > [<ffffffff81196232>] SyS_read+0x62/0xb0
> > [<ffffffff815a6102>] system_call_fastpath+0x16/0x1b
> >
> > -> #0 (s_active#73){++++.+}:
> > [<ffffffff810ba148>] check_prev_add+0x598/0x5d0
> > [<ffffffff810ba88c>] validate_chain+0x70c/0x870
> > [<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
> > [<ffffffff810bb080>] lock_acquire+0xa0/0x130
> > [<ffffffff8120fed7>] sysfs_deactivate+0x157/0x1c0
> > [<ffffffff8121062b>] sysfs_addrm_finish+0x3b/0x70
> > [<ffffffff8120e280>] sysfs_hash_and_remove+0x60/0xb0
> > [<ffffffff8120f2a9>] sysfs_remove_file+0x39/0x50
> > [<ffffffff813a2977>] device_remove_file+0x17/0x20
> > [<ffffffff813a29aa>] device_remove_attrs+0x2a/0xe0
> > [<ffffffff813a2b8b>] device_del+0x12b/0x1f0
> > [<ffffffff813a2c72>] device_unregister+0x22/0x60
> > [<ffffffff813b94ed>] remove_memory_block+0x6d/0xa0
> > [<ffffffff813b953f>] unregister_memory_section+0x1f/0x30
> > [<ffffffff81189468>] __remove_pages+0xc8/0x150
> > [<ffffffff8158f994>] arch_remove_memory+0x94/0xd0
> > [<ffffffff81590bef>] remove_memory+0x6f/0x90
> > [<ffffffff81347ccc>] acpi_memory_remove_memory+0x7e/0xa3
> > [<ffffffff81347d18>] acpi_memory_device_remove+0x27/0x33
> > [<ffffffff8131a8a2>] acpi_bus_device_detach+0x3d/0x5e
> > [<ffffffff81336685>] acpi_ns_walk_namespace+0xd6/0x17d
> > [<ffffffff81336b03>] acpi_walk_namespace+0x8a/0xc4
> > [<ffffffff8131a8f6>] acpi_bus_trim+0x33/0x7a
> > [<ffffffff8131b4c0>] acpi_scan_hot_remove+0x160/0x281
>
> --> device_hotplug_lock is already held by show_online() at this point (or if
> it is not held, that function will return -EBUSY after attempting to grab
> device_remove_rwsem for reading), so acpi_scan_hot_remove() will wait for it
> to be released before calling acpi_bus_trim().
>
> So the situation in which one thread holds s_active for reading and blocks on
> device_hotplug_lock while another thread is holding it over device removal is
> clearly impossible to me.
>
> So I'm not sure how device_hotplug_lock is still involved?

Well, it is not involved, but lockdep doesn't see that anyway. Bummer.

> > [<ffffffff8131b6fc>] acpi_bus_hot_remove_device+0x37/0x73
> > [<ffffffff81315dac>] acpi_os_execute_deferred+0x27/0x34
> > [<ffffffff8106bec8>] process_one_work+0x1e8/0x560
> > [<ffffffff8106d0a0>] worker_thread+0x120/0x3a0
> > [<ffffffff81073b5e>] kthread+0xee/0x100
> > [<ffffffff815a605c>] ret_from_fork+0x7c/0xb0
> >
> > other info that might help us debug this:
> >
> > Chain exists of:
> > s_active#73 --> pm_mutex --> mem_sysfs_mutex
> >
> > Possible unsafe locking scenario:
> >
> > CPU0 CPU1
> > ---- ----
> > lock(mem_sysfs_mutex);
> > lock(pm_mutex);
> > lock(mem_sysfs_mutex);
> > lock(s_active#73);
> >
> > *** DEADLOCK ***

OK, so the patch below is quick and dirty and overkill, but it should make the
splat go away at least.

Thanks,
Rafael


---
drivers/acpi/scan.c | 3 ++-
drivers/base/core.c | 36 ++++++++++++++++++++----------------
2 files changed, 22 insertions(+), 17 deletions(-)

Index: linux-pm/drivers/base/core.c
===================================================================
--- linux-pm.orig/drivers/base/core.c
+++ linux-pm/drivers/base/core.c
@@ -49,6 +49,18 @@ static struct kobject *dev_kobj;
struct kobject *sysfs_dev_char_kobj;
struct kobject *sysfs_dev_block_kobj;

+static DEFINE_MUTEX(device_hotplug_lock);
+
+void lock_device_hotplug(void)
+{
+ mutex_lock(&device_hotplug_lock);
+}
+
+void unlock_device_hotplug(void)
+{
+ mutex_unlock(&device_hotplug_lock);
+}
+
#ifdef CONFIG_BLOCK
static inline int device_is_not_partition(struct device *dev)
{
@@ -408,9 +420,11 @@ static ssize_t show_online(struct device
{
bool val;

- lock_device_hotplug();
+ if (!mutex_trylock(&device_hotplug_lock))
+ return -EAGAIN;
+
val = !dev->offline;
- unlock_device_hotplug();
+ mutex_unlock(&device_hotplug_lock);
return sprintf(buf, "%u\n", val);
}

@@ -424,9 +438,11 @@ static ssize_t store_online(struct devic
if (ret < 0)
return ret;

- lock_device_hotplug();
+ if (!mutex_trylock(&device_hotplug_lock))
+ return -EAGAIN;
+
ret = val ? device_online(dev) : device_offline(dev);
- unlock_device_hotplug();
+ mutex_unlock(&device_hotplug_lock);
return ret < 0 ? ret : count;
}

@@ -1479,18 +1495,6 @@ EXPORT_SYMBOL_GPL(put_device);
EXPORT_SYMBOL_GPL(device_create_file);
EXPORT_SYMBOL_GPL(device_remove_file);

-static DEFINE_MUTEX(device_hotplug_lock);
-
-void lock_device_hotplug(void)
-{
- mutex_lock(&device_hotplug_lock);
-}
-
-void unlock_device_hotplug(void)
-{
- mutex_unlock(&device_hotplug_lock);
-}
-
static int device_check_offline(struct device *dev, void *not_used)
{
int ret;
Index: linux-pm/drivers/acpi/scan.c
===================================================================
--- linux-pm.orig/drivers/acpi/scan.c
+++ linux-pm/drivers/acpi/scan.c
@@ -510,7 +510,8 @@ acpi_eject_store(struct device *d, struc
if (ACPI_FAILURE(status) || !acpi_device->flags.ejectable)
return -ENODEV;

- mutex_lock(&acpi_scan_lock);
+ if (!mutex_trylock(&acpi_scan_lock))
+ return -EAGAIN;

if (acpi_device->flags.eject_pending) {
/* ACPI eject notification event. */

2013-08-26 14:51:29

by Rafael J. Wysocki

[permalink] [raw]
Subject: Re: [PATCH] driver core / ACPI: Avoid device removal locking problems

On Monday, August 26, 2013 04:43:26 PM Rafael J. Wysocki wrote:
> On Monday, August 26, 2013 02:42:09 PM Rafael J. Wysocki wrote:
> > On Monday, August 26, 2013 11:13:13 AM Gu Zheng wrote:
> > > Hi Rafael,

[...]

>
> OK, so the patch below is quick and dirty and overkill, but it should make the
> splat go away at least.

And if this patch does make the splat go away for you, please also test the
appended one (Tejun, thanks for the hint!).

I'll address the ACPI part differently later.

> ---
> drivers/acpi/scan.c | 3 ++-
> drivers/base/core.c | 36 ++++++++++++++++++++----------------
> 2 files changed, 22 insertions(+), 17 deletions(-)
>
> Index: linux-pm/drivers/base/core.c
> ===================================================================
> --- linux-pm.orig/drivers/base/core.c
> +++ linux-pm/drivers/base/core.c
> @@ -49,6 +49,18 @@ static struct kobject *dev_kobj;
> struct kobject *sysfs_dev_char_kobj;
> struct kobject *sysfs_dev_block_kobj;
>
> +static DEFINE_MUTEX(device_hotplug_lock);
> +
> +void lock_device_hotplug(void)
> +{
> + mutex_lock(&device_hotplug_lock);
> +}
> +
> +void unlock_device_hotplug(void)
> +{
> + mutex_unlock(&device_hotplug_lock);
> +}
> +
> #ifdef CONFIG_BLOCK
> static inline int device_is_not_partition(struct device *dev)
> {
> @@ -408,9 +420,11 @@ static ssize_t show_online(struct device
> {
> bool val;
>
> - lock_device_hotplug();
> + if (!mutex_trylock(&device_hotplug_lock))
> + return -EAGAIN;
> +
> val = !dev->offline;
> - unlock_device_hotplug();
> + mutex_unlock(&device_hotplug_lock);
> return sprintf(buf, "%u\n", val);
> }
>
> @@ -424,9 +438,11 @@ static ssize_t store_online(struct devic
> if (ret < 0)
> return ret;
>
> - lock_device_hotplug();
> + if (!mutex_trylock(&device_hotplug_lock))
> + return -EAGAIN;
> +
> ret = val ? device_online(dev) : device_offline(dev);
> - unlock_device_hotplug();
> + mutex_unlock(&device_hotplug_lock);
> return ret < 0 ? ret : count;
> }
>
> @@ -1479,18 +1495,6 @@ EXPORT_SYMBOL_GPL(put_device);
> EXPORT_SYMBOL_GPL(device_create_file);
> EXPORT_SYMBOL_GPL(device_remove_file);
>
> -static DEFINE_MUTEX(device_hotplug_lock);
> -
> -void lock_device_hotplug(void)
> -{
> - mutex_lock(&device_hotplug_lock);
> -}
> -
> -void unlock_device_hotplug(void)
> -{
> - mutex_unlock(&device_hotplug_lock);
> -}
> -
> static int device_check_offline(struct device *dev, void *not_used)
> {
> int ret;
> Index: linux-pm/drivers/acpi/scan.c
> ===================================================================
> --- linux-pm.orig/drivers/acpi/scan.c
> +++ linux-pm/drivers/acpi/scan.c
> @@ -510,7 +510,8 @@ acpi_eject_store(struct device *d, struc
> if (ACPI_FAILURE(status) || !acpi_device->flags.ejectable)
> return -ENODEV;
>
> - mutex_lock(&acpi_scan_lock);
> + if (!mutex_trylock(&acpi_scan_lock))
> + return -EAGAIN;
>
> if (acpi_device->flags.eject_pending) {
> /* ACPI eject notification event. */

Thanks,
Rafael


---
drivers/base/core.c | 45 +++++++++++++++++++++++++++++++--------------
1 file changed, 31 insertions(+), 14 deletions(-)

Index: linux-pm/drivers/base/core.c
===================================================================
--- linux-pm.orig/drivers/base/core.c
+++ linux-pm/drivers/base/core.c
@@ -49,6 +49,28 @@ static struct kobject *dev_kobj;
struct kobject *sysfs_dev_char_kobj;
struct kobject *sysfs_dev_block_kobj;

+static DEFINE_MUTEX(device_hotplug_lock);
+
+void lock_device_hotplug(void)
+{
+ mutex_lock(&device_hotplug_lock);
+}
+
+void unlock_device_hotplug(void)
+{
+ mutex_unlock(&device_hotplug_lock);
+}
+
+static int try_to_lock_device_hotplug(void)
+{
+ if (!mutex_trylock(&device_hotplug_lock)) {
+ /* Avoid busy waiting, 10 ms of sleep should do. */
+ msleep(10);
+ return restart_syscall();
+ }
+ return 0;
+}
+
#ifdef CONFIG_BLOCK
static inline int device_is_not_partition(struct device *dev)
{
@@ -407,8 +429,12 @@ static ssize_t show_online(struct device
char *buf)
{
bool val;
+ int ret;
+
+ ret = try_to_lock_device_hotplug();
+ if (ret)
+ return ret;

- lock_device_hotplug();
val = !dev->offline;
unlock_device_hotplug();
return sprintf(buf, "%u\n", val);
@@ -424,7 +450,10 @@ static ssize_t store_online(struct devic
if (ret < 0)
return ret;

- lock_device_hotplug();
+ ret = try_to_lock_device_hotplug();
+ if (ret)
+ return ret;
+
ret = val ? device_online(dev) : device_offline(dev);
unlock_device_hotplug();
return ret < 0 ? ret : count;
@@ -1479,18 +1508,6 @@ EXPORT_SYMBOL_GPL(put_device);
EXPORT_SYMBOL_GPL(device_create_file);
EXPORT_SYMBOL_GPL(device_remove_file);

-static DEFINE_MUTEX(device_hotplug_lock);
-
-void lock_device_hotplug(void)
-{
- mutex_lock(&device_hotplug_lock);
-}
-
-void unlock_device_hotplug(void)
-{
- mutex_unlock(&device_hotplug_lock);
-}
-
static int device_check_offline(struct device *dev, void *not_used)
{
int ret;

2013-08-27 02:08:14

by Gu Zheng

[permalink] [raw]
Subject: Re: [PATCH] driver core / ACPI: Avoid device removal locking problems

Hi Rafael,

On 08/26/2013 10:43 PM, Rafael J. Wysocki wrote:

> On Monday, August 26, 2013 02:42:09 PM Rafael J. Wysocki wrote:
>> On Monday, August 26, 2013 11:13:13 AM Gu Zheng wrote:
>>> Hi Rafael,
>>
>> Hi,
>>
>>> On 08/26/2013 04:09 AM, Rafael J. Wysocki wrote:
>>>
>>>> From: Rafael J. Wysocki <[email protected]>
>>>>
>>>> There are two mutexes, device_hotplug_lock and acpi_scan_lock, held
>>>> around the acpi_bus_trim() call in acpi_scan_hot_remove() which
>>>> generally removes devices (it removes ACPI device objects at least,
>>>> but it may also remove "physical" device objects through .detach()
>>>> callbacks of ACPI scan handlers). Thus, potentially, device sysfs
>>>> attributes are removed under these locks and to remove those
>>>> attributes it is necessary to hold the s_active references of their
>>>> directory entries for writing.
>>>>
>>>> On the other hand, the execution of a .show() or .store() callback
>>>> from a sysfs attribute is carried out with that attribute's s_active
>>>> reference held for reading. Consequently, if any device sysfs
>>>> attribute that may be removed from within acpi_scan_hot_remove()
>>>> through acpi_bus_trim() has a .store() or .show() callback which
>>>> acquires either acpi_scan_lock or device_hotplug_lock, the execution
>>>> of that callback may deadlock with the removal of the attribute.
>>>> [Unfortunately, the "online" device attribute of CPUs and memory
>>>> blocks and the "eject" attribute of ACPI device objects are affected
>>>> by this issue.]
>>>>
>>>> To avoid those deadlocks introduce a new protection mechanism that
>>>> can be used by the device sysfs attributes in question. Namely,
>>>> if a device sysfs attribute's .store() or .show() callback routine
>>>> is about to acquire device_hotplug_lock or acpi_scan_lock, it can
>>>> first execute read_lock_device_remove() and return an error code if
>>>> that function returns false. If true is returned, the lock in
>>>> question may be acquired and read_unlock_device_remove() must be
>>>> called. [This mechanism is implemented by means of an additional
>>>> rwsem in drivers/base/core.c.]
>>>>
>>>> Make the affected sysfs attributes in the driver core and ACPI core
>>>> use read_lock_device_remove() and read_unlock_device_remove() as
>>>> described above.
>>>>
>>>> Signed-off-by: Rafael J. Wysocki <[email protected]>
>>>> Reported-by: Gu Zheng <[email protected]>
>>>
>>> I'm sorry to forget to mention that the original reporter is
>>> Yasuaki Ishimatsu <[email protected]>. I continued
>>> the investigation and found more issues.
>>>
>>> We tested this patch on kernel 3.11-rc6, but it seems that the
>>> issue is still there. Detail info as following.
>>
>> Well, taking pm_mutex under acpi_scan_lock (trace #2) is a bad idea anyway,
>> because we'll need to take acpi_scan_lock during system suspend for PCI hot
>> remove to work and that's under pm_mutex. So I wonder if we can simply
>> drop the system sleep locking from lock/unlock_memory_hotplug(). But that's
>> a side note, because dropping it won't help here.
>>
>> Now ->
>>
>>> ======================================================
>>> [ INFO: possible circular locking dependency detected ]
>>> 3.11.0-rc6-lockdebug-refea+ #162 Tainted: GF
>>> -------------------------------------------------------
>>> kworker/0:2/754 is trying to acquire lock:
>>> (s_active#73){++++.+}, at: [<ffffffff8121062b>] sysfs_addrm_finish+0x3b/0x70
>>>
>>> but task is already holding lock:
>>> (mem_sysfs_mutex){+.+.+.}, at: [<ffffffff813b949d>] remove_memory_block+0x1d/0xa0
>>>
>>> which lock already depends on the new lock.
>>>
>>>
>>> the existing dependency chain (in reverse order) is:
>>>
>>> -> #4 (mem_sysfs_mutex){+.+.+.}:
>>> [<ffffffff810ba88c>] validate_chain+0x70c/0x870
>>> [<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
>>> [<ffffffff810bb080>] lock_acquire+0xa0/0x130
>>> [<ffffffff8159781b>] mutex_lock_nested+0x7b/0x3b0
>>> [<ffffffff813b9361>] add_memory_section+0x51/0x150
>>> [<ffffffff813b947b>] register_new_memory+0x1b/0x20
>>> [<ffffffff81590d21>] __add_pages+0x111/0x120
>>> [<ffffffff81041224>] arch_add_memory+0x64/0xf0
>>> [<ffffffff81590e07>] add_memory+0xd7/0x1e0
>>> [<ffffffff81347db4>] acpi_memory_enable_device+0x77/0x12b
>>> [<ffffffff8134801e>] acpi_memory_device_add+0x18f/0x198
>>> [<ffffffff8131aa5a>] acpi_bus_device_attach+0x9a/0x100
>>> [<ffffffff8133666d>] acpi_ns_walk_namespace+0xbe/0x17d
>>> [<ffffffff81336b03>] acpi_walk_namespace+0x8a/0xc4
>>> [<ffffffff8131c126>] acpi_bus_scan+0x91/0x9c
>>> [<ffffffff8131c1af>] acpi_scan_bus_device_check+0x7e/0x10f
>>> [<ffffffff8131c253>] acpi_scan_device_check+0x13/0x15
>>> [<ffffffff81315dac>] acpi_os_execute_deferred+0x27/0x34
>>> [<ffffffff8106bec8>] process_one_work+0x1e8/0x560
>>> [<ffffffff8106d0a0>] worker_thread+0x120/0x3a0
>>> [<ffffffff81073b5e>] kthread+0xee/0x100
>>> [<ffffffff815a605c>] ret_from_fork+0x7c/0xb0
>>>
>>> -> #3 (pm_mutex){+.+.+.}:
>>> [<ffffffff810ba88c>] validate_chain+0x70c/0x870
>>> [<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
>>> [<ffffffff810bb080>] lock_acquire+0xa0/0x130
>>> [<ffffffff8159781b>] mutex_lock_nested+0x7b/0x3b0
>>> [<ffffffff81188795>] lock_memory_hotplug+0x35/0x40
>>> [<ffffffff81590d5f>] add_memory+0x2f/0x1e0
>>> [<ffffffff81347db4>] acpi_memory_enable_device+0x77/0x12b
>>> [<ffffffff8134801e>] acpi_memory_device_add+0x18f/0x198
>>> [<ffffffff8131aa5a>] acpi_bus_device_attach+0x9a/0x100
>>> [<ffffffff8133666d>] acpi_ns_walk_namespace+0xbe/0x17d
>>> [<ffffffff81336b03>] acpi_walk_namespace+0x8a/0xc4
>>> [<ffffffff8131c126>] acpi_bus_scan+0x91/0x9c
>>> [<ffffffff81d39862>] acpi_scan_init+0x66/0x15a
>>> [<ffffffff81d397a0>] acpi_init+0xa1/0xbb
>>> [<ffffffff810002c2>] do_one_initcall+0xf2/0x1a0
>>> [<ffffffff81d018da>] do_basic_setup+0x9d/0xbb
>>> [<ffffffff81d01b02>] kernel_init_freeable+0x20a/0x28a
>>> [<ffffffff8158f20e>] kernel_init+0xe/0xf0
>>> [<ffffffff815a605c>] ret_from_fork+0x7c/0xb0
>>>
>>> -> #2 (mem_hotplug_mutex){+.+.+.}:
>>> [<ffffffff810ba88c>] validate_chain+0x70c/0x870
>>> [<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
>>> [<ffffffff810bb080>] lock_acquire+0xa0/0x130
>>> [<ffffffff8159781b>] mutex_lock_nested+0x7b/0x3b0
>>> [<ffffffff81188777>] lock_memory_hotplug+0x17/0x40
>>> [<ffffffff81590d5f>] add_memory+0x2f/0x1e0
>>> [<ffffffff81347db4>] acpi_memory_enable_device+0x77/0x12b
>>> [<ffffffff8134801e>] acpi_memory_device_add+0x18f/0x198
>>> [<ffffffff8131aa5a>] acpi_bus_device_attach+0x9a/0x100
>>> [<ffffffff8133666d>] acpi_ns_walk_namespace+0xbe/0x17d
>>> [<ffffffff81336b03>] acpi_walk_namespace+0x8a/0xc4
>>> [<ffffffff8131c126>] acpi_bus_scan+0x91/0x9c
>>> [<ffffffff8131c1af>] acpi_scan_bus_device_check+0x7e/0x10f
>>> [<ffffffff8131c253>] acpi_scan_device_check+0x13/0x15
>>> [<ffffffff81315dac>] acpi_os_execute_deferred+0x27/0x34
>>> [<ffffffff8106bec8>] process_one_work+0x1e8/0x560
>>> [<ffffffff8106d0a0>] worker_thread+0x120/0x3a0
>>> [<ffffffff81073b5e>] kthread+0xee/0x100
>>> [<ffffffff815a605c>] ret_from_fork+0x7c/0xb0
>>>
>>> -> #1 (device_hotplug_lock){+.+.+.}:
>>> [<ffffffff810ba88c>] validate_chain+0x70c/0x870
>>> [<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
>>> [<ffffffff810bb080>] lock_acquire+0xa0/0x130
>>> [<ffffffff8159781b>] mutex_lock_nested+0x7b/0x3b0
>>> [<ffffffff813a16e7>] lock_device_hotplug+0x17/0x20
>>
>> -> we should have grabbed device_remove_rwsem for reading here with the patch
>> applied, which means that -->
>>
>>> [<ffffffff813a2553>] show_online+0x33/0x80
>>> [<ffffffff813a1ce7>] dev_attr_show+0x27/0x50
>>> [<ffffffff8120ee94>] fill_read_buffer+0x74/0x100
>>> [<ffffffff8120efcc>] sysfs_read_file+0xac/0xe0
>>> [<ffffffff81195d21>] vfs_read+0xb1/0x130
>>> [<ffffffff81196232>] SyS_read+0x62/0xb0
>>> [<ffffffff815a6102>] system_call_fastpath+0x16/0x1b
>>>
>>> -> #0 (s_active#73){++++.+}:
>>> [<ffffffff810ba148>] check_prev_add+0x598/0x5d0
>>> [<ffffffff810ba88c>] validate_chain+0x70c/0x870
>>> [<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
>>> [<ffffffff810bb080>] lock_acquire+0xa0/0x130
>>> [<ffffffff8120fed7>] sysfs_deactivate+0x157/0x1c0
>>> [<ffffffff8121062b>] sysfs_addrm_finish+0x3b/0x70
>>> [<ffffffff8120e280>] sysfs_hash_and_remove+0x60/0xb0
>>> [<ffffffff8120f2a9>] sysfs_remove_file+0x39/0x50
>>> [<ffffffff813a2977>] device_remove_file+0x17/0x20
>>> [<ffffffff813a29aa>] device_remove_attrs+0x2a/0xe0
>>> [<ffffffff813a2b8b>] device_del+0x12b/0x1f0
>>> [<ffffffff813a2c72>] device_unregister+0x22/0x60
>>> [<ffffffff813b94ed>] remove_memory_block+0x6d/0xa0
>>> [<ffffffff813b953f>] unregister_memory_section+0x1f/0x30
>>> [<ffffffff81189468>] __remove_pages+0xc8/0x150
>>> [<ffffffff8158f994>] arch_remove_memory+0x94/0xd0
>>> [<ffffffff81590bef>] remove_memory+0x6f/0x90
>>> [<ffffffff81347ccc>] acpi_memory_remove_memory+0x7e/0xa3
>>> [<ffffffff81347d18>] acpi_memory_device_remove+0x27/0x33
>>> [<ffffffff8131a8a2>] acpi_bus_device_detach+0x3d/0x5e
>>> [<ffffffff81336685>] acpi_ns_walk_namespace+0xd6/0x17d
>>> [<ffffffff81336b03>] acpi_walk_namespace+0x8a/0xc4
>>> [<ffffffff8131a8f6>] acpi_bus_trim+0x33/0x7a
>>> [<ffffffff8131b4c0>] acpi_scan_hot_remove+0x160/0x281
>>
>> --> device_hotplug_lock is already held by show_online() at this point (or if
>> it is not held, that function will return -EBUSY after attempting to grab
>> device_remove_rwsem for reading), so acpi_scan_hot_remove() will wait for it
>> to be released before calling acpi_bus_trim().
>>
>> So the situation in which one thread holds s_active for reading and blocks on
>> device_hotplug_lock while another thread is holding it over device removal is
>> clearly impossible to me.
>>
>> So I'm not sure how device_hotplug_lock is still involved?
>
> Well, it is not involved, but lockdep doesn't see that anyway. Bummer.
>
>>> [<ffffffff8131b6fc>] acpi_bus_hot_remove_device+0x37/0x73
>>> [<ffffffff81315dac>] acpi_os_execute_deferred+0x27/0x34
>>> [<ffffffff8106bec8>] process_one_work+0x1e8/0x560
>>> [<ffffffff8106d0a0>] worker_thread+0x120/0x3a0
>>> [<ffffffff81073b5e>] kthread+0xee/0x100
>>> [<ffffffff815a605c>] ret_from_fork+0x7c/0xb0
>>>
>>> other info that might help us debug this:
>>>
>>> Chain exists of:
>>> s_active#73 --> pm_mutex --> mem_sysfs_mutex
>>>
>>> Possible unsafe locking scenario:
>>>
>>> CPU0 CPU1
>>> ---- ----
>>> lock(mem_sysfs_mutex);
>>> lock(pm_mutex);
>>> lock(mem_sysfs_mutex);
>>> lock(s_active#73);
>>>
>>> *** DEADLOCK ***
>
> OK, so the patch below is quick and dirty and overkill, but it should make the
> splat go away at least.

Yeah, this patch is the same as the one I attached in previous discussion threads.
It does make the splat go away, but as you said, it's a bit overkill.

Regards,
Gu

>
> Thanks,
> Rafael
>
>
> ---
> drivers/acpi/scan.c | 3 ++-
> drivers/base/core.c | 36 ++++++++++++++++++++----------------
> 2 files changed, 22 insertions(+), 17 deletions(-)
>
> Index: linux-pm/drivers/base/core.c
> ===================================================================
> --- linux-pm.orig/drivers/base/core.c
> +++ linux-pm/drivers/base/core.c
> @@ -49,6 +49,18 @@ static struct kobject *dev_kobj;
> struct kobject *sysfs_dev_char_kobj;
> struct kobject *sysfs_dev_block_kobj;
>
> +static DEFINE_MUTEX(device_hotplug_lock);
> +
> +void lock_device_hotplug(void)
> +{
> + mutex_lock(&device_hotplug_lock);
> +}
> +
> +void unlock_device_hotplug(void)
> +{
> + mutex_unlock(&device_hotplug_lock);
> +}
> +
> #ifdef CONFIG_BLOCK
> static inline int device_is_not_partition(struct device *dev)
> {
> @@ -408,9 +420,11 @@ static ssize_t show_online(struct device
> {
> bool val;
>
> - lock_device_hotplug();
> + if (!mutex_trylock(&device_hotplug_lock))
> + return -EAGAIN;
> +
> val = !dev->offline;
> - unlock_device_hotplug();
> + mutex_unlock(&device_hotplug_lock);
> return sprintf(buf, "%u\n", val);
> }
>
> @@ -424,9 +438,11 @@ static ssize_t store_online(struct devic
> if (ret < 0)
> return ret;
>
> - lock_device_hotplug();
> + if (!mutex_trylock(&device_hotplug_lock))
> + return -EAGAIN;
> +
> ret = val ? device_online(dev) : device_offline(dev);
> - unlock_device_hotplug();
> + mutex_unlock(&device_hotplug_lock);
> return ret < 0 ? ret : count;
> }
>
> @@ -1479,18 +1495,6 @@ EXPORT_SYMBOL_GPL(put_device);
> EXPORT_SYMBOL_GPL(device_create_file);
> EXPORT_SYMBOL_GPL(device_remove_file);
>
> -static DEFINE_MUTEX(device_hotplug_lock);
> -
> -void lock_device_hotplug(void)
> -{
> - mutex_lock(&device_hotplug_lock);
> -}
> -
> -void unlock_device_hotplug(void)
> -{
> - mutex_unlock(&device_hotplug_lock);
> -}
> -
> static int device_check_offline(struct device *dev, void *not_used)
> {
> int ret;
> Index: linux-pm/drivers/acpi/scan.c
> ===================================================================
> --- linux-pm.orig/drivers/acpi/scan.c
> +++ linux-pm/drivers/acpi/scan.c
> @@ -510,7 +510,8 @@ acpi_eject_store(struct device *d, struc
> if (ACPI_FAILURE(status) || !acpi_device->flags.ejectable)
> return -ENODEV;
>
> - mutex_lock(&acpi_scan_lock);
> + if (!mutex_trylock(&acpi_scan_lock))
> + return -EAGAIN;
>
> if (acpi_device->flags.eject_pending) {
> /* ACPI eject notification event. */
>
>

2013-08-27 02:44:24

by Gu Zheng

[permalink] [raw]
Subject: Re: [PATCH] driver core / ACPI: Avoid device removal locking problems

Hi Rafael,

On 08/26/2013 10:43 PM, Rafael J. Wysocki wrote:

> On Monday, August 26, 2013 02:42:09 PM Rafael J. Wysocki wrote:
>> On Monday, August 26, 2013 11:13:13 AM Gu Zheng wrote:
>>> Hi Rafael,
>>
>> Hi,
>>
>>> On 08/26/2013 04:09 AM, Rafael J. Wysocki wrote:
>>>
>>>> From: Rafael J. Wysocki <[email protected]>
>>>>
>>>> There are two mutexes, device_hotplug_lock and acpi_scan_lock, held
>>>> around the acpi_bus_trim() call in acpi_scan_hot_remove() which
>>>> generally removes devices (it removes ACPI device objects at least,
>>>> but it may also remove "physical" device objects through .detach()
>>>> callbacks of ACPI scan handlers). Thus, potentially, device sysfs
>>>> attributes are removed under these locks and to remove those
>>>> attributes it is necessary to hold the s_active references of their
>>>> directory entries for writing.
>>>>
>>>> On the other hand, the execution of a .show() or .store() callback
>>>> from a sysfs attribute is carried out with that attribute's s_active
>>>> reference held for reading. Consequently, if any device sysfs
>>>> attribute that may be removed from within acpi_scan_hot_remove()
>>>> through acpi_bus_trim() has a .store() or .show() callback which
>>>> acquires either acpi_scan_lock or device_hotplug_lock, the execution
>>>> of that callback may deadlock with the removal of the attribute.
>>>> [Unfortunately, the "online" device attribute of CPUs and memory
>>>> blocks and the "eject" attribute of ACPI device objects are affected
>>>> by this issue.]
>>>>
>>>> To avoid those deadlocks introduce a new protection mechanism that
>>>> can be used by the device sysfs attributes in question. Namely,
>>>> if a device sysfs attribute's .store() or .show() callback routine
>>>> is about to acquire device_hotplug_lock or acpi_scan_lock, it can
>>>> first execute read_lock_device_remove() and return an error code if
>>>> that function returns false. If true is returned, the lock in
>>>> question may be acquired and read_unlock_device_remove() must be
>>>> called. [This mechanism is implemented by means of an additional
>>>> rwsem in drivers/base/core.c.]
>>>>
>>>> Make the affected sysfs attributes in the driver core and ACPI core
>>>> use read_lock_device_remove() and read_unlock_device_remove() as
>>>> described above.
>>>>
>>>> Signed-off-by: Rafael J. Wysocki <[email protected]>
>>>> Reported-by: Gu Zheng <[email protected]>
>>>
>>> I'm sorry to forget to mention that the original reporter is
>>> Yasuaki Ishimatsu <[email protected]>. I continued
>>> the investigation and found more issues.
>>>
>>> We tested this patch on kernel 3.11-rc6, but it seems that the
>>> issue is still there. Detail info as following.
>>
>> Well, taking pm_mutex under acpi_scan_lock (trace #2) is a bad idea anyway,
>> because we'll need to take acpi_scan_lock during system suspend for PCI hot
>> remove to work and that's under pm_mutex. So I wonder if we can simply
>> drop the system sleep locking from lock/unlock_memory_hotplug(). But that's
>> a side note, because dropping it won't help here.
>>
>> Now ->
>>
>>> ======================================================
>>> [ INFO: possible circular locking dependency detected ]
>>> 3.11.0-rc6-lockdebug-refea+ #162 Tainted: GF
>>> -------------------------------------------------------
>>> kworker/0:2/754 is trying to acquire lock:
>>> (s_active#73){++++.+}, at: [<ffffffff8121062b>] sysfs_addrm_finish+0x3b/0x70
>>>
>>> but task is already holding lock:
>>> (mem_sysfs_mutex){+.+.+.}, at: [<ffffffff813b949d>] remove_memory_block+0x1d/0xa0
>>>
>>> which lock already depends on the new lock.
>>>
>>>
>>> the existing dependency chain (in reverse order) is:
>>>
>>> -> #4 (mem_sysfs_mutex){+.+.+.}:
>>> [<ffffffff810ba88c>] validate_chain+0x70c/0x870
>>> [<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
>>> [<ffffffff810bb080>] lock_acquire+0xa0/0x130
>>> [<ffffffff8159781b>] mutex_lock_nested+0x7b/0x3b0
>>> [<ffffffff813b9361>] add_memory_section+0x51/0x150
>>> [<ffffffff813b947b>] register_new_memory+0x1b/0x20
>>> [<ffffffff81590d21>] __add_pages+0x111/0x120
>>> [<ffffffff81041224>] arch_add_memory+0x64/0xf0
>>> [<ffffffff81590e07>] add_memory+0xd7/0x1e0
>>> [<ffffffff81347db4>] acpi_memory_enable_device+0x77/0x12b
>>> [<ffffffff8134801e>] acpi_memory_device_add+0x18f/0x198
>>> [<ffffffff8131aa5a>] acpi_bus_device_attach+0x9a/0x100
>>> [<ffffffff8133666d>] acpi_ns_walk_namespace+0xbe/0x17d
>>> [<ffffffff81336b03>] acpi_walk_namespace+0x8a/0xc4
>>> [<ffffffff8131c126>] acpi_bus_scan+0x91/0x9c
>>> [<ffffffff8131c1af>] acpi_scan_bus_device_check+0x7e/0x10f
>>> [<ffffffff8131c253>] acpi_scan_device_check+0x13/0x15
>>> [<ffffffff81315dac>] acpi_os_execute_deferred+0x27/0x34
>>> [<ffffffff8106bec8>] process_one_work+0x1e8/0x560
>>> [<ffffffff8106d0a0>] worker_thread+0x120/0x3a0
>>> [<ffffffff81073b5e>] kthread+0xee/0x100
>>> [<ffffffff815a605c>] ret_from_fork+0x7c/0xb0
>>>
>>> -> #3 (pm_mutex){+.+.+.}:
>>> [<ffffffff810ba88c>] validate_chain+0x70c/0x870
>>> [<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
>>> [<ffffffff810bb080>] lock_acquire+0xa0/0x130
>>> [<ffffffff8159781b>] mutex_lock_nested+0x7b/0x3b0
>>> [<ffffffff81188795>] lock_memory_hotplug+0x35/0x40
>>> [<ffffffff81590d5f>] add_memory+0x2f/0x1e0
>>> [<ffffffff81347db4>] acpi_memory_enable_device+0x77/0x12b
>>> [<ffffffff8134801e>] acpi_memory_device_add+0x18f/0x198
>>> [<ffffffff8131aa5a>] acpi_bus_device_attach+0x9a/0x100
>>> [<ffffffff8133666d>] acpi_ns_walk_namespace+0xbe/0x17d
>>> [<ffffffff81336b03>] acpi_walk_namespace+0x8a/0xc4
>>> [<ffffffff8131c126>] acpi_bus_scan+0x91/0x9c
>>> [<ffffffff81d39862>] acpi_scan_init+0x66/0x15a
>>> [<ffffffff81d397a0>] acpi_init+0xa1/0xbb
>>> [<ffffffff810002c2>] do_one_initcall+0xf2/0x1a0
>>> [<ffffffff81d018da>] do_basic_setup+0x9d/0xbb
>>> [<ffffffff81d01b02>] kernel_init_freeable+0x20a/0x28a
>>> [<ffffffff8158f20e>] kernel_init+0xe/0xf0
>>> [<ffffffff815a605c>] ret_from_fork+0x7c/0xb0
>>>
>>> -> #2 (mem_hotplug_mutex){+.+.+.}:
>>> [<ffffffff810ba88c>] validate_chain+0x70c/0x870
>>> [<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
>>> [<ffffffff810bb080>] lock_acquire+0xa0/0x130
>>> [<ffffffff8159781b>] mutex_lock_nested+0x7b/0x3b0
>>> [<ffffffff81188777>] lock_memory_hotplug+0x17/0x40
>>> [<ffffffff81590d5f>] add_memory+0x2f/0x1e0
>>> [<ffffffff81347db4>] acpi_memory_enable_device+0x77/0x12b
>>> [<ffffffff8134801e>] acpi_memory_device_add+0x18f/0x198
>>> [<ffffffff8131aa5a>] acpi_bus_device_attach+0x9a/0x100
>>> [<ffffffff8133666d>] acpi_ns_walk_namespace+0xbe/0x17d
>>> [<ffffffff81336b03>] acpi_walk_namespace+0x8a/0xc4
>>> [<ffffffff8131c126>] acpi_bus_scan+0x91/0x9c
>>> [<ffffffff8131c1af>] acpi_scan_bus_device_check+0x7e/0x10f
>>> [<ffffffff8131c253>] acpi_scan_device_check+0x13/0x15
>>> [<ffffffff81315dac>] acpi_os_execute_deferred+0x27/0x34
>>> [<ffffffff8106bec8>] process_one_work+0x1e8/0x560
>>> [<ffffffff8106d0a0>] worker_thread+0x120/0x3a0
>>> [<ffffffff81073b5e>] kthread+0xee/0x100
>>> [<ffffffff815a605c>] ret_from_fork+0x7c/0xb0
>>>
>>> -> #1 (device_hotplug_lock){+.+.+.}:
>>> [<ffffffff810ba88c>] validate_chain+0x70c/0x870
>>> [<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
>>> [<ffffffff810bb080>] lock_acquire+0xa0/0x130
>>> [<ffffffff8159781b>] mutex_lock_nested+0x7b/0x3b0
>>> [<ffffffff813a16e7>] lock_device_hotplug+0x17/0x20
>>
>> -> we should have grabbed device_remove_rwsem for reading here with the patch
>> applied, which means that -->
>>
>>> [<ffffffff813a2553>] show_online+0x33/0x80
>>> [<ffffffff813a1ce7>] dev_attr_show+0x27/0x50
>>> [<ffffffff8120ee94>] fill_read_buffer+0x74/0x100
>>> [<ffffffff8120efcc>] sysfs_read_file+0xac/0xe0
>>> [<ffffffff81195d21>] vfs_read+0xb1/0x130
>>> [<ffffffff81196232>] SyS_read+0x62/0xb0
>>> [<ffffffff815a6102>] system_call_fastpath+0x16/0x1b
>>>
>>> -> #0 (s_active#73){++++.+}:
>>> [<ffffffff810ba148>] check_prev_add+0x598/0x5d0
>>> [<ffffffff810ba88c>] validate_chain+0x70c/0x870
>>> [<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
>>> [<ffffffff810bb080>] lock_acquire+0xa0/0x130
>>> [<ffffffff8120fed7>] sysfs_deactivate+0x157/0x1c0
>>> [<ffffffff8121062b>] sysfs_addrm_finish+0x3b/0x70
>>> [<ffffffff8120e280>] sysfs_hash_and_remove+0x60/0xb0
>>> [<ffffffff8120f2a9>] sysfs_remove_file+0x39/0x50
>>> [<ffffffff813a2977>] device_remove_file+0x17/0x20
>>> [<ffffffff813a29aa>] device_remove_attrs+0x2a/0xe0
>>> [<ffffffff813a2b8b>] device_del+0x12b/0x1f0
>>> [<ffffffff813a2c72>] device_unregister+0x22/0x60
>>> [<ffffffff813b94ed>] remove_memory_block+0x6d/0xa0
>>> [<ffffffff813b953f>] unregister_memory_section+0x1f/0x30
>>> [<ffffffff81189468>] __remove_pages+0xc8/0x150
>>> [<ffffffff8158f994>] arch_remove_memory+0x94/0xd0
>>> [<ffffffff81590bef>] remove_memory+0x6f/0x90
>>> [<ffffffff81347ccc>] acpi_memory_remove_memory+0x7e/0xa3
>>> [<ffffffff81347d18>] acpi_memory_device_remove+0x27/0x33
>>> [<ffffffff8131a8a2>] acpi_bus_device_detach+0x3d/0x5e
>>> [<ffffffff81336685>] acpi_ns_walk_namespace+0xd6/0x17d
>>> [<ffffffff81336b03>] acpi_walk_namespace+0x8a/0xc4
>>> [<ffffffff8131a8f6>] acpi_bus_trim+0x33/0x7a
>>> [<ffffffff8131b4c0>] acpi_scan_hot_remove+0x160/0x281
>>
>> --> device_hotplug_lock is already held by show_online() at this point (or if
>> it is not held, that function will return -EBUSY after attempting to grab
>> device_remove_rwsem for reading), so acpi_scan_hot_remove() will wait for it
>> to be released before calling acpi_bus_trim().
>>
>> So the situation in which one thread holds s_active for reading and blocks on
>> device_hotplug_lock while another thread is holding it over device removal is
>> clearly impossible to me.
>>
>> So I'm not sure how device_hotplug_lock is still involved?
>
> Well, it is not involved, but lockdep doesn't see that anyway. Bummer.

Yeah, I also guess this is a wrong detection, but I'm not familiar with this filed.
If it is really so, the lockdep guys maybe need to fix it, rather than altering an overkill
solution to accommodate it. What's your opinion?
In fact, I think "[PATCH] driver core / ACPI: Avoid device removal locking problems" is
a good solution.:)

Regards,
Gu

>
>>> [<ffffffff8131b6fc>] acpi_bus_hot_remove_device+0x37/0x73
>>> [<ffffffff81315dac>] acpi_os_execute_deferred+0x27/0x34
>>> [<ffffffff8106bec8>] process_one_work+0x1e8/0x560
>>> [<ffffffff8106d0a0>] worker_thread+0x120/0x3a0
>>> [<ffffffff81073b5e>] kthread+0xee/0x100
>>> [<ffffffff815a605c>] ret_from_fork+0x7c/0xb0
>>>
>>> other info that might help us debug this:
>>>
>>> Chain exists of:
>>> s_active#73 --> pm_mutex --> mem_sysfs_mutex
>>>
>>> Possible unsafe locking scenario:
>>>
>>> CPU0 CPU1
>>> ---- ----
>>> lock(mem_sysfs_mutex);
>>> lock(pm_mutex);
>>> lock(mem_sysfs_mutex);
>>> lock(s_active#73);
>>>
>>> *** DEADLOCK ***
>
> OK, so the patch below is quick and dirty and overkill, but it should make the
> splat go away at least.
>
> Thanks,
> Rafael
>
>
> ---
> drivers/acpi/scan.c | 3 ++-
> drivers/base/core.c | 36 ++++++++++++++++++++----------------
> 2 files changed, 22 insertions(+), 17 deletions(-)
>
> Index: linux-pm/drivers/base/core.c
> ===================================================================
> --- linux-pm.orig/drivers/base/core.c
> +++ linux-pm/drivers/base/core.c
> @@ -49,6 +49,18 @@ static struct kobject *dev_kobj;
> struct kobject *sysfs_dev_char_kobj;
> struct kobject *sysfs_dev_block_kobj;
>
> +static DEFINE_MUTEX(device_hotplug_lock);
> +
> +void lock_device_hotplug(void)
> +{
> + mutex_lock(&device_hotplug_lock);
> +}
> +
> +void unlock_device_hotplug(void)
> +{
> + mutex_unlock(&device_hotplug_lock);
> +}
> +
> #ifdef CONFIG_BLOCK
> static inline int device_is_not_partition(struct device *dev)
> {
> @@ -408,9 +420,11 @@ static ssize_t show_online(struct device
> {
> bool val;
>
> - lock_device_hotplug();
> + if (!mutex_trylock(&device_hotplug_lock))
> + return -EAGAIN;
> +
> val = !dev->offline;
> - unlock_device_hotplug();
> + mutex_unlock(&device_hotplug_lock);
> return sprintf(buf, "%u\n", val);
> }
>
> @@ -424,9 +438,11 @@ static ssize_t store_online(struct devic
> if (ret < 0)
> return ret;
>
> - lock_device_hotplug();
> + if (!mutex_trylock(&device_hotplug_lock))
> + return -EAGAIN;
> +
> ret = val ? device_online(dev) : device_offline(dev);
> - unlock_device_hotplug();
> + mutex_unlock(&device_hotplug_lock);
> return ret < 0 ? ret : count;
> }
>
> @@ -1479,18 +1495,6 @@ EXPORT_SYMBOL_GPL(put_device);
> EXPORT_SYMBOL_GPL(device_create_file);
> EXPORT_SYMBOL_GPL(device_remove_file);
>
> -static DEFINE_MUTEX(device_hotplug_lock);
> -
> -void lock_device_hotplug(void)
> -{
> - mutex_lock(&device_hotplug_lock);
> -}
> -
> -void unlock_device_hotplug(void)
> -{
> - mutex_unlock(&device_hotplug_lock);
> -}
> -
> static int device_check_offline(struct device *dev, void *not_used)
> {
> int ret;
> Index: linux-pm/drivers/acpi/scan.c
> ===================================================================
> --- linux-pm.orig/drivers/acpi/scan.c
> +++ linux-pm/drivers/acpi/scan.c
> @@ -510,7 +510,8 @@ acpi_eject_store(struct device *d, struc
> if (ACPI_FAILURE(status) || !acpi_device->flags.ejectable)
> return -ENODEV;
>
> - mutex_lock(&acpi_scan_lock);
> + if (!mutex_trylock(&acpi_scan_lock))
> + return -EAGAIN;
>
> if (acpi_device->flags.eject_pending) {
> /* ACPI eject notification event. */
>
>

2013-08-27 03:30:56

by Gu Zheng

[permalink] [raw]
Subject: Re: [PATCH] driver core / ACPI: Avoid device removal locking problems

Hi Rafael,

On 08/26/2013 11:02 PM, Rafael J. Wysocki wrote:

> On Monday, August 26, 2013 04:43:26 PM Rafael J. Wysocki wrote:
>> On Monday, August 26, 2013 02:42:09 PM Rafael J. Wysocki wrote:
>>> On Monday, August 26, 2013 11:13:13 AM Gu Zheng wrote:
>>>> Hi Rafael,
>
> [...]
>
>>
>> OK, so the patch below is quick and dirty and overkill, but it should make the
>> splat go away at least.
>
> And if this patch does make the splat go away for you, please also test the
> appended one (Tejun, thanks for the hint!).

Yes, this one works too, and as expected, the ACPI part is still there.

Thanks,
Gu

======================================================
[ INFO: possible circular locking dependency detected ]
3.11.0-rc6-fix-refeal-fix-01+ #171 Tainted: GF
-------------------------------------------------------
kworker/0:1/96 is trying to acquire lock:
(s_active#245){++++.+}, at: [<ffffffff8121062b>] sysfs_addrm_finish+0x3b/0x70

but task is already holding lock:
(device_hotplug_lock){+.+.+.}, at: [<ffffffff813a16b7>] lock_device_hotplug+0x17/0x20

which lock already depends on the new lock.


the existing dependency chain (in reverse order) is:

-> #2 (device_hotplug_lock){+.+.+.}:
[<ffffffff810ba88c>] validate_chain+0x70c/0x870
[<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
[<ffffffff810bb080>] lock_acquire+0xa0/0x130
[<ffffffff8159779b>] mutex_lock_nested+0x7b/0x3b0
[<ffffffff813a16b7>] lock_device_hotplug+0x17/0x20
[<ffffffff8131c131>] acpi_scan_bus_device_check+0x33/0x10f
[<ffffffff8131c220>] acpi_scan_device_check+0x13/0x15
[<ffffffff81315dac>] acpi_os_execute_deferred+0x27/0x34
[<ffffffff8106bec8>] process_one_work+0x1e8/0x560
[<ffffffff8106d0a0>] worker_thread+0x120/0x3a0
[<ffffffff81073b5e>] kthread+0xee/0x100
[<ffffffff815a5fdc>] ret_from_fork+0x7c/0xb0

-> #1 (acpi_scan_lock){+.+.+.}:
[<ffffffff810ba88c>] validate_chain+0x70c/0x870
[<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
[<ffffffff810bb080>] lock_acquire+0xa0/0x130
[<ffffffff8159779b>] mutex_lock_nested+0x7b/0x3b0
[<ffffffff8131a58a>] acpi_eject_store+0x88/0x170
[<ffffffff813a0f40>] dev_attr_store+0x20/0x30
[<ffffffff8120ed96>] sysfs_write_file+0xe6/0x170
[<ffffffff81195bc8>] vfs_write+0xc8/0x170
[<ffffffff81196182>] SyS_write+0x62/0xb0
[<ffffffff815a6082>] system_call_fastpath+0x16/0x1b

-> #0 (s_active#245){++++.+}:
[<ffffffff810ba148>] check_prev_add+0x598/0x5d0
[<ffffffff810ba88c>] validate_chain+0x70c/0x870
[<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
[<ffffffff810bb080>] lock_acquire+0xa0/0x130
[<ffffffff8120fed7>] sysfs_deactivate+0x157/0x1c0
[<ffffffff8121062b>] sysfs_addrm_finish+0x3b/0x70
[<ffffffff8120e280>] sysfs_hash_and_remove+0x60/0xb0
[<ffffffff8120f2a9>] sysfs_remove_file+0x39/0x50
[<ffffffff813a28f7>] device_remove_file+0x17/0x20
[<ffffffff8131a271>] acpi_device_remove_files+0xa9/0x14b
[<ffffffff8131a377>] acpi_device_unregister+0x64/0xa6
[<ffffffff8131a3e4>] acpi_bus_remove+0x2b/0x2f
[<ffffffff8131a91b>] acpi_bus_trim+0x73/0x7a
[<ffffffff8131b4a5>] acpi_scan_hot_remove+0x160/0x281
[<ffffffff8131b6ce>] acpi_bus_hot_remove_device+0x32/0x69
[<ffffffff81315dac>] acpi_os_execute_deferred+0x27/0x34
[<ffffffff8106bec8>] process_one_work+0x1e8/0x560
[<ffffffff8106d0a0>] worker_thread+0x120/0x3a0
[<ffffffff81073b5e>] kthread+0xee/0x100
[<ffffffff815a5fdc>] ret_from_fork+0x7c/0xb0

other info that might help us debug this:

Chain exists of:
s_active#245 --> acpi_scan_lock --> device_hotplug_lock

Possible unsafe locking scenario:

CPU0 CPU1
---- ----
lock(device_hotplug_lock);
lock(acpi_scan_lock);
lock(device_hotplug_lock);
lock(s_active#245);

*** DEADLOCK ***

4 locks held by kworker/0:1/96:
#0: (kacpi_hotplug){.+.+.+}, at: [<ffffffff8106be53>] process_one_work+0x173/0x560
#1: ((&dpc->work)#3){+.+.+.}, at: [<ffffffff8106be53>] process_one_work+0x173/0x560
#2: (acpi_scan_lock){+.+.+.}, at: [<ffffffff8131b6c6>] acpi_bus_hot_remove_device+0x2a/0x69
#3: (device_hotplug_lock){+.+.+.}, at: [<ffffffff813a16b7>] lock_device_hotplug+0x17/0x20

stack backtrace:
CPU: 0 PID: 96 Comm: kworker/0:1 Tainted: GF 3.11.0-rc6-fix-refeal-fix-01+ #171
Hardware name: FUJITSU-SV PRIMEQUEST 1800E/SB, BIOS PRIMEQUEST 1000 Series BIOS Version 89.32 DP Proto 08/16/2012
Workqueue: kacpi_hotplug acpi_os_execute_deferred
ffff8807bf6a2c00 ffff8807bf6a97a8 ffffffff81596d07 0000000000000002
0000000000000000 ffff8807bf6a97f8 ffffffff810b7ec9 ffff8807bf6a9818
ffffffff82171ce0 ffff8807bf6a97f8 ffff8807bf6a2bc8 ffff8807bf6a2c00
Call Trace:
[<ffffffff81596d07>] dump_stack+0x59/0x82
[<ffffffff810b7ec9>] print_circular_bug+0x109/0x110
[<ffffffff810ba148>] check_prev_add+0x598/0x5d0
[<ffffffff810b9a35>] ? debug_check_no_locks_freed+0x95/0xd0
[<ffffffff810ba88c>] validate_chain+0x70c/0x870
[<ffffffff810b9151>] ? mark_lock+0x161/0x240
[<ffffffff810bad5f>] __lock_acquire+0x36f/0x5f0
[<ffffffff8121062b>] ? sysfs_addrm_finish+0x3b/0x70
[<ffffffff810bb080>] lock_acquire+0xa0/0x130
[<ffffffff8121062b>] ? sysfs_addrm_finish+0x3b/0x70
[<ffffffff810b64e7>] ? lockdep_init_map+0x57/0x170
[<ffffffff8120fed7>] sysfs_deactivate+0x157/0x1c0
[<ffffffff8121062b>] ? sysfs_addrm_finish+0x3b/0x70
[<ffffffff8121062b>] sysfs_addrm_finish+0x3b/0x70
[<ffffffff8120e280>] sysfs_hash_and_remove+0x60/0xb0
[<ffffffff8120f2a9>] sysfs_remove_file+0x39/0x50
[<ffffffff813a28f7>] device_remove_file+0x17/0x20
[<ffffffff8131a271>] acpi_device_remove_files+0xa9/0x14b
[<ffffffff8131a377>] acpi_device_unregister+0x64/0xa6
[<ffffffff8131a3e4>] acpi_bus_remove+0x2b/0x2f
[<ffffffff8131a91b>] acpi_bus_trim+0x73/0x7a
[<ffffffff8131b4a5>] acpi_scan_hot_remove+0x160/0x281
[<ffffffff8131b6c6>] ? acpi_bus_hot_remove_device+0x2a/0x69
[<ffffffff8131b6ce>] acpi_bus_hot_remove_device+0x32/0x69
[<ffffffff81315dac>] acpi_os_execute_deferred+0x27/0x34
[<ffffffff8106bec8>] process_one_work+0x1e8/0x560
[<ffffffff8106be53>] ? process_one_work+0x173/0x560
[<ffffffff8106d0a0>] worker_thread+0x120/0x3a0
[<ffffffff8106cf80>] ? manage_workers+0x170/0x170
[<ffffffff81073b5e>] kthread+0xee/0x100
[<ffffffff810bb59c>] ? __lock_release+0x9c/0x200
[<ffffffff81073a70>] ? __init_kthread_worker+0x70/0x70
[<ffffffff815a5fdc>] ret_from_fork+0x7c/0xb0
[<ffffffff81073a70>] ? __init_kthread_worker+0x70/0x70

>
> I'll address the ACPI part differently later.
>
>> ---
>> drivers/acpi/scan.c | 3 ++-
>> drivers/base/core.c | 36 ++++++++++++++++++++----------------
>> 2 files changed, 22 insertions(+), 17 deletions(-)
>>
>> Index: linux-pm/drivers/base/core.c
>> ===================================================================
>> --- linux-pm.orig/drivers/base/core.c
>> +++ linux-pm/drivers/base/core.c
>> @@ -49,6 +49,18 @@ static struct kobject *dev_kobj;
>> struct kobject *sysfs_dev_char_kobj;
>> struct kobject *sysfs_dev_block_kobj;
>>
>> +static DEFINE_MUTEX(device_hotplug_lock);
>> +
>> +void lock_device_hotplug(void)
>> +{
>> + mutex_lock(&device_hotplug_lock);
>> +}
>> +
>> +void unlock_device_hotplug(void)
>> +{
>> + mutex_unlock(&device_hotplug_lock);
>> +}
>> +
>> #ifdef CONFIG_BLOCK
>> static inline int device_is_not_partition(struct device *dev)
>> {
>> @@ -408,9 +420,11 @@ static ssize_t show_online(struct device
>> {
>> bool val;
>>
>> - lock_device_hotplug();
>> + if (!mutex_trylock(&device_hotplug_lock))
>> + return -EAGAIN;
>> +
>> val = !dev->offline;
>> - unlock_device_hotplug();
>> + mutex_unlock(&device_hotplug_lock);
>> return sprintf(buf, "%u\n", val);
>> }
>>
>> @@ -424,9 +438,11 @@ static ssize_t store_online(struct devic
>> if (ret < 0)
>> return ret;
>>
>> - lock_device_hotplug();
>> + if (!mutex_trylock(&device_hotplug_lock))
>> + return -EAGAIN;
>> +
>> ret = val ? device_online(dev) : device_offline(dev);
>> - unlock_device_hotplug();
>> + mutex_unlock(&device_hotplug_lock);
>> return ret < 0 ? ret : count;
>> }
>>
>> @@ -1479,18 +1495,6 @@ EXPORT_SYMBOL_GPL(put_device);
>> EXPORT_SYMBOL_GPL(device_create_file);
>> EXPORT_SYMBOL_GPL(device_remove_file);
>>
>> -static DEFINE_MUTEX(device_hotplug_lock);
>> -
>> -void lock_device_hotplug(void)
>> -{
>> - mutex_lock(&device_hotplug_lock);
>> -}
>> -
>> -void unlock_device_hotplug(void)
>> -{
>> - mutex_unlock(&device_hotplug_lock);
>> -}
>> -
>> static int device_check_offline(struct device *dev, void *not_used)
>> {
>> int ret;
>> Index: linux-pm/drivers/acpi/scan.c
>> ===================================================================
>> --- linux-pm.orig/drivers/acpi/scan.c
>> +++ linux-pm/drivers/acpi/scan.c
>> @@ -510,7 +510,8 @@ acpi_eject_store(struct device *d, struc
>> if (ACPI_FAILURE(status) || !acpi_device->flags.ejectable)
>> return -ENODEV;
>>
>> - mutex_lock(&acpi_scan_lock);
>> + if (!mutex_trylock(&acpi_scan_lock))
>> + return -EAGAIN;
>>
>> if (acpi_device->flags.eject_pending) {
>> /* ACPI eject notification event. */
>
> Thanks,
> Rafael
>
>
> ---
> drivers/base/core.c | 45 +++++++++++++++++++++++++++++++--------------
> 1 file changed, 31 insertions(+), 14 deletions(-)
>
> Index: linux-pm/drivers/base/core.c
> ===================================================================
> --- linux-pm.orig/drivers/base/core.c
> +++ linux-pm/drivers/base/core.c
> @@ -49,6 +49,28 @@ static struct kobject *dev_kobj;
> struct kobject *sysfs_dev_char_kobj;
> struct kobject *sysfs_dev_block_kobj;
>
> +static DEFINE_MUTEX(device_hotplug_lock);
> +
> +void lock_device_hotplug(void)
> +{
> + mutex_lock(&device_hotplug_lock);
> +}
> +
> +void unlock_device_hotplug(void)
> +{
> + mutex_unlock(&device_hotplug_lock);
> +}
> +
> +static int try_to_lock_device_hotplug(void)
> +{
> + if (!mutex_trylock(&device_hotplug_lock)) {
> + /* Avoid busy waiting, 10 ms of sleep should do. */
> + msleep(10);
> + return restart_syscall();
> + }
> + return 0;
> +}
> +
> #ifdef CONFIG_BLOCK
> static inline int device_is_not_partition(struct device *dev)
> {
> @@ -407,8 +429,12 @@ static ssize_t show_online(struct device
> char *buf)
> {
> bool val;
> + int ret;
> +
> + ret = try_to_lock_device_hotplug();
> + if (ret)
> + return ret;
>
> - lock_device_hotplug();
> val = !dev->offline;
> unlock_device_hotplug();
> return sprintf(buf, "%u\n", val);
> @@ -424,7 +450,10 @@ static ssize_t store_online(struct devic
> if (ret < 0)
> return ret;
>
> - lock_device_hotplug();
> + ret = try_to_lock_device_hotplug();
> + if (ret)
> + return ret;
> +
> ret = val ? device_online(dev) : device_offline(dev);
> unlock_device_hotplug();
> return ret < 0 ? ret : count;
> @@ -1479,18 +1508,6 @@ EXPORT_SYMBOL_GPL(put_device);
> EXPORT_SYMBOL_GPL(device_create_file);
> EXPORT_SYMBOL_GPL(device_remove_file);
>
> -static DEFINE_MUTEX(device_hotplug_lock);
> -
> -void lock_device_hotplug(void)
> -{
> - mutex_lock(&device_hotplug_lock);
> -}
> -
> -void unlock_device_hotplug(void)
> -{
> - mutex_unlock(&device_hotplug_lock);
> -}
> -
> static int device_check_offline(struct device *dev, void *not_used)
> {
> int ret;
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to [email protected]
> More majordomo info at http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at http://www.tux.org/lkml/
>

2013-08-27 09:26:07

by Gu Zheng

[permalink] [raw]
Subject: Re: [PATCH] driver core / ACPI: Avoid device removal locking problems

Hi Rafael,

On 08/26/2013 11:02 PM, Rafael J. Wysocki wrote:

> On Monday, August 26, 2013 04:43:26 PM Rafael J. Wysocki wrote:
>> On Monday, August 26, 2013 02:42:09 PM Rafael J. Wysocki wrote:
>>> On Monday, August 26, 2013 11:13:13 AM Gu Zheng wrote:
>>>> Hi Rafael,
>
> [...]
>
>>
>> OK, so the patch below is quick and dirty and overkill, but it should make the
>> splat go away at least.
>
> And if this patch does make the splat go away for you, please also test the
> appended one (Tejun, thanks for the hint!).
>
> I'll address the ACPI part differently later.

What about changing device_hotplug_lock and acpi_scan_lock to rwsem? like the
attached one(With a preliminary test, it also can make the splat go away).:)

Regards,
Gu

>
[...]
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to [email protected]
> More majordomo info at http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at http://www.tux.org/lkml/
>



Attachments:
0001-acpi-fix-removal-lock-dep.patch (8.54 kB)

2013-08-27 18:36:50

by Tejun Heo

[permalink] [raw]
Subject: Re: [PATCH] driver core / ACPI: Avoid device removal locking problems

Hello,

On Tue, Aug 27, 2013 at 05:21:44PM +0800, Gu Zheng wrote:
> >> OK, so the patch below is quick and dirty and overkill, but it should make the
> >> splat go away at least.
> >
> > And if this patch does make the splat go away for you, please also test the
> > appended one (Tejun, thanks for the hint!).
> >
> > I'll address the ACPI part differently later.
>
> What about changing device_hotplug_lock and acpi_scan_lock to rwsem? like the
> attached one(With a preliminary test, it also can make the splat go away).:)

Hmmm.. I don't get it. How is introducing another rwlock whic may
cause the operation, even reading the status, to fail randomly a
better option? It's harier and more brittle. We probably want to
implement better solution in sysfs for files which interact with
device addition / removal paths but for now I think Rafael's patch is
the right direction.

Thanks.

--
tejun

2013-08-27 21:35:07

by Rafael J. Wysocki

[permalink] [raw]
Subject: Re: [PATCH] driver core / ACPI: Avoid device removal locking problems

On Tuesday, August 27, 2013 02:36:44 PM Tejun Heo wrote:
> Hello,
>
> On Tue, Aug 27, 2013 at 05:21:44PM +0800, Gu Zheng wrote:
> > >> OK, so the patch below is quick and dirty and overkill, but it should make the
> > >> splat go away at least.
> > >
> > > And if this patch does make the splat go away for you, please also test the
> > > appended one (Tejun, thanks for the hint!).
> > >
> > > I'll address the ACPI part differently later.
> >
> > What about changing device_hotplug_lock and acpi_scan_lock to rwsem? like the
> > attached one(With a preliminary test, it also can make the splat go away).:)
>
> Hmmm.. I don't get it. How is introducing another rwlock whic may
> cause the operation, even reading the status, to fail randomly a
> better option?

That's more about replacing an existing mutex with an rwsem, but I don't think
it helps anyway. And as I said before, the acpi_eject_store() case is entirely
separate in my opinion.

> It's harier and more brittle. We probably want to
> implement better solution in sysfs for files which interact with
> device addition / removal paths but for now I think Rafael's patch is
> the right direction.

I've thought about that a bit over the last several hours and I'm still
thinking that that patch is a bit overkill, because it will trigger the
restart_syscall() for all cases when device_hotplug_lock is locked, even
if they can't lead to any deadlocks. The only deadlockish situation is
when device *removal* is in progress when store_online(), for example,
is called.

So to address that particular situation without adding too much overhead for
other cases, I've come up with the appended patch (untested for now).

This is how it is supposed to work.

There are three "lock levels" for device hotplug, "normal", "remove"
and "weak". The difference is related to how __lock_device_hotplug()
works. Namely, if device hotplug is currently locked, that function
will either block or return false, depending on the "current lock
level" and its argument (the "new lock level"). The rules here are
that false is returned immediately if the "current lock level" is
"remove" and the "new lock level" is "weak". The function blocks
for all other combinations of the two.

There are two functions supposed to use device hotplug "lock levels"
other than "normal": store_online() and acpi_scan_hot_remove().
Everybody else is supposed to use "normal" (well, there are more
potential users of "weak" in drivers/base/memory.c).

acpi_scan_hot_remove() uses the "remove" lock level to indicate
that it is going to remove devices while holding device hotplug
locked. In turn, store_online() uses the "weak" lock level so
that it doesn't block when devices are being removed with device
hotplug locked, because that may lead to a deadlock.

show_online() actually doesn't need to lock device hotplug, but
it is useful to serialize it with respect to device_offline()
and device_online() (in case user space attempts to run them
concurrently).

---
drivers/acpi/scan.c | 4 +-
drivers/base/core.c | 72 ++++++++++++++++++++++++++++++++++++++-----------
include/linux/device.h | 25 ++++++++++++++++-
3 files changed, 83 insertions(+), 18 deletions(-)

Index: linux-pm/drivers/base/core.c
===================================================================
--- linux-pm.orig/drivers/base/core.c
+++ linux-pm/drivers/base/core.c
@@ -49,6 +49,55 @@ static struct kobject *dev_kobj;
struct kobject *sysfs_dev_char_kobj;
struct kobject *sysfs_dev_block_kobj;

+static struct {
+ struct task_struct *holder;
+ enum dev_hotplug_lock_type type;
+ struct mutex lock; /* Synchronizes accesses to holder and type */
+ wait_queue_head_t wait_queue;
+} device_hotplug = {
+ .holder = NULL,
+ .type = DEV_HOTPLUG_LOCK_NONE,
+ .lock = __MUTEX_INITIALIZER(device_hotplug.lock),
+ .wait_queue = __WAIT_QUEUE_HEAD_INITIALIZER(device_hotplug.wait_queue),
+};
+
+bool __lock_device_hotplug(enum dev_hotplug_lock_type type)
+{
+ DEFINE_WAIT(wait);
+ bool ret = true;
+
+ mutex_lock(&device_hotplug.lock);
+ for (;;) {
+ prepare_to_wait(&device_hotplug.wait_queue, &wait,
+ TASK_UNINTERRUPTIBLE);
+ if (!device_hotplug.holder) {
+ device_hotplug.holder = current;
+ device_hotplug.type = type;
+ break;
+ } else if (type == DEV_HOTPLUG_LOCK_WEAK
+ && device_hotplug.type == DEV_HOTPLUG_LOCK_REMOVE) {
+ ret = false;
+ break;
+ }
+ mutex_unlock(&device_hotplug.lock);
+ schedule();
+ mutex_lock(&device_hotplug.lock);
+ }
+ finish_wait(&device_hotplug.wait_queue, &wait);
+ mutex_unlock(&device_hotplug.lock);
+ return ret;
+}
+
+void unlock_device_hotplug(void)
+{
+ mutex_lock(&device_hotplug.lock);
+ BUG_ON(device_hotplug.holder != current);
+ device_hotplug.holder = NULL;
+ device_hotplug.type = DEV_HOTPLUG_LOCK_NONE;
+ wake_up(&device_hotplug.wait_queue);
+ mutex_unlock(&device_hotplug.lock);
+}
+
#ifdef CONFIG_BLOCK
static inline int device_is_not_partition(struct device *dev)
{
@@ -408,9 +457,10 @@ static ssize_t show_online(struct device
{
bool val;

- lock_device_hotplug();
+ /* Serialize against device_online() and device_offline(). */
+ device_lock(dev);
val = !dev->offline;
- unlock_device_hotplug();
+ device_unlock(dev);
return sprintf(buf, "%u\n", val);
}

@@ -424,7 +474,11 @@ static ssize_t store_online(struct devic
if (ret < 0)
return ret;

- lock_device_hotplug();
+ if (!__lock_device_hotplug(DEV_HOTPLUG_LOCK_WEAK)) {
+ /* Avoid busy looping (10 ms delay should do). */
+ msleep(10);
+ return restart_syscall();
+ }
ret = val ? device_online(dev) : device_offline(dev);
unlock_device_hotplug();
return ret < 0 ? ret : count;
@@ -1479,18 +1533,6 @@ EXPORT_SYMBOL_GPL(put_device);
EXPORT_SYMBOL_GPL(device_create_file);
EXPORT_SYMBOL_GPL(device_remove_file);

-static DEFINE_MUTEX(device_hotplug_lock);
-
-void lock_device_hotplug(void)
-{
- mutex_lock(&device_hotplug_lock);
-}
-
-void unlock_device_hotplug(void)
-{
- mutex_unlock(&device_hotplug_lock);
-}
-
static int device_check_offline(struct device *dev, void *not_used)
{
int ret;
Index: linux-pm/include/linux/device.h
===================================================================
--- linux-pm.orig/include/linux/device.h
+++ linux-pm/include/linux/device.h
@@ -893,8 +893,31 @@ static inline bool device_supports_offli
return dev->bus && dev->bus->offline && dev->bus->online;
}

-extern void lock_device_hotplug(void);
+enum dev_hotplug_lock_type {
+ DEV_HOTPLUG_LOCK_NONE,
+ DEV_HOTPLUG_LOCK_NORMAL,
+ DEV_HOTPLUG_LOCK_REMOVE,
+ DEV_HOTPLUG_LOCK_WEAK,
+};
+
+extern bool __lock_device_hotplug(enum dev_hotplug_lock_type);
extern void unlock_device_hotplug(void);
+
+static inline void lock_device_hotplug(void)
+{
+ __lock_device_hotplug(DEV_HOTPLUG_LOCK_NORMAL);
+}
+
+static inline void lock_device_hot_remove(void)
+{
+ __lock_device_hotplug(DEV_HOTPLUG_LOCK_REMOVE);
+}
+
+static inline void unlock_device_hot_remove(void)
+{
+ unlock_device_hotplug();
+}
+
extern int device_offline(struct device *dev);
extern int device_online(struct device *dev);
/*
Index: linux-pm/drivers/acpi/scan.c
===================================================================
--- linux-pm.orig/drivers/acpi/scan.c
+++ linux-pm/drivers/acpi/scan.c
@@ -204,7 +204,7 @@ static int acpi_scan_hot_remove(struct a
return -EINVAL;
}

- lock_device_hotplug();
+ lock_device_hot_remove();

/*
* Carry out two passes here and ignore errors in the first pass,
@@ -249,7 +249,7 @@ static int acpi_scan_hot_remove(struct a

acpi_bus_trim(device);

- unlock_device_hotplug();
+ unlock_device_hot_remove();

/* Device node has been unregistered. */
put_device(&device->dev);

2013-08-27 21:40:27

by Toshi Kani

[permalink] [raw]
Subject: Re: [PATCH] driver core / ACPI: Avoid device removal locking problems

On Tue, 2013-08-27 at 17:21 +0800, Gu Zheng wrote:
> Hi Rafael,
>
> On 08/26/2013 11:02 PM, Rafael J. Wysocki wrote:
>
> > On Monday, August 26, 2013 04:43:26 PM Rafael J. Wysocki wrote:
> >> On Monday, August 26, 2013 02:42:09 PM Rafael J. Wysocki wrote:
> >>> On Monday, August 26, 2013 11:13:13 AM Gu Zheng wrote:
> >>>> Hi Rafael,
> >
> > [...]
> >
> >>
> >> OK, so the patch below is quick and dirty and overkill, but it should make the
> >> splat go away at least.
> >
> > And if this patch does make the splat go away for you, please also test the
> > appended one (Tejun, thanks for the hint!).
> >
> > I'll address the ACPI part differently later.
>
> What about changing device_hotplug_lock and acpi_scan_lock to rwsem? like the
> attached one(With a preliminary test, it also can make the splat go away).:)

I am curious how msleep(10) & restart_syscall() work in the change
below. Doesn't the msleep() make s_active held longer time, which can
lead the thread holding device_hotplug_lock to wait it for deletion?
Also, does restart_syscall() release s_active and reopen this file
again?

@@ -408,9 +408,13 @@ static ssize_t show_online(struct device *dev,
struct device_attribute *attr,
{
bool val;

- lock_device_hotplug();
+ if (!read_lock_device_hotplug()) {
+ msleep(10);
+ return restart_syscall();
+ }
+

Thanks,
-Toshi

2013-08-28 02:16:45

by Gu Zheng

[permalink] [raw]
Subject: Re: [PATCH] driver core / ACPI: Avoid device removal locking problems

Hi Toshi,

On 08/28/2013 05:38 AM, Toshi Kani wrote:

> On Tue, 2013-08-27 at 17:21 +0800, Gu Zheng wrote:
>> Hi Rafael,
>>
>> On 08/26/2013 11:02 PM, Rafael J. Wysocki wrote:
>>
>>> On Monday, August 26, 2013 04:43:26 PM Rafael J. Wysocki wrote:
>>>> On Monday, August 26, 2013 02:42:09 PM Rafael J. Wysocki wrote:
>>>>> On Monday, August 26, 2013 11:13:13 AM Gu Zheng wrote:
>>>>>> Hi Rafael,
>>>
>>> [...]
>>>
>>>>
>>>> OK, so the patch below is quick and dirty and overkill, but it should make the
>>>> splat go away at least.
>>>
>>> And if this patch does make the splat go away for you, please also test the
>>> appended one (Tejun, thanks for the hint!).
>>>
>>> I'll address the ACPI part differently later.
>>
>> What about changing device_hotplug_lock and acpi_scan_lock to rwsem? like the
>> attached one(With a preliminary test, it also can make the splat go away).:)
>
> I am curious how msleep(10) & restart_syscall() work in the change
> below. Doesn't the msleep() make s_active held longer time, which can
> lead the thread holding device_hotplug_lock to wait it for deletion?

Yes, but it can avoid busy waiting.

> Also, does restart_syscall() release s_active and reopen this file
> again?

Sure, it just set a TIF_SIGPENDING flag and return an -ERESTARTNOINTR error, s_active/file
will be released/closed in the failed path. And when do_signal() catches the -ERESTARTNOINTR,
it will change the regs to restart the syscall.

Thanks,
Gu

>
> @@ -408,9 +408,13 @@ static ssize_t show_online(struct device *dev,
> struct device_attribute *attr,
> {
> bool val;
>
> - lock_device_hotplug();
> + if (!read_lock_device_hotplug()) {
> + msleep(10);
> + return restart_syscall();
> + }
> +
>
> Thanks,
> -Toshi
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to [email protected]
> More majordomo info at http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at http://www.tux.org/lkml/
>

2013-08-28 10:13:55

by Gu Zheng

[permalink] [raw]
Subject: Re: [PATCH] driver core / ACPI: Avoid device removal locking problems

Hi Rafael,

On 08/28/2013 05:45 AM, Rafael J. Wysocki wrote:

> On Tuesday, August 27, 2013 02:36:44 PM Tejun Heo wrote:
>> Hello,
>>
[...]
>
> I've thought about that a bit over the last several hours and I'm still
> thinking that that patch is a bit overkill, because it will trigger the
> restart_syscall() for all cases when device_hotplug_lock is locked, even
> if they can't lead to any deadlocks. The only deadlockish situation is
> when device *removal* is in progress when store_online(), for example,
> is called.
>
> So to address that particular situation without adding too much overhead for
> other cases, I've come up with the appended patch (untested for now).
>
> This is how it is supposed to work.
>
> There are three "lock levels" for device hotplug, "normal", "remove"
> and "weak". The difference is related to how __lock_device_hotplug()
> works. Namely, if device hotplug is currently locked, that function
> will either block or return false, depending on the "current lock
> level" and its argument (the "new lock level"). The rules here are
> that false is returned immediately if the "current lock level" is
> "remove" and the "new lock level" is "weak". The function blocks
> for all other combinations of the two.
>
> There are two functions supposed to use device hotplug "lock levels"
> other than "normal": store_online() and acpi_scan_hot_remove().
> Everybody else is supposed to use "normal" (well, there are more
> potential users of "weak" in drivers/base/memory.c).
>
> acpi_scan_hot_remove() uses the "remove" lock level to indicate
> that it is going to remove devices while holding device hotplug
> locked. In turn, store_online() uses the "weak" lock level so
> that it doesn't block when devices are being removed with device
> hotplug locked, because that may lead to a deadlock.
>
> show_online() actually doesn't need to lock device hotplug, but
> it is useful to serialize it with respect to device_offline()
> and device_online() (in case user space attempts to run them
> concurrently).

Yeah. I tested this one on latest kernel tree, it does make the splat go away.
Looking forward to the ACPI part one.:)

Regards,
Gu

>
> ---
> drivers/acpi/scan.c | 4 +-
> drivers/base/core.c | 72 ++++++++++++++++++++++++++++++++++++++-----------
> include/linux/device.h | 25 ++++++++++++++++-
> 3 files changed, 83 insertions(+), 18 deletions(-)
>
> Index: linux-pm/drivers/base/core.c
> ===================================================================
> --- linux-pm.orig/drivers/base/core.c
> +++ linux-pm/drivers/base/core.c
> @@ -49,6 +49,55 @@ static struct kobject *dev_kobj;
> struct kobject *sysfs_dev_char_kobj;
> struct kobject *sysfs_dev_block_kobj;
>
> +static struct {
> + struct task_struct *holder;
> + enum dev_hotplug_lock_type type;
> + struct mutex lock; /* Synchronizes accesses to holder and type */
> + wait_queue_head_t wait_queue;
> +} device_hotplug = {
> + .holder = NULL,
> + .type = DEV_HOTPLUG_LOCK_NONE,
> + .lock = __MUTEX_INITIALIZER(device_hotplug.lock),
> + .wait_queue = __WAIT_QUEUE_HEAD_INITIALIZER(device_hotplug.wait_queue),
> +};
> +
> +bool __lock_device_hotplug(enum dev_hotplug_lock_type type)
> +{
> + DEFINE_WAIT(wait);
> + bool ret = true;
> +
> + mutex_lock(&device_hotplug.lock);
> + for (;;) {
> + prepare_to_wait(&device_hotplug.wait_queue, &wait,
> + TASK_UNINTERRUPTIBLE);
> + if (!device_hotplug.holder) {
> + device_hotplug.holder = current;
> + device_hotplug.type = type;
> + break;
> + } else if (type == DEV_HOTPLUG_LOCK_WEAK
> + && device_hotplug.type == DEV_HOTPLUG_LOCK_REMOVE) {
> + ret = false;
> + break;
> + }
> + mutex_unlock(&device_hotplug.lock);
> + schedule();
> + mutex_lock(&device_hotplug.lock);
> + }
> + finish_wait(&device_hotplug.wait_queue, &wait);
> + mutex_unlock(&device_hotplug.lock);
> + return ret;
> +}
> +
> +void unlock_device_hotplug(void)
> +{
> + mutex_lock(&device_hotplug.lock);
> + BUG_ON(device_hotplug.holder != current);
> + device_hotplug.holder = NULL;
> + device_hotplug.type = DEV_HOTPLUG_LOCK_NONE;
> + wake_up(&device_hotplug.wait_queue);
> + mutex_unlock(&device_hotplug.lock);
> +}
> +
> #ifdef CONFIG_BLOCK
> static inline int device_is_not_partition(struct device *dev)
> {
> @@ -408,9 +457,10 @@ static ssize_t show_online(struct device
> {
> bool val;
>
> - lock_device_hotplug();
> + /* Serialize against device_online() and device_offline(). */
> + device_lock(dev);
> val = !dev->offline;
> - unlock_device_hotplug();
> + device_unlock(dev);
> return sprintf(buf, "%u\n", val);
> }
>
> @@ -424,7 +474,11 @@ static ssize_t store_online(struct devic
> if (ret < 0)
> return ret;
>
> - lock_device_hotplug();
> + if (!__lock_device_hotplug(DEV_HOTPLUG_LOCK_WEAK)) {
> + /* Avoid busy looping (10 ms delay should do). */
> + msleep(10);
> + return restart_syscall();
> + }
> ret = val ? device_online(dev) : device_offline(dev);
> unlock_device_hotplug();
> return ret < 0 ? ret : count;
> @@ -1479,18 +1533,6 @@ EXPORT_SYMBOL_GPL(put_device);
> EXPORT_SYMBOL_GPL(device_create_file);
> EXPORT_SYMBOL_GPL(device_remove_file);
>
> -static DEFINE_MUTEX(device_hotplug_lock);
> -
> -void lock_device_hotplug(void)
> -{
> - mutex_lock(&device_hotplug_lock);
> -}
> -
> -void unlock_device_hotplug(void)
> -{
> - mutex_unlock(&device_hotplug_lock);
> -}
> -
> static int device_check_offline(struct device *dev, void *not_used)
> {
> int ret;
> Index: linux-pm/include/linux/device.h
> ===================================================================
> --- linux-pm.orig/include/linux/device.h
> +++ linux-pm/include/linux/device.h
> @@ -893,8 +893,31 @@ static inline bool device_supports_offli
> return dev->bus && dev->bus->offline && dev->bus->online;
> }
>
> -extern void lock_device_hotplug(void);
> +enum dev_hotplug_lock_type {
> + DEV_HOTPLUG_LOCK_NONE,
> + DEV_HOTPLUG_LOCK_NORMAL,
> + DEV_HOTPLUG_LOCK_REMOVE,
> + DEV_HOTPLUG_LOCK_WEAK,
> +};
> +
> +extern bool __lock_device_hotplug(enum dev_hotplug_lock_type);
> extern void unlock_device_hotplug(void);
> +
> +static inline void lock_device_hotplug(void)
> +{
> + __lock_device_hotplug(DEV_HOTPLUG_LOCK_NORMAL);
> +}
> +
> +static inline void lock_device_hot_remove(void)
> +{
> + __lock_device_hotplug(DEV_HOTPLUG_LOCK_REMOVE);
> +}
> +
> +static inline void unlock_device_hot_remove(void)
> +{
> + unlock_device_hotplug();
> +}
> +
> extern int device_offline(struct device *dev);
> extern int device_online(struct device *dev);
> /*
> Index: linux-pm/drivers/acpi/scan.c
> ===================================================================
> --- linux-pm.orig/drivers/acpi/scan.c
> +++ linux-pm/drivers/acpi/scan.c
> @@ -204,7 +204,7 @@ static int acpi_scan_hot_remove(struct a
> return -EINVAL;
> }
>
> - lock_device_hotplug();
> + lock_device_hot_remove();
>
> /*
> * Carry out two passes here and ignore errors in the first pass,
> @@ -249,7 +249,7 @@ static int acpi_scan_hot_remove(struct a
>
> acpi_bus_trim(device);
>
> - unlock_device_hotplug();
> + unlock_device_hot_remove();
>
> /* Device node has been unregistered. */
> put_device(&device->dev);
>
>

2013-08-28 12:24:28

by Tejun Heo

[permalink] [raw]
Subject: Re: [PATCH] driver core / ACPI: Avoid device removal locking problems

Hello, Rafael.

On Tue, Aug 27, 2013 at 11:45:46PM +0200, Rafael J. Wysocki wrote:
> I've thought about that a bit over the last several hours and I'm still
> thinking that that patch is a bit overkill, because it will trigger the
> restart_syscall() for all cases when device_hotplug_lock is locked, even
> if they can't lead to any deadlocks. The only deadlockish situation is
> when device *removal* is in progress when store_online(), for example,
> is called.
>
> So to address that particular situation without adding too much overhead for
> other cases, I've come up with the appended patch (untested for now).
>
> This is how it is supposed to work.
>
> There are three "lock levels" for device hotplug, "normal", "remove"
> and "weak". The difference is related to how __lock_device_hotplug()
> works. Namely, if device hotplug is currently locked, that function
> will either block or return false, depending on the "current lock
> level" and its argument (the "new lock level"). The rules here are
> that false is returned immediately if the "current lock level" is
> "remove" and the "new lock level" is "weak". The function blocks
> for all other combinations of the two.

I think this is going way too far and a massive overkill in terms of
complexity, which is way more important than for doing some
restart_syscall loops during some rare sysfs file access, which isn't
gonna be that common anyway. Fancy locking always sucks. The root
cause of the problem is file removal ref draining from sysfs side and
a proper solution should be implemented there. Adding all this
complexity to device hotplug lock won't solve problems involving other
locks while obfuscating what's going on through all the complexity.
Also, when sysfs is updated with a proper solution, a simpler
workaround from device hotplug side would be far easier to convert to
the new solution than this, which over time is likely to grow
interesting uses.

Let's *please* stick to something simple.

Thanks.

--
tejun

2013-08-28 13:14:09

by Rafael J. Wysocki

[permalink] [raw]
Subject: Re: [PATCH] driver core / ACPI: Avoid device removal locking problems

On Wednesday, August 28, 2013 08:24:22 AM Tejun Heo wrote:
> Hello, Rafael.
>
> On Tue, Aug 27, 2013 at 11:45:46PM +0200, Rafael J. Wysocki wrote:
> > I've thought about that a bit over the last several hours and I'm still
> > thinking that that patch is a bit overkill, because it will trigger the
> > restart_syscall() for all cases when device_hotplug_lock is locked, even
> > if they can't lead to any deadlocks. The only deadlockish situation is
> > when device *removal* is in progress when store_online(), for example,
> > is called.
> >
> > So to address that particular situation without adding too much overhead for
> > other cases, I've come up with the appended patch (untested for now).
> >
> > This is how it is supposed to work.
> >
> > There are three "lock levels" for device hotplug, "normal", "remove"
> > and "weak". The difference is related to how __lock_device_hotplug()
> > works. Namely, if device hotplug is currently locked, that function
> > will either block or return false, depending on the "current lock
> > level" and its argument (the "new lock level"). The rules here are
> > that false is returned immediately if the "current lock level" is
> > "remove" and the "new lock level" is "weak". The function blocks
> > for all other combinations of the two.
>
> I think this is going way too far and a massive overkill in terms of
> complexity, which is way more important than for doing some
> restart_syscall loops during some rare sysfs file access, which isn't
> gonna be that common anyway. Fancy locking always sucks. The root
> cause of the problem is file removal ref draining from sysfs side and
> a proper solution should be implemented there. Adding all this
> complexity to device hotplug lock won't solve problems involving other
> locks while obfuscating what's going on through all the complexity.
> Also, when sysfs is updated with a proper solution, a simpler
> workaround from device hotplug side would be far easier to convert to
> the new solution than this, which over time is likely to grow
> interesting uses.
>
> Let's *please* stick to something simple.

OK, I'll use restart_syscall() approach in the simplest form, then.

Thanks,
Rafael


--
I speak only for myself.
Rafael J. Wysocki, Intel Open Source Technology Center.

2013-08-28 13:41:09

by Rafael J. Wysocki

[permalink] [raw]
Subject: [PATCH 0/2] driver core / ACPI: Avoid device removal locking problems

Hi All,

The following two patches are to address possible deadlocks related to
device removal and device sysfs attribute access. In short, some device
sysfs attribute callbacks need to acquire locks that are also held around
device removal and that may lead to deadlocks with s_active draining in
sysfs_deactivate().

[1/2] Avoid possible device removal deadlocks related to device_hotplug_lock.
[2/2] Rework the handling of containers by ACPI hotplug (which makes a possible
device removal deadlock related to acpi_scan_lock go away).

On top of linux-next, for v3.12.

Thanks,
Rafael


--
I speak only for myself.
Rafael J. Wysocki, Intel Open Source Technology Center.

2013-08-28 13:41:07

by Rafael J. Wysocki

[permalink] [raw]
Subject: [PATCH 1/2] driver core / ACPI: Avoid device hot remove locking issues

From: Rafael J. Wysocki <[email protected]>

device_hotplug_lock is held around the acpi_bus_trim() call in
acpi_scan_hot_remove() which generally removes devices (it removes
ACPI device objects at least, but it may also remove "physical"
device objects through .detach() callbacks of ACPI scan handlers).
Thus, potentially, device sysfs attributes are removed under that
lock and to remove those attributes it is necessary to hold the
s_active references of their directory entries for writing.

On the other hand, the execution of a .show() or .store() callback
from a sysfs attribute is carried out with that attribute's s_active
reference held for reading. Consequently, if any device sysfs
attribute that may be removed from within acpi_scan_hot_remove()
through acpi_bus_trim() has a .store() or .show() callback which
acquires device_hotplug_lock, the execution of that callback may
deadlock with the removal of the attribute. [Unfortunately, the
"online" device attribute of CPUs and memory blocks is one of them.]

To avoid such deadlocks, make all of the sysfs attribute callbacks
that need to lock device hotplug, for example store_online(), use
a special function, lock_device_hotplug_sysfs(), to lock device
hotplug and return the result of that function immediately if it is
not zero. This will cause the s_active reference of the directory
entry in question to be released and the syscall to be restarted
if device_hotplug_lock cannot be acquired.

[show_online() actually doesn't need to lock device hotplug, but
it is useful to serialize it with respect to device_offline() and
device_online() for the same device (in case user space attempts to
run them concurrently) which can be done with the help of
device_lock().]

Signed-off-by: Rafael J. Wysocki <[email protected]>
Reported-by: Yasuaki Ishimatsu <[email protected]>
Reported-by: Gu Zheng <[email protected]>
---
drivers/acpi/sysfs.c | 5 ++++-
drivers/base/core.c | 43 ++++++++++++++++++++++++++++---------------
drivers/base/memory.c | 4 +++-
include/linux/device.h | 1 +
4 files changed, 36 insertions(+), 17 deletions(-)

Index: linux-pm/drivers/base/core.c
===================================================================
--- linux-pm.orig/drivers/base/core.c
+++ linux-pm/drivers/base/core.c
@@ -49,6 +49,28 @@ static struct kobject *dev_kobj;
struct kobject *sysfs_dev_char_kobj;
struct kobject *sysfs_dev_block_kobj;

+static DEFINE_MUTEX(device_hotplug_lock);
+
+void lock_device_hotplug(void)
+{
+ mutex_lock(&device_hotplug_lock);
+}
+
+void unlock_device_hotplug(void)
+{
+ mutex_unlock(&device_hotplug_lock);
+}
+
+int lock_device_hotplug_sysfs(void)
+{
+ if (mutex_trylock(&device_hotplug_lock))
+ return 0;
+
+ /* Avoid busy looping (5 ms of sleep should do). */
+ msleep(5);
+ return restart_syscall();
+}
+
#ifdef CONFIG_BLOCK
static inline int device_is_not_partition(struct device *dev)
{
@@ -408,9 +430,9 @@ static ssize_t show_online(struct device
{
bool val;

- lock_device_hotplug();
+ device_lock(dev);
val = !dev->offline;
- unlock_device_hotplug();
+ device_unlock(dev);
return sprintf(buf, "%u\n", val);
}

@@ -424,7 +446,10 @@ static ssize_t store_online(struct devic
if (ret < 0)
return ret;

- lock_device_hotplug();
+ ret = lock_device_hotplug_sysfs();
+ if (ret)
+ return ret;
+
ret = val ? device_online(dev) : device_offline(dev);
unlock_device_hotplug();
return ret < 0 ? ret : count;
@@ -1479,18 +1504,6 @@ EXPORT_SYMBOL_GPL(put_device);
EXPORT_SYMBOL_GPL(device_create_file);
EXPORT_SYMBOL_GPL(device_remove_file);

-static DEFINE_MUTEX(device_hotplug_lock);
-
-void lock_device_hotplug(void)
-{
- mutex_lock(&device_hotplug_lock);
-}
-
-void unlock_device_hotplug(void)
-{
- mutex_unlock(&device_hotplug_lock);
-}
-
static int device_check_offline(struct device *dev, void *not_used)
{
int ret;
Index: linux-pm/drivers/base/memory.c
===================================================================
--- linux-pm.orig/drivers/base/memory.c
+++ linux-pm/drivers/base/memory.c
@@ -351,7 +351,9 @@ store_mem_state(struct device *dev,

mem = container_of(dev, struct memory_block, dev);

- lock_device_hotplug();
+ ret = lock_device_hotplug_sysfs();
+ if (ret)
+ return ret;

if (!strncmp(buf, "online_kernel", min_t(int, count, 13))) {
offline = false;
Index: linux-pm/drivers/acpi/sysfs.c
===================================================================
--- linux-pm.orig/drivers/acpi/sysfs.c
+++ linux-pm/drivers/acpi/sysfs.c
@@ -796,7 +796,10 @@ static ssize_t force_remove_store(struct
if (ret < 0)
return ret;

- lock_device_hotplug();
+ ret = lock_device_hotplug_sysfs();
+ if (ret)
+ return ret;
+
acpi_force_hot_remove = val;
unlock_device_hotplug();
return size;
Index: linux-pm/include/linux/device.h
===================================================================
--- linux-pm.orig/include/linux/device.h
+++ linux-pm/include/linux/device.h
@@ -895,6 +895,7 @@ static inline bool device_supports_offli

extern void lock_device_hotplug(void);
extern void unlock_device_hotplug(void);
+extern int lock_device_hotplug_sysfs(void);
extern int device_offline(struct device *dev);
extern int device_online(struct device *dev);
/*

2013-08-28 13:41:06

by Rafael J. Wysocki

[permalink] [raw]
Subject: [PATCH 2/2] ACPI / hotplug: Remove containers synchronously

From: Rafael J. Wysocki <[email protected]>

The current protocol for handling hot remove of containers is very
fragile and causes acpi_eject_store() to acquire acpi_scan_lock
which may deadlock with the removal of the device that it is called
for (the reason is that device sysfs attributes cannot be removed
while their callbacks are being executed and ACPI device objects
are removed under acpi_scan_lock).

The problem is related to the fact that containers are handled by
acpi_bus_device_eject() in a special way, which is to emit an
offline uevent instead of just removing the container. Then, user
space is expected to handle that uevent and use the container's
"eject" attribute to actually remove it. That is fragile, because
user space may fail to complete the ejection (for example, by not
using the container's "eject" attribute at all) leaving the BIOS
kind of in a limbo. Moreover, if the eject event is not signaled
for a container itself, but for its parent device object (or
generally, for an ancestor above it in the ACPI namespace), the
container will be removed straight away without doing that whole
dance.

For this reason, modify acpi_bus_device_eject() to remove containers
synchronously like any other objects (user space will get its uevent
anyway in case it does some other things in response to it) and
remove the eject_pending ACPI device flag that is not used any more.
This way acpi_eject_store() doesn't have a reason to acquire
acpi_scan_lock any more and one possible deadlock scenario goes
away (plus the code is simplified a bit).

Signed-off-by: Rafael J. Wysocki <[email protected]>
Reported-by: Gu Zheng <[email protected]>
---
drivers/acpi/scan.c | 49 ++++++++++++++----------------------------------
include/acpi/acpi_bus.h | 3 --
2 files changed, 16 insertions(+), 36 deletions(-)

Index: linux-pm/drivers/acpi/scan.c
===================================================================
--- linux-pm.orig/drivers/acpi/scan.c
+++ linux-pm/drivers/acpi/scan.c
@@ -287,6 +287,7 @@ static void acpi_bus_device_eject(void *
struct acpi_device *device = NULL;
struct acpi_scan_handler *handler;
u32 ost_code = ACPI_OST_SC_NON_SPECIFIC_FAILURE;
+ int error;

mutex_lock(&acpi_scan_lock);

@@ -301,17 +302,13 @@ static void acpi_bus_device_eject(void *
}
acpi_evaluate_hotplug_ost(handle, ACPI_NOTIFY_EJECT_REQUEST,
ACPI_OST_SC_EJECT_IN_PROGRESS, NULL);
- if (handler->hotplug.mode == AHM_CONTAINER) {
- device->flags.eject_pending = true;
+ if (handler->hotplug.mode == AHM_CONTAINER)
kobject_uevent(&device->dev.kobj, KOBJ_OFFLINE);
- } else {
- int error;

- get_device(&device->dev);
- error = acpi_scan_hot_remove(device);
- if (error)
- goto err_out;
- }
+ get_device(&device->dev);
+ error = acpi_scan_hot_remove(device);
+ if (error)
+ goto err_out;

out:
mutex_unlock(&acpi_scan_lock);
@@ -496,7 +493,6 @@ acpi_eject_store(struct device *d, struc
struct acpi_eject_event *ej_event;
acpi_object_type not_used;
acpi_status status;
- u32 ost_source;
int ret;

if (!count || buf[0] != '1')
@@ -510,43 +506,28 @@ acpi_eject_store(struct device *d, struc
if (ACPI_FAILURE(status) || !acpi_device->flags.ejectable)
return -ENODEV;

- mutex_lock(&acpi_scan_lock);
-
- if (acpi_device->flags.eject_pending) {
- /* ACPI eject notification event. */
- ost_source = ACPI_NOTIFY_EJECT_REQUEST;
- acpi_device->flags.eject_pending = 0;
- } else {
- /* Eject initiated by user space. */
- ost_source = ACPI_OST_EC_OSPM_EJECT;
- }
ej_event = kmalloc(sizeof(*ej_event), GFP_KERNEL);
if (!ej_event) {
ret = -ENOMEM;
goto err_out;
}
- acpi_evaluate_hotplug_ost(acpi_device->handle, ost_source,
+ acpi_evaluate_hotplug_ost(acpi_device->handle, ACPI_OST_EC_OSPM_EJECT,
ACPI_OST_SC_EJECT_IN_PROGRESS, NULL);
ej_event->device = acpi_device;
- ej_event->event = ost_source;
+ ej_event->event = ACPI_OST_EC_OSPM_EJECT;
get_device(&acpi_device->dev);
status = acpi_os_hotplug_execute(acpi_bus_hot_remove_device, ej_event);
- if (ACPI_FAILURE(status)) {
- put_device(&acpi_device->dev);
- kfree(ej_event);
- ret = status == AE_NO_MEMORY ? -ENOMEM : -EAGAIN;
- goto err_out;
- }
- ret = count;
+ if (ACPI_SUCCESS(status))
+ return count;

- out:
- mutex_unlock(&acpi_scan_lock);
- return ret;
+ put_device(&acpi_device->dev);
+ kfree(ej_event);
+ ret = status == AE_NO_MEMORY ? -ENOMEM : -EAGAIN;

err_out:
- acpi_evaluate_hotplug_ost(acpi_device->handle, ost_source,
+ acpi_evaluate_hotplug_ost(acpi_device->handle, ACPI_OST_EC_OSPM_EJECT,
ACPI_OST_SC_NON_SPECIFIC_FAILURE, NULL);
- goto out;
+ return ret;
}

static DEVICE_ATTR(eject, 0200, NULL, acpi_eject_store);
Index: linux-pm/include/acpi/acpi_bus.h
===================================================================
--- linux-pm.orig/include/acpi/acpi_bus.h
+++ linux-pm/include/acpi/acpi_bus.h
@@ -167,9 +167,8 @@ struct acpi_device_flags {
u32 removable:1;
u32 ejectable:1;
u32 power_manageable:1;
- u32 eject_pending:1;
u32 match_driver:1;
- u32 reserved:26;
+ u32 reserved:27;
};

/* File System */

2013-08-28 16:57:16

by Toshi Kani

[permalink] [raw]
Subject: Re: [PATCH] driver core / ACPI: Avoid device removal locking problems

On Wed, 2013-08-28 at 10:12 +0800, Gu Zheng wrote:
> On 08/28/2013 05:38 AM, Toshi Kani wrote:
:
> >>
> >> What about changing device_hotplug_lock and acpi_scan_lock to rwsem? like the
> >> attached one(With a preliminary test, it also can make the splat go away).:)
> >
> > I am curious how msleep(10) & restart_syscall() work in the change
> > below. Doesn't the msleep() make s_active held longer time, which can
> > lead the thread holding device_hotplug_lock to wait it for deletion?
>
> Yes, but it can avoid busy waiting.

I know, but it's kinda unfortunate to sleep with s_active held in this
situation. But I am fine with the 5ms Rafael used in this latest
patchset since this is a rare case anyway.

> > Also, does restart_syscall() release s_active and reopen this file
> > again?
>
> Sure, it just set a TIF_SIGPENDING flag and return an -ERESTARTNOINTR error, s_active/file
> will be released/closed in the failed path. And when do_signal() catches the -ERESTARTNOINTR,
> it will change the regs to restart the syscall.

I see. This is a clever functionality.

Thanks,
-Toshi

2013-08-28 17:08:09

by Toshi Kani

[permalink] [raw]
Subject: Re: [PATCH 0/2] driver core / ACPI: Avoid device removal locking problems

On Wed, 2013-08-28 at 15:45 +0200, Rafael J. Wysocki wrote:
> Hi All,
>
> The following two patches are to address possible deadlocks related to
> device removal and device sysfs attribute access. In short, some device
> sysfs attribute callbacks need to acquire locks that are also held around
> device removal and that may lead to deadlocks with s_active draining in
> sysfs_deactivate().
>
> [1/2] Avoid possible device removal deadlocks related to device_hotplug_lock.
> [2/2] Rework the handling of containers by ACPI hotplug (which makes a possible
> device removal deadlock related to acpi_scan_lock go away).

I like the simplicity of this version. :-) I also agree to remove the
eject_pending flag. For the series:

Acked-by: Toshi Kani <[email protected]>

Thanks,
-Toshi

2013-08-28 18:51:20

by Greg Kroah-Hartman

[permalink] [raw]
Subject: Re: [PATCH 1/2] driver core / ACPI: Avoid device hot remove locking issues

On Wed, Aug 28, 2013 at 03:48:11PM +0200, Rafael J. Wysocki wrote:
> From: Rafael J. Wysocki <[email protected]>
>
> device_hotplug_lock is held around the acpi_bus_trim() call in
> acpi_scan_hot_remove() which generally removes devices (it removes
> ACPI device objects at least, but it may also remove "physical"
> device objects through .detach() callbacks of ACPI scan handlers).
> Thus, potentially, device sysfs attributes are removed under that
> lock and to remove those attributes it is necessary to hold the
> s_active references of their directory entries for writing.
>
> On the other hand, the execution of a .show() or .store() callback
> from a sysfs attribute is carried out with that attribute's s_active
> reference held for reading. Consequently, if any device sysfs
> attribute that may be removed from within acpi_scan_hot_remove()
> through acpi_bus_trim() has a .store() or .show() callback which
> acquires device_hotplug_lock, the execution of that callback may
> deadlock with the removal of the attribute. [Unfortunately, the
> "online" device attribute of CPUs and memory blocks is one of them.]
>
> To avoid such deadlocks, make all of the sysfs attribute callbacks
> that need to lock device hotplug, for example store_online(), use
> a special function, lock_device_hotplug_sysfs(), to lock device
> hotplug and return the result of that function immediately if it is
> not zero. This will cause the s_active reference of the directory
> entry in question to be released and the syscall to be restarted
> if device_hotplug_lock cannot be acquired.
>
> [show_online() actually doesn't need to lock device hotplug, but
> it is useful to serialize it with respect to device_offline() and
> device_online() for the same device (in case user space attempts to
> run them concurrently) which can be done with the help of
> device_lock().]
>
> Signed-off-by: Rafael J. Wysocki <[email protected]>
> Reported-by: Yasuaki Ishimatsu <[email protected]>
> Reported-by: Gu Zheng <[email protected]>

Acked-by: Greg Kroah-Hartman <[email protected]>

2013-08-28 18:51:31

by Greg Kroah-Hartman

[permalink] [raw]
Subject: Re: [PATCH 2/2] ACPI / hotplug: Remove containers synchronously

On Wed, Aug 28, 2013 at 03:51:41PM +0200, Rafael J. Wysocki wrote:
> From: Rafael J. Wysocki <[email protected]>
>
> The current protocol for handling hot remove of containers is very
> fragile and causes acpi_eject_store() to acquire acpi_scan_lock
> which may deadlock with the removal of the device that it is called
> for (the reason is that device sysfs attributes cannot be removed
> while their callbacks are being executed and ACPI device objects
> are removed under acpi_scan_lock).
>
> The problem is related to the fact that containers are handled by
> acpi_bus_device_eject() in a special way, which is to emit an
> offline uevent instead of just removing the container. Then, user
> space is expected to handle that uevent and use the container's
> "eject" attribute to actually remove it. That is fragile, because
> user space may fail to complete the ejection (for example, by not
> using the container's "eject" attribute at all) leaving the BIOS
> kind of in a limbo. Moreover, if the eject event is not signaled
> for a container itself, but for its parent device object (or
> generally, for an ancestor above it in the ACPI namespace), the
> container will be removed straight away without doing that whole
> dance.
>
> For this reason, modify acpi_bus_device_eject() to remove containers
> synchronously like any other objects (user space will get its uevent
> anyway in case it does some other things in response to it) and
> remove the eject_pending ACPI device flag that is not used any more.
> This way acpi_eject_store() doesn't have a reason to acquire
> acpi_scan_lock any more and one possible deadlock scenario goes
> away (plus the code is simplified a bit).
>
> Signed-off-by: Rafael J. Wysocki <[email protected]>
> Reported-by: Gu Zheng <[email protected]>

Acked-by: Greg Kroah-Hartman <[email protected]>

2013-08-29 02:05:07

by Gu Zheng

[permalink] [raw]
Subject: Re: [PATCH 0/2] driver core / ACPI: Avoid device removal locking problems

Hi Rafael,

On 08/28/2013 09:45 PM, Rafael J. Wysocki wrote:

> Hi All,
>
> The following two patches are to address possible deadlocks related to
> device removal and device sysfs attribute access. In short, some device
> sysfs attribute callbacks need to acquire locks that are also held around
> device removal and that may lead to deadlocks with s_active draining in
> sysfs_deactivate().
>
> [1/2] Avoid possible device removal deadlocks related to device_hotplug_lock.
> [2/2] Rework the handling of containers by ACPI hotplug (which makes a possible
> device removal deadlock related to acpi_scan_lock go away).
>

This version is concise and friendly. It works well on latest kernel tree, and all
the splat goes away.:)

Best regards,
Gu

> On top of linux-next, for v3.12.

>
> Thanks,
> Rafael
>
>

2013-08-29 02:06:47

by Gu Zheng

[permalink] [raw]
Subject: Re: [PATCH 1/2] driver core / ACPI: Avoid device hot remove locking issues

On 08/28/2013 09:48 PM, Rafael J. Wysocki wrote:

> From: Rafael J. Wysocki <[email protected]>
>
> device_hotplug_lock is held around the acpi_bus_trim() call in
> acpi_scan_hot_remove() which generally removes devices (it removes
> ACPI device objects at least, but it may also remove "physical"
> device objects through .detach() callbacks of ACPI scan handlers).
> Thus, potentially, device sysfs attributes are removed under that
> lock and to remove those attributes it is necessary to hold the
> s_active references of their directory entries for writing.
>
> On the other hand, the execution of a .show() or .store() callback
> from a sysfs attribute is carried out with that attribute's s_active
> reference held for reading. Consequently, if any device sysfs
> attribute that may be removed from within acpi_scan_hot_remove()
> through acpi_bus_trim() has a .store() or .show() callback which
> acquires device_hotplug_lock, the execution of that callback may
> deadlock with the removal of the attribute. [Unfortunately, the
> "online" device attribute of CPUs and memory blocks is one of them.]
>
> To avoid such deadlocks, make all of the sysfs attribute callbacks
> that need to lock device hotplug, for example store_online(), use
> a special function, lock_device_hotplug_sysfs(), to lock device
> hotplug and return the result of that function immediately if it is
> not zero. This will cause the s_active reference of the directory
> entry in question to be released and the syscall to be restarted
> if device_hotplug_lock cannot be acquired.
>
> [show_online() actually doesn't need to lock device hotplug, but
> it is useful to serialize it with respect to device_offline() and
> device_online() for the same device (in case user space attempts to
> run them concurrently) which can be done with the help of
> device_lock().]
>
> Signed-off-by: Rafael J. Wysocki <[email protected]>
> Reported-by: Yasuaki Ishimatsu <[email protected]>
> Reported-by: Gu Zheng <[email protected]>


Tested-by: Gu Zheng <[email protected]>

> ---
> drivers/acpi/sysfs.c | 5 ++++-
> drivers/base/core.c | 43 ++++++++++++++++++++++++++++---------------
> drivers/base/memory.c | 4 +++-
> include/linux/device.h | 1 +
> 4 files changed, 36 insertions(+), 17 deletions(-)
>
> Index: linux-pm/drivers/base/core.c
> ===================================================================
> --- linux-pm.orig/drivers/base/core.c
> +++ linux-pm/drivers/base/core.c
> @@ -49,6 +49,28 @@ static struct kobject *dev_kobj;
> struct kobject *sysfs_dev_char_kobj;
> struct kobject *sysfs_dev_block_kobj;
>
> +static DEFINE_MUTEX(device_hotplug_lock);
> +
> +void lock_device_hotplug(void)
> +{
> + mutex_lock(&device_hotplug_lock);
> +}
> +
> +void unlock_device_hotplug(void)
> +{
> + mutex_unlock(&device_hotplug_lock);
> +}
> +
> +int lock_device_hotplug_sysfs(void)
> +{
> + if (mutex_trylock(&device_hotplug_lock))
> + return 0;
> +
> + /* Avoid busy looping (5 ms of sleep should do). */
> + msleep(5);
> + return restart_syscall();
> +}
> +
> #ifdef CONFIG_BLOCK
> static inline int device_is_not_partition(struct device *dev)
> {
> @@ -408,9 +430,9 @@ static ssize_t show_online(struct device
> {
> bool val;
>
> - lock_device_hotplug();
> + device_lock(dev);
> val = !dev->offline;
> - unlock_device_hotplug();
> + device_unlock(dev);
> return sprintf(buf, "%u\n", val);
> }
>
> @@ -424,7 +446,10 @@ static ssize_t store_online(struct devic
> if (ret < 0)
> return ret;
>
> - lock_device_hotplug();
> + ret = lock_device_hotplug_sysfs();
> + if (ret)
> + return ret;
> +
> ret = val ? device_online(dev) : device_offline(dev);
> unlock_device_hotplug();
> return ret < 0 ? ret : count;
> @@ -1479,18 +1504,6 @@ EXPORT_SYMBOL_GPL(put_device);
> EXPORT_SYMBOL_GPL(device_create_file);
> EXPORT_SYMBOL_GPL(device_remove_file);
>
> -static DEFINE_MUTEX(device_hotplug_lock);
> -
> -void lock_device_hotplug(void)
> -{
> - mutex_lock(&device_hotplug_lock);
> -}
> -
> -void unlock_device_hotplug(void)
> -{
> - mutex_unlock(&device_hotplug_lock);
> -}
> -
> static int device_check_offline(struct device *dev, void *not_used)
> {
> int ret;
> Index: linux-pm/drivers/base/memory.c
> ===================================================================
> --- linux-pm.orig/drivers/base/memory.c
> +++ linux-pm/drivers/base/memory.c
> @@ -351,7 +351,9 @@ store_mem_state(struct device *dev,
>
> mem = container_of(dev, struct memory_block, dev);
>
> - lock_device_hotplug();
> + ret = lock_device_hotplug_sysfs();
> + if (ret)
> + return ret;
>
> if (!strncmp(buf, "online_kernel", min_t(int, count, 13))) {
> offline = false;
> Index: linux-pm/drivers/acpi/sysfs.c
> ===================================================================
> --- linux-pm.orig/drivers/acpi/sysfs.c
> +++ linux-pm/drivers/acpi/sysfs.c
> @@ -796,7 +796,10 @@ static ssize_t force_remove_store(struct
> if (ret < 0)
> return ret;
>
> - lock_device_hotplug();
> + ret = lock_device_hotplug_sysfs();
> + if (ret)
> + return ret;
> +
> acpi_force_hot_remove = val;
> unlock_device_hotplug();
> return size;
> Index: linux-pm/include/linux/device.h
> ===================================================================
> --- linux-pm.orig/include/linux/device.h
> +++ linux-pm/include/linux/device.h
> @@ -895,6 +895,7 @@ static inline bool device_supports_offli
>
> extern void lock_device_hotplug(void);
> extern void unlock_device_hotplug(void);
> +extern int lock_device_hotplug_sysfs(void);
> extern int device_offline(struct device *dev);
> extern int device_online(struct device *dev);
> /*
>
>

2013-08-29 02:07:11

by Gu Zheng

[permalink] [raw]
Subject: Re: [PATCH 2/2] ACPI / hotplug: Remove containers synchronously

On 08/28/2013 09:51 PM, Rafael J. Wysocki wrote:

> From: Rafael J. Wysocki <[email protected]>
>
> The current protocol for handling hot remove of containers is very
> fragile and causes acpi_eject_store() to acquire acpi_scan_lock
> which may deadlock with the removal of the device that it is called
> for (the reason is that device sysfs attributes cannot be removed
> while their callbacks are being executed and ACPI device objects
> are removed under acpi_scan_lock).
>
> The problem is related to the fact that containers are handled by
> acpi_bus_device_eject() in a special way, which is to emit an
> offline uevent instead of just removing the container. Then, user
> space is expected to handle that uevent and use the container's
> "eject" attribute to actually remove it. That is fragile, because
> user space may fail to complete the ejection (for example, by not
> using the container's "eject" attribute at all) leaving the BIOS
> kind of in a limbo. Moreover, if the eject event is not signaled
> for a container itself, but for its parent device object (or
> generally, for an ancestor above it in the ACPI namespace), the
> container will be removed straight away without doing that whole
> dance.
>
> For this reason, modify acpi_bus_device_eject() to remove containers
> synchronously like any other objects (user space will get its uevent
> anyway in case it does some other things in response to it) and
> remove the eject_pending ACPI device flag that is not used any more.
> This way acpi_eject_store() doesn't have a reason to acquire
> acpi_scan_lock any more and one possible deadlock scenario goes
> away (plus the code is simplified a bit).
>
> Signed-off-by: Rafael J. Wysocki <[email protected]>
> Reported-by: Gu Zheng <[email protected]>


Tested-by: Gu Zheng <[email protected]>

> ---
> drivers/acpi/scan.c | 49 ++++++++++++++----------------------------------
> include/acpi/acpi_bus.h | 3 --
> 2 files changed, 16 insertions(+), 36 deletions(-)
>
> Index: linux-pm/drivers/acpi/scan.c
> ===================================================================
> --- linux-pm.orig/drivers/acpi/scan.c
> +++ linux-pm/drivers/acpi/scan.c
> @@ -287,6 +287,7 @@ static void acpi_bus_device_eject(void *
> struct acpi_device *device = NULL;
> struct acpi_scan_handler *handler;
> u32 ost_code = ACPI_OST_SC_NON_SPECIFIC_FAILURE;
> + int error;
>
> mutex_lock(&acpi_scan_lock);
>
> @@ -301,17 +302,13 @@ static void acpi_bus_device_eject(void *
> }
> acpi_evaluate_hotplug_ost(handle, ACPI_NOTIFY_EJECT_REQUEST,
> ACPI_OST_SC_EJECT_IN_PROGRESS, NULL);
> - if (handler->hotplug.mode == AHM_CONTAINER) {
> - device->flags.eject_pending = true;
> + if (handler->hotplug.mode == AHM_CONTAINER)
> kobject_uevent(&device->dev.kobj, KOBJ_OFFLINE);
> - } else {
> - int error;
>
> - get_device(&device->dev);
> - error = acpi_scan_hot_remove(device);
> - if (error)
> - goto err_out;
> - }
> + get_device(&device->dev);
> + error = acpi_scan_hot_remove(device);
> + if (error)
> + goto err_out;
>
> out:
> mutex_unlock(&acpi_scan_lock);
> @@ -496,7 +493,6 @@ acpi_eject_store(struct device *d, struc
> struct acpi_eject_event *ej_event;
> acpi_object_type not_used;
> acpi_status status;
> - u32 ost_source;
> int ret;
>
> if (!count || buf[0] != '1')
> @@ -510,43 +506,28 @@ acpi_eject_store(struct device *d, struc
> if (ACPI_FAILURE(status) || !acpi_device->flags.ejectable)
> return -ENODEV;
>
> - mutex_lock(&acpi_scan_lock);
> -
> - if (acpi_device->flags.eject_pending) {
> - /* ACPI eject notification event. */
> - ost_source = ACPI_NOTIFY_EJECT_REQUEST;
> - acpi_device->flags.eject_pending = 0;
> - } else {
> - /* Eject initiated by user space. */
> - ost_source = ACPI_OST_EC_OSPM_EJECT;
> - }
> ej_event = kmalloc(sizeof(*ej_event), GFP_KERNEL);
> if (!ej_event) {
> ret = -ENOMEM;
> goto err_out;
> }
> - acpi_evaluate_hotplug_ost(acpi_device->handle, ost_source,
> + acpi_evaluate_hotplug_ost(acpi_device->handle, ACPI_OST_EC_OSPM_EJECT,
> ACPI_OST_SC_EJECT_IN_PROGRESS, NULL);
> ej_event->device = acpi_device;
> - ej_event->event = ost_source;
> + ej_event->event = ACPI_OST_EC_OSPM_EJECT;
> get_device(&acpi_device->dev);
> status = acpi_os_hotplug_execute(acpi_bus_hot_remove_device, ej_event);
> - if (ACPI_FAILURE(status)) {
> - put_device(&acpi_device->dev);
> - kfree(ej_event);
> - ret = status == AE_NO_MEMORY ? -ENOMEM : -EAGAIN;
> - goto err_out;
> - }
> - ret = count;
> + if (ACPI_SUCCESS(status))
> + return count;
>
> - out:
> - mutex_unlock(&acpi_scan_lock);
> - return ret;
> + put_device(&acpi_device->dev);
> + kfree(ej_event);
> + ret = status == AE_NO_MEMORY ? -ENOMEM : -EAGAIN;
>
> err_out:
> - acpi_evaluate_hotplug_ost(acpi_device->handle, ost_source,
> + acpi_evaluate_hotplug_ost(acpi_device->handle, ACPI_OST_EC_OSPM_EJECT,
> ACPI_OST_SC_NON_SPECIFIC_FAILURE, NULL);
> - goto out;
> + return ret;
> }
>
> static DEVICE_ATTR(eject, 0200, NULL, acpi_eject_store);
> Index: linux-pm/include/acpi/acpi_bus.h
> ===================================================================
> --- linux-pm.orig/include/acpi/acpi_bus.h
> +++ linux-pm/include/acpi/acpi_bus.h
> @@ -167,9 +167,8 @@ struct acpi_device_flags {
> u32 removable:1;
> u32 ejectable:1;
> u32 power_manageable:1;
> - u32 eject_pending:1;
> u32 match_driver:1;
> - u32 reserved:26;
> + u32 reserved:27;
> };
>
> /* File System */
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to [email protected]
> More majordomo info at http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at http://www.tux.org/lkml/
>