From: Yu Kuai <[email protected]>
Is is reported that for dm-raid10, lvextend + lvchange --syncaction will
trigger following softlockup:
kernel:watchdog: BUG: soft lockup - CPU#3 stuck for 26s! [mdX_resync:6976]
CPU: 7 PID: 3588 Comm: mdX_resync Kdump: loaded Not tainted 6.9.0-rc4-next-20240419 #1
RIP: 0010:_raw_spin_unlock_irq+0x13/0x30
Call Trace:
<TASK>
md_bitmap_start_sync+0x6b/0xf0
raid10_sync_request+0x25c/0x1b40 [raid10]
md_do_sync+0x64b/0x1020
md_thread+0xa7/0x170
kthread+0xcf/0x100
ret_from_fork+0x30/0x50
ret_from_fork_asm+0x1a/0x30
And the detailed process is as follows:
md_do_sync
j = mddev->resync_min
while (j < max_sectors)
sectors = raid10_sync_request(mddev, j, &skipped)
if (!md_bitmap_start_sync(..., &sync_blocks))
// md_bitmap_start_sync set sync_blocks to 0
return sync_blocks + sectors_skippe;
// sectors = 0;
j += sectors;
// j never change
Root cause is that commit 301867b1c168 ("md/raid10: check
slab-out-of-bounds in md_bitmap_get_counter") return early from
md_bitmap_get_counter(), without setting returned blocks.
Fix this problem by always set returned blocks from
md_bitmap_get_counter"(), as it used to be.
Noted that this patch just fix the softlockup problem in kernel, the
case that bitmap size doesn't match array size still need to be fixed.
Fixes: 301867b1c168 ("md/raid10: check slab-out-of-bounds in md_bitmap_get_counter")
Reported-and-tested-by: Nigel Croxon <[email protected]>
Closes: https://lore.kernel.org/all/[email protected]/
Signed-off-by: Yu Kuai <[email protected]>
---
drivers/md/md-bitmap.c | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c
index 059afc24c08b..f5b66d52cbe3 100644
--- a/drivers/md/md-bitmap.c
+++ b/drivers/md/md-bitmap.c
@@ -1424,15 +1424,17 @@ __acquires(bitmap->lock)
sector_t chunk = offset >> bitmap->chunkshift;
unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
unsigned long pageoff = (chunk & PAGE_COUNTER_MASK) << COUNTER_BYTE_SHIFT;
- sector_t csize;
+ sector_t csize = ((sector_t)1) << bitmap->chunkshift;
int err;
+
if (page >= bitmap->pages) {
/*
* This can happen if bitmap_start_sync goes beyond
* End-of-device while looking for a whole page or
* user set a huge number to sysfs bitmap_set_bits.
*/
+ *blocks = csize - (offset & (csize - 1));
return NULL;
}
err = md_bitmap_checkpage(bitmap, page, create, 0);
@@ -1441,8 +1443,7 @@ __acquires(bitmap->lock)
bitmap->bp[page].map == NULL)
csize = ((sector_t)1) << (bitmap->chunkshift +
PAGE_COUNTER_SHIFT);
- else
- csize = ((sector_t)1) << bitmap->chunkshift;
+
*blocks = csize - (offset & (csize - 1));
if (err < 0)
--
2.39.2
+CC Nigel
Sorry I forgot that.
?? 2024/04/22 14:58, Yu Kuai д??:
> From: Yu Kuai <[email protected]>
>
> Is is reported that for dm-raid10, lvextend + lvchange --syncaction will
> trigger following softlockup:
>
> kernel:watchdog: BUG: soft lockup - CPU#3 stuck for 26s! [mdX_resync:6976]
> CPU: 7 PID: 3588 Comm: mdX_resync Kdump: loaded Not tainted 6.9.0-rc4-next-20240419 #1
> RIP: 0010:_raw_spin_unlock_irq+0x13/0x30
> Call Trace:
> <TASK>
> md_bitmap_start_sync+0x6b/0xf0
> raid10_sync_request+0x25c/0x1b40 [raid10]
> md_do_sync+0x64b/0x1020
> md_thread+0xa7/0x170
> kthread+0xcf/0x100
> ret_from_fork+0x30/0x50
> ret_from_fork_asm+0x1a/0x30
>
> And the detailed process is as follows:
>
> md_do_sync
> j = mddev->resync_min
> while (j < max_sectors)
> sectors = raid10_sync_request(mddev, j, &skipped)
> if (!md_bitmap_start_sync(..., &sync_blocks))
> // md_bitmap_start_sync set sync_blocks to 0
> return sync_blocks + sectors_skippe;
> // sectors = 0;
> j += sectors;
> // j never change
>
> Root cause is that commit 301867b1c168 ("md/raid10: check
> slab-out-of-bounds in md_bitmap_get_counter") return early from
> md_bitmap_get_counter(), without setting returned blocks.
>
> Fix this problem by always set returned blocks from
> md_bitmap_get_counter"(), as it used to be.
>
> Noted that this patch just fix the softlockup problem in kernel, the
> case that bitmap size doesn't match array size still need to be fixed.
>
> Fixes: 301867b1c168 ("md/raid10: check slab-out-of-bounds in md_bitmap_get_counter")
> Reported-and-tested-by: Nigel Croxon <[email protected]>
> Closes: https://lore.kernel.org/all/[email protected]/
> Signed-off-by: Yu Kuai <[email protected]>
> ---
> drivers/md/md-bitmap.c | 7 ++++---
> 1 file changed, 4 insertions(+), 3 deletions(-)
>
> diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c
> index 059afc24c08b..f5b66d52cbe3 100644
> --- a/drivers/md/md-bitmap.c
> +++ b/drivers/md/md-bitmap.c
> @@ -1424,15 +1424,17 @@ __acquires(bitmap->lock)
> sector_t chunk = offset >> bitmap->chunkshift;
> unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
> unsigned long pageoff = (chunk & PAGE_COUNTER_MASK) << COUNTER_BYTE_SHIFT;
> - sector_t csize;
> + sector_t csize = ((sector_t)1) << bitmap->chunkshift;
> int err;
>
> +
> if (page >= bitmap->pages) {
> /*
> * This can happen if bitmap_start_sync goes beyond
> * End-of-device while looking for a whole page or
> * user set a huge number to sysfs bitmap_set_bits.
> */
> + *blocks = csize - (offset & (csize - 1));
> return NULL;
> }
> err = md_bitmap_checkpage(bitmap, page, create, 0);
> @@ -1441,8 +1443,7 @@ __acquires(bitmap->lock)
> bitmap->bp[page].map == NULL)
> csize = ((sector_t)1) << (bitmap->chunkshift +
> PAGE_COUNTER_SHIFT);
> - else
> - csize = ((sector_t)1) << bitmap->chunkshift;
> +
> *blocks = csize - (offset & (csize - 1));
>
> if (err < 0)
>
On Mon, Apr 22, 2024 at 12:07 AM Yu Kuai <[email protected]> wrote:
>
> From: Yu Kuai <[email protected]>
>
> Is is reported that for dm-raid10, lvextend + lvchange --syncaction will
> trigger following softlockup:
>
> kernel:watchdog: BUG: soft lockup - CPU#3 stuck for 26s! [mdX_resync:6976]
> CPU: 7 PID: 3588 Comm: mdX_resync Kdump: loaded Not tainted 6.9.0-rc4-next-20240419 #1
> RIP: 0010:_raw_spin_unlock_irq+0x13/0x30
> Call Trace:
> <TASK>
> md_bitmap_start_sync+0x6b/0xf0
> raid10_sync_request+0x25c/0x1b40 [raid10]
> md_do_sync+0x64b/0x1020
> md_thread+0xa7/0x170
> kthread+0xcf/0x100
> ret_from_fork+0x30/0x50
> ret_from_fork_asm+0x1a/0x30
>
> And the detailed process is as follows:
>
> md_do_sync
> j = mddev->resync_min
> while (j < max_sectors)
> sectors = raid10_sync_request(mddev, j, &skipped)
> if (!md_bitmap_start_sync(..., &sync_blocks))
> // md_bitmap_start_sync set sync_blocks to 0
> return sync_blocks + sectors_skippe;
> // sectors = 0;
> j += sectors;
> // j never change
>
> Root cause is that commit 301867b1c168 ("md/raid10: check
> slab-out-of-bounds in md_bitmap_get_counter") return early from
> md_bitmap_get_counter(), without setting returned blocks.
>
> Fix this problem by always set returned blocks from
> md_bitmap_get_counter"(), as it used to be.
>
> Noted that this patch just fix the softlockup problem in kernel, the
> case that bitmap size doesn't match array size still need to be fixed.
>
> Fixes: 301867b1c168 ("md/raid10: check slab-out-of-bounds in md_bitmap_get_counter")
> Reported-and-tested-by: Nigel Croxon <[email protected]>
> Closes: https://lore.kernel.org/all/[email protected]/
> Signed-off-by: Yu Kuai <[email protected]>
Applied to md-6.10. Thanks!
Song