From: Li Nan <[email protected]>
Changes in v2:
- don't clear MD_CLOSING in md_clean().
- set MD_CLOSING and sync blockdev in array_state_store().
Li Nan (3):
md: Don't clear MD_CLOSING when the raid is about to stop
md: factor out a helper mddev_sync_blockdev() to sync mddev
md: sync blockdev before stopping raid or setting readonly
drivers/md/md.c | 62 ++++++++++++++++++++++++++++++++++++++-----------
1 file changed, 49 insertions(+), 13 deletions(-)
--
2.39.2
From: Li Nan <[email protected]>
The raid should not be opened anymore when it is about to be stopped.
However, other processes can open it again if the flag MD_CLOSING is
cleared before exiting. From now on, this flag will not be cleared when
the raid will be stopped.
Fixes: 065e519e71b2 ("md: MD_CLOSING needs to be cleared after called md_set_readonly or do_md_stop")
Signed-off-by: Li Nan <[email protected]>
---
drivers/md/md.c | 16 +++++++++++++++-
1 file changed, 15 insertions(+), 1 deletion(-)
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 9bdd57324c37..4bf821b89415 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -6254,7 +6254,15 @@ static void md_clean(struct mddev *mddev)
mddev->persistent = 0;
mddev->level = LEVEL_NONE;
mddev->clevel[0] = 0;
- mddev->flags = 0;
+ /*
+ * Don't clear MD_CLOSING, or mddev can be opened again.
+ * 'hold_active != 0' means mddev is still in the creation
+ * process and will be used later.
+ */
+ if (mddev->hold_active)
+ mddev->flags = 0;
+ else
+ mddev->flags &= BIT_ULL_MASK(MD_CLOSING);
mddev->sb_flags = 0;
mddev->ro = MD_RDWR;
mddev->metadata_type[0] = 0;
@@ -7728,6 +7736,12 @@ static int md_ioctl(struct block_device *bdev, blk_mode_t mode,
case STOP_ARRAY:
err = do_md_stop(mddev, 0, bdev);
+ if (!err)
+ /*
+ * mddev has been stopped, keep flag the
+ * MD_CLOSING to prevent reuse.
+ */
+ did_set_md_closing = false;
goto unlock;
case STOP_ARRAY_RO:
--
2.39.2
From: Li Nan <[email protected]>
There are no functional changes, prepare to sync mddev in
array_state_store().
Signed-off-by: Li Nan <[email protected]>
---
drivers/md/md.c | 31 +++++++++++++++++++------------
1 file changed, 19 insertions(+), 12 deletions(-)
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 4bf821b89415..2c793992a604 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -529,6 +529,23 @@ void mddev_resume(struct mddev *mddev)
}
EXPORT_SYMBOL_GPL(mddev_resume);
+/* sync bdev before setting device to readonly or stopping raid*/
+static int mddev_sync_blockdev(struct mddev *mddev) {
+ mutex_lock(&mddev->open_mutex);
+ if (mddev->pers && atomic_read(&mddev->openers) > 1) {
+ mutex_unlock(&mddev->open_mutex);
+ return -EBUSY;
+ }
+ if (test_and_set_bit(MD_CLOSING, &mddev->flags)) {
+ mutex_unlock(&mddev->open_mutex);
+ return -EBUSY;
+ }
+ mutex_unlock(&mddev->open_mutex);
+
+ sync_blockdev(mddev->gendisk->part0);
+ return 0;
+}
+
/*
* Generic flush handling for md
*/
@@ -7681,20 +7698,10 @@ static int md_ioctl(struct block_device *bdev, blk_mode_t mode,
/* Need to flush page cache, and ensure no-one else opens
* and writes
*/
- mutex_lock(&mddev->open_mutex);
- if (mddev->pers && atomic_read(&mddev->openers) > 1) {
- mutex_unlock(&mddev->open_mutex);
- err = -EBUSY;
- goto out;
- }
- if (test_and_set_bit(MD_CLOSING, &mddev->flags)) {
- mutex_unlock(&mddev->open_mutex);
- err = -EBUSY;
+ err = mddev_sync_blockdev(mddev);
+ if (err)
goto out;
- }
did_set_md_closing = true;
- mutex_unlock(&mddev->open_mutex);
- sync_blockdev(bdev);
}
if (!md_is_rdwr(mddev))
--
2.39.2
From: Li Nan <[email protected]>
Commit a05b7ea03d72 ("md: avoid crash when stopping md array races
with closing other open fds.") added sync_block before stopping raid and
setting readonly. Later in commit 260fa034ef7a ("md: avoid deadlock when
dirty buffers during md_stop.") it is moved to ioctl. array_state_store()
was ignored. Add sync blockdev to array_state_store() now.
Signed-off-by: Li Nan <[email protected]>
---
drivers/md/md.c | 15 +++++++++++++++
1 file changed, 15 insertions(+)
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 2c793992a604..aea39598457c 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -4477,6 +4477,7 @@ array_state_store(struct mddev *mddev, const char *buf, size_t len)
{
int err = 0;
enum array_state st = match_word(buf, array_states);
+ bool clear_md_closing = false;
/* No lock dependent actions */
switch (st) {
@@ -4511,6 +4512,16 @@ array_state_store(struct mddev *mddev, const char *buf, size_t len)
spin_unlock(&mddev->lock);
return err ?: len;
}
+
+ /* we will call set readonly or stop raid, sync blockdev */
+ if (st == clear || (mddev->pers && (st == readonly ||
+ st == inactive || (st == read_auto && md_is_rdwr(mddev))))) {
+ err = mddev_sync_blockdev(mddev);
+ if (err)
+ return err;
+ clear_md_closing = true;
+ }
+
err = mddev_lock(mddev);
if (err)
return err;
@@ -4523,6 +4534,8 @@ array_state_store(struct mddev *mddev, const char *buf, size_t len)
break;
case clear:
err = do_md_stop(mddev, 0, NULL);
+ if (!err)
+ clear_md_closing = false;
break;
case readonly:
if (mddev->pers)
@@ -4585,6 +4598,8 @@ array_state_store(struct mddev *mddev, const char *buf, size_t len)
sysfs_notify_dirent_safe(mddev->sysfs_state);
}
mddev_unlock(mddev);
+ if (clear_md_closing)
+ clear_bit(MD_CLOSING, &mddev->flags);
return err ?: len;
}
static struct md_sysfs_entry md_array_state =
--
2.39.2
On Wed, 17 Jan 2024 17:37:06 +0800
[email protected] wrote:
> From: Li Nan <[email protected]>
>
> There are no functional changes, prepare to sync mddev in
> array_state_store().
>
> Signed-off-by: Li Nan <[email protected]>
> ---
> drivers/md/md.c | 31 +++++++++++++++++++------------
> 1 file changed, 19 insertions(+), 12 deletions(-)
>
> diff --git a/drivers/md/md.c b/drivers/md/md.c
> index 4bf821b89415..2c793992a604 100644
> --- a/drivers/md/md.c
> +++ b/drivers/md/md.c
> @@ -529,6 +529,23 @@ void mddev_resume(struct mddev *mddev)
> }
> EXPORT_SYMBOL_GPL(mddev_resume);
>
> +/* sync bdev before setting device to readonly or stopping raid*/
> +static int mddev_sync_blockdev(struct mddev *mddev) {
Please add something about MD_CLOSING to the function name. Comment is good but
you need open function to get it. Something like:
mddev_set_closing_and_sync_blockdev() is more reader friendly.
Anyway, LGTM.
Acked-by: Mariusz Tkaczyk <[email protected]>
Thanks,
Mariusz
On Wed, 17 Jan 2024 17:37:05 +0800
[email protected] wrote:
> From: Li Nan <[email protected]>
>
> The raid should not be opened anymore when it is about to be stopped.
> However, other processes can open it again if the flag MD_CLOSING is
> cleared before exiting. From now on, this flag will not be cleared when
> the raid will be stopped.
>
> Fixes: 065e519e71b2 ("md: MD_CLOSING needs to be cleared after called
> md_set_readonly or do_md_stop") Signed-off-by: Li Nan <[email protected]>
> ---
> drivers/md/md.c | 16 +++++++++++++++-
> 1 file changed, 15 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/md/md.c b/drivers/md/md.c
> index 9bdd57324c37..4bf821b89415 100644
> --- a/drivers/md/md.c
> +++ b/drivers/md/md.c
> @@ -6254,7 +6254,15 @@ static void md_clean(struct mddev *mddev)
> mddev->persistent = 0;
> mddev->level = LEVEL_NONE;
> mddev->clevel[0] = 0;
> - mddev->flags = 0;
> + /*
> + * Don't clear MD_CLOSING, or mddev can be opened again.
> + * 'hold_active != 0' means mddev is still in the creation
> + * process and will be used later.
> + */
> + if (mddev->hold_active)
> + mddev->flags = 0;
> + else
> + mddev->flags &= BIT_ULL_MASK(MD_CLOSING);
> mddev->sb_flags = 0;
> mddev->ro = MD_RDWR;
> mddev->metadata_type[0] = 0;
> @@ -7728,6 +7736,12 @@ static int md_ioctl(struct block_device *bdev,
> blk_mode_t mode,
> case STOP_ARRAY:
> err = do_md_stop(mddev, 0, bdev);
> + if (!err)
> + /*
> + * mddev has been stopped, keep flag the
> + * MD_CLOSING to prevent reuse.
> + */
> + did_set_md_closing = false;
Hello Nan,
The meaning of the "did_set_md_closing" is to notify that MD_CLOSING was set in
this function, to know how to behave on error.
You gave it another meaning "Do not clear MD_CLOSING because we want it to stay"
Please consider how to solve this confusion. I see the comment you added but I
think we can have this solved better, maybe just name it as "clear_md_closing"?
Anyway it looks acceptable to me:
Acked-by: Mariusz Tkaczyk <[email protected]>
Thanks,
Mariusz
On Wed, 17 Jan 2024 17:37:07 +0800
[email protected] wrote:
> From: Li Nan <[email protected]>
>
> Commit a05b7ea03d72 ("md: avoid crash when stopping md array races
> with closing other open fds.") added sync_block before stopping raid and
> setting readonly. Later in commit 260fa034ef7a ("md: avoid deadlock when
> dirty buffers during md_stop.") it is moved to ioctl. array_state_store()
> was ignored. Add sync blockdev to array_state_store() now.
>
> Signed-off-by: Li Nan <[email protected]>
> ---
> drivers/md/md.c | 15 +++++++++++++++
> 1 file changed, 15 insertions(+)
>
> diff --git a/drivers/md/md.c b/drivers/md/md.c
> index 2c793992a604..aea39598457c 100644
> --- a/drivers/md/md.c
> +++ b/drivers/md/md.c
> @@ -4477,6 +4477,7 @@ array_state_store(struct mddev *mddev, const char *buf,
> size_t len) {
> int err = 0;
> enum array_state st = match_word(buf, array_states);
> + bool clear_md_closing = false;
>
> /* No lock dependent actions */
> switch (st) {
> @@ -4511,6 +4512,16 @@ array_state_store(struct mddev *mddev, const char
> *buf, size_t len) spin_unlock(&mddev->lock);
> return err ?: len;
> }
> +
> + /* we will call set readonly or stop raid, sync blockdev */
> + if (st == clear || (mddev->pers && (st == readonly ||
> + st == inactive || (st == read_auto && md_is_rdwr(mddev))))) {
> + err = mddev_sync_blockdev(mddev);
> + if (err)
> + return err;
> + clear_md_closing = true;
> + }
> +
Please reorganize it a little for readability:
I think if no mddev->pers we don't need to consider sync_blockdev at all. If
personality is there we can probably check for read-write. If it is not
read-write then nothing to sync. What about that:
if (mddev->pers && md_is_rdwr(mddev) &&
(st == clear || st == readonly || st == inactive || st == read_auto))
Please note that I didn't test it so please let me know if you see issue in
proposed logic.
I think that we may be able to include it in "/* No lock dependent actions */"
switch. Please consider it too:
case clear:
case readonly:
case inactive:
case read_auto:
if(!mddev->pers || !md_is_rdwr(mddev))
break;
err = mddev_sync_blockdev(mddev);
if (err)
return err;
clear_md_closing = true;
> err = mddev_lock(mddev);
> if (err)
> return err;
> @@ -4523,6 +4534,8 @@ array_state_store(struct mddev *mddev, const char *buf,
> size_t len) break;
> case clear:
> err = do_md_stop(mddev, 0, NULL);
> + if (!err)
> + clear_md_closing = false;
> break;
> case readonly:
> if (mddev->pers)
> @@ -4585,6 +4598,8 @@ array_state_store(struct mddev *mddev, const char *buf,
> size_t len) sysfs_notify_dirent_safe(mddev->sysfs_state);
> }
> mddev_unlock(mddev);
> + if (clear_md_closing)
> + clear_bit(MD_CLOSING, &mddev->flags);
Please add spaces before and after if.
> return err ?: len;
> }
> static struct md_sysfs_entry md_array_state =
Thanks,
Mariusz
在 2024/1/18 16:08, Mariusz Tkaczyk 写道:
> On Wed, 17 Jan 2024 17:37:06 +0800
> [email protected] wrote:
>
>> From: Li Nan <[email protected]>
>>
>> There are no functional changes, prepare to sync mddev in
>> array_state_store().
>>
>> Signed-off-by: Li Nan <[email protected]>
>> ---
>> drivers/md/md.c | 31 +++++++++++++++++++------------
>> 1 file changed, 19 insertions(+), 12 deletions(-)
>>
>> diff --git a/drivers/md/md.c b/drivers/md/md.c
>> index 4bf821b89415..2c793992a604 100644
>> --- a/drivers/md/md.c
>> +++ b/drivers/md/md.c
>> @@ -529,6 +529,23 @@ void mddev_resume(struct mddev *mddev)
>> }
>> EXPORT_SYMBOL_GPL(mddev_resume);
>>
>> +/* sync bdev before setting device to readonly or stopping raid*/
>> +static int mddev_sync_blockdev(struct mddev *mddev) {
>
> Please add something about MD_CLOSING to the function name. Comment is good but
> you need open function to get it. Something like:
> mddev_set_closing_and_sync_blockdev() is more reader friendly.
>
I agree. Let me improve this.
> Anyway, LGTM.
> Acked-by: Mariusz Tkaczyk <[email protected]>
>
> Thanks,
> Mariusz
>
> .
--
Thanks,
Nan
在 2024/1/18 15:35, Mariusz Tkaczyk 写道:
> On Wed, 17 Jan 2024 17:37:05 +0800
> [email protected] wrote:
>
>> From: Li Nan <[email protected]>
>>
>> The raid should not be opened anymore when it is about to be stopped.
>> However, other processes can open it again if the flag MD_CLOSING is
>> cleared before exiting. From now on, this flag will not be cleared when
>> the raid will be stopped.
>>
>> Fixes: 065e519e71b2 ("md: MD_CLOSING needs to be cleared after called
>> md_set_readonly or do_md_stop") Signed-off-by: Li Nan <[email protected]>
>> ---
>> drivers/md/md.c | 16 +++++++++++++++-
>> 1 file changed, 15 insertions(+), 1 deletion(-)
>>
>> diff --git a/drivers/md/md.c b/drivers/md/md.c
>> index 9bdd57324c37..4bf821b89415 100644
>> --- a/drivers/md/md.c
>> +++ b/drivers/md/md.c
>> @@ -6254,7 +6254,15 @@ static void md_clean(struct mddev *mddev)
>> mddev->persistent = 0;
>> mddev->level = LEVEL_NONE;
>> mddev->clevel[0] = 0;
>> - mddev->flags = 0;
>> + /*
>> + * Don't clear MD_CLOSING, or mddev can be opened again.
>> + * 'hold_active != 0' means mddev is still in the creation
>> + * process and will be used later.
>> + */
>> + if (mddev->hold_active)
>> + mddev->flags = 0;
>> + else
>> + mddev->flags &= BIT_ULL_MASK(MD_CLOSING);
>> mddev->sb_flags = 0;
>> mddev->ro = MD_RDWR;
>> mddev->metadata_type[0] = 0;
>> @@ -7728,6 +7736,12 @@ static int md_ioctl(struct block_device *bdev,
>> blk_mode_t mode,
>> case STOP_ARRAY:
>> err = do_md_stop(mddev, 0, bdev);
>> + if (!err)
>> + /*
>> + * mddev has been stopped, keep flag the
>> + * MD_CLOSING to prevent reuse.
>> + */
>> + did_set_md_closing = false;
>
> Hello Nan,
> The meaning of the "did_set_md_closing" is to notify that MD_CLOSING was set in
> this function, to know how to behave on error.
> You gave it another meaning "Do not clear MD_CLOSING because we want it to stay"
> Please consider how to solve this confusion. I see the comment you added but I
> think we can have this solved better, maybe just name it as "clear_md_closing"?
>
Thanks for your review, I will rename it in next version.
> Anyway it looks acceptable to me:
> Acked-by: Mariusz Tkaczyk <[email protected]>
>
> Thanks,
> Mariusz
>
> .
--
Thanks,
Nan
在 2024/1/18 16:02, Mariusz Tkaczyk 写道:
> On Wed, 17 Jan 2024 17:37:07 +0800
> [email protected] wrote:
>
>> From: Li Nan <[email protected]>
>>
>> Commit a05b7ea03d72 ("md: avoid crash when stopping md array races
>> with closing other open fds.") added sync_block before stopping raid and
>> setting readonly. Later in commit 260fa034ef7a ("md: avoid deadlock when
>> dirty buffers during md_stop.") it is moved to ioctl. array_state_store()
>> was ignored. Add sync blockdev to array_state_store() now.
>>
>> Signed-off-by: Li Nan <[email protected]>
[...]
>> +
>> + /* we will call set readonly or stop raid, sync blockdev */
>> + if (st == clear || (mddev->pers && (st == readonly ||
>> + st == inactive || (st == read_auto && md_is_rdwr(mddev))))) {
>> + err = mddev_sync_blockdev(mddev);
>> + if (err)
>> + return err;
>> + clear_md_closing = true;
>> + }
>> +
>
> Please reorganize it a little for readability:
> I think if no mddev->pers we don't need to consider sync_blockdev at all. If
> personality is there we can probably check for read-write. If it is not
> read-write then nothing to sync. What about that:
>
> if (mddev->pers && md_is_rdwr(mddev) &&
> (st == clear || st == readonly || st == inactive || st == read_auto))
>
> Please note that I didn't test it so please let me know if you see issue in
> proposed logic.
> I think that we may be able to include it in "/* No lock dependent actions */"
> switch. Please consider it too:
>
Thanks for your review.
It is a really good idea. I will test and improve it.
> case clear:
> case readonly:
> case inactive:
> case read_auto:
> if(!mddev->pers || !md_is_rdwr(mddev))
> break;
> err = mddev_sync_blockdev(mddev);
> if (err)
> return err;
> clear_md_closing = true;
>
--
Thanks,
Nan