It is trivial to trigger a WARN_ON_ONCE(1) in iomap_dio_actor() by
unprivileged users which would taint the kernel, or worse - panic if
panic_on_warn or panic_on_taint is set. Hence, just convert it to
pr_warn_ratelimited() to let users know their workloads are racing.
Thanks Dave Chinner for the initial analysis of the racing reproducers.
Signed-off-by: Qian Cai <[email protected]>
---
v2: Record the path, pid and command as well.
fs/iomap/direct-io.c | 17 ++++++++++++++++-
1 file changed, 16 insertions(+), 1 deletion(-)
diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c
index c1aafb2ab990..66a4502ef675 100644
--- a/fs/iomap/direct-io.c
+++ b/fs/iomap/direct-io.c
@@ -374,6 +374,7 @@ iomap_dio_actor(struct inode *inode, loff_t pos, loff_t length,
void *data, struct iomap *iomap, struct iomap *srcmap)
{
struct iomap_dio *dio = data;
+ char pathname[128], *path;
switch (iomap->type) {
case IOMAP_HOLE:
@@ -389,7 +390,21 @@ iomap_dio_actor(struct inode *inode, loff_t pos, loff_t length,
case IOMAP_INLINE:
return iomap_dio_inline_actor(inode, pos, length, dio, iomap);
default:
- WARN_ON_ONCE(1);
+ /*
+ * DIO is not serialised against mmap() access at all, and so
+ * if the page_mkwrite occurs between the writeback and the
+ * iomap_apply() call in the DIO path, then it will see the
+ * DELALLOC block that the page-mkwrite allocated.
+ */
+ path = file_path(dio->iocb->ki_filp, pathname,
+ sizeof(pathname));
+ if (IS_ERR(path))
+ path = "(unknown)";
+
+ pr_warn_ratelimited("page_mkwrite() is racing with DIO read (iomap->type = %u).\n"
+ "File: %s PID: %d Comm: %.20s\n",
+ iomap->type, path, current->pid,
+ current->comm);
return -EIO;
}
}
--
2.18.4
On Mon, Aug 31, 2020 at 10:48:59AM -0500, Eric Sandeen wrote:
> On 8/30/20 8:45 PM, Qian Cai wrote:
> > It is trivial to trigger a WARN_ON_ONCE(1) in iomap_dio_actor() by
> > unprivileged users which would taint the kernel, or worse - panic if
> > panic_on_warn or panic_on_taint is set. Hence, just convert it to
> > pr_warn_ratelimited() to let users know their workloads are racing.
> > Thanks Dave Chinner for the initial analysis of the racing reproducers.
> >
> > Signed-off-by: Qian Cai <[email protected]>
> > ---
> >
> > v2: Record the path, pid and command as well.
> >
> > fs/iomap/direct-io.c | 17 ++++++++++++++++-
> > 1 file changed, 16 insertions(+), 1 deletion(-)
> >
> > diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c
> > index c1aafb2ab990..66a4502ef675 100644
> > --- a/fs/iomap/direct-io.c
> > +++ b/fs/iomap/direct-io.c
> > @@ -374,6 +374,7 @@ iomap_dio_actor(struct inode *inode, loff_t pos, loff_t length,
> > void *data, struct iomap *iomap, struct iomap *srcmap)
> > {
> > struct iomap_dio *dio = data;
> > + char pathname[128], *path;
> >
> > switch (iomap->type) {
> > case IOMAP_HOLE:
> > @@ -389,7 +390,21 @@ iomap_dio_actor(struct inode *inode, loff_t pos, loff_t length,
> > case IOMAP_INLINE:
> > return iomap_dio_inline_actor(inode, pos, length, dio, iomap);
> > default:
> > - WARN_ON_ONCE(1);
>
> It seems like we should explicitly catch IOMAP_DELALLOC for this case, and leave the
> default: as a WARN_ON that is not user-triggerable? i.e.
>
> case IOMAP_DELALLOC:
> <all the fancy warnings>
> return -EIO;
> default:
> WARN_ON_ONCE(1);
> return -EIO;
>
> > + /*
> > + * DIO is not serialised against mmap() access at all, and so
> > + * if the page_mkwrite occurs between the writeback and the
> > + * iomap_apply() call in the DIO path, then it will see the
> > + * DELALLOC block that the page-mkwrite allocated.
> > + */
> > + path = file_path(dio->iocb->ki_filp, pathname,
> > + sizeof(pathname));
> > + if (IS_ERR(path))
> > + path = "(unknown)";
> > +
> > + pr_warn_ratelimited("page_mkwrite() is racing with DIO read (iomap->type = %u).\n"
> > + "File: %s PID: %d Comm: %.20s\n",
> > + iomap->type, path, current->pid,
> > + current->comm);
>
> This is very specific ...
>
> Do we know that mmap/page_mkwrite is (and will always be) the only way to reach this
> point?
>
> It seems to me that this message won't be very useful for the admin; "pg_mkwrite" may
> mean something to us, but doubtful for the general public. And "type = 1" won't mean
> much to the reader, either.
>
> Maybe something like:
>
> "DIO encountered delayed allocation block, racing buffered+direct? File: %s Comm: %.20s\n"
>
> It just seems that a user-facing warning should be something the admin has a chance of
> acting on without needing to file a bug for analysis by the developers.
>
> (though TBH "delayed allocation" probably doesn't mean much to the admin, either)
/me suggests
"Direct I/O collision with buffered write! File: %s..."?
I concede that we ought to leave the nastier WARN for the default
case since there are no other IOMAP_ types and so any other code is
a sign of a serious screwup.
--D
>
> -Eric
>
> > return -EIO;
> > }
> > }
> >
On Mon, Aug 31, 2020 at 10:48:59AM -0500, Eric Sandeen wrote:
> On 8/30/20 8:45 PM, Qian Cai wrote:
> > It is trivial to trigger a WARN_ON_ONCE(1) in iomap_dio_actor() by
> > unprivileged users which would taint the kernel, or worse - panic if
> > panic_on_warn or panic_on_taint is set. Hence, just convert it to
> > pr_warn_ratelimited() to let users know their workloads are racing.
> > Thanks Dave Chinner for the initial analysis of the racing reproducers.
> >
> > Signed-off-by: Qian Cai <[email protected]>
> > ---
> >
> > v2: Record the path, pid and command as well.
> >
> > fs/iomap/direct-io.c | 17 ++++++++++++++++-
> > 1 file changed, 16 insertions(+), 1 deletion(-)
> >
> > diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c
> > index c1aafb2ab990..66a4502ef675 100644
> > --- a/fs/iomap/direct-io.c
> > +++ b/fs/iomap/direct-io.c
> > @@ -374,6 +374,7 @@ iomap_dio_actor(struct inode *inode, loff_t pos, loff_t length,
> > void *data, struct iomap *iomap, struct iomap *srcmap)
> > {
> > struct iomap_dio *dio = data;
> > + char pathname[128], *path;
> >
> > switch (iomap->type) {
> > case IOMAP_HOLE:
> > @@ -389,7 +390,21 @@ iomap_dio_actor(struct inode *inode, loff_t pos, loff_t length,
> > case IOMAP_INLINE:
> > return iomap_dio_inline_actor(inode, pos, length, dio, iomap);
> > default:
> > - WARN_ON_ONCE(1);
>
> It seems like we should explicitly catch IOMAP_DELALLOC for this case, and leave the
> default: as a WARN_ON that is not user-triggerable? i.e.
>
> case IOMAP_DELALLOC:
> <all the fancy warnings>
> return -EIO;
> default:
> WARN_ON_ONCE(1);
> return -EIO;
>
> > + /*
> > + * DIO is not serialised against mmap() access at all, and so
> > + * if the page_mkwrite occurs between the writeback and the
> > + * iomap_apply() call in the DIO path, then it will see the
> > + * DELALLOC block that the page-mkwrite allocated.
> > + */
> > + path = file_path(dio->iocb->ki_filp, pathname,
> > + sizeof(pathname));
> > + if (IS_ERR(path))
> > + path = "(unknown)";
> > +
> > + pr_warn_ratelimited("page_mkwrite() is racing with DIO read (iomap->type = %u).\n"
> > + "File: %s PID: %d Comm: %.20s\n",
> > + iomap->type, path, current->pid,
> > + current->comm);
>
> This is very specific ...
>
> Do we know that mmap/page_mkwrite is (and will always be) the only way to reach this
> point?
I don't know, so this could indeed be a bit misleading.
>
> It seems to me that this message won't be very useful for the admin; "pg_mkwrite" may
> mean something to us, but doubtful for the general public. And "type = 1" won't mean
> much to the reader, either.
>
> Maybe something like:
>
> "DIO encountered delayed allocation block, racing buffered+direct? File: %s Comm: %.20s\n"
>
> It just seems that a user-facing warning should be something the admin has a chance of
> acting on without needing to file a bug for analysis by the developers.
>
> (though TBH "delayed allocation" probably doesn't mean much to the admin, either)
I agree with your suggestions. I'll submit a new version.
On 8/31/20 10:56 AM, Darrick J. Wong wrote:
> On Mon, Aug 31, 2020 at 10:48:59AM -0500, Eric Sandeen wrote:
>> On 8/30/20 8:45 PM, Qian Cai wrote:
>>> It is trivial to trigger a WARN_ON_ONCE(1) in iomap_dio_actor() by
>>> unprivileged users which would taint the kernel, or worse - panic if
>>> panic_on_warn or panic_on_taint is set. Hence, just convert it to
>>> pr_warn_ratelimited() to let users know their workloads are racing.
>>> Thanks Dave Chinner for the initial analysis of the racing reproducers.
>>>
>>> Signed-off-by: Qian Cai <[email protected]>
>>> ---
>>>
>>> v2: Record the path, pid and command as well.
>>>
>>> fs/iomap/direct-io.c | 17 ++++++++++++++++-
>>> 1 file changed, 16 insertions(+), 1 deletion(-)
>>>
>>> diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c
>>> index c1aafb2ab990..66a4502ef675 100644
>>> --- a/fs/iomap/direct-io.c
>>> +++ b/fs/iomap/direct-io.c
>>> @@ -374,6 +374,7 @@ iomap_dio_actor(struct inode *inode, loff_t pos, loff_t length,
>>> void *data, struct iomap *iomap, struct iomap *srcmap)
>>> {
>>> struct iomap_dio *dio = data;
>>> + char pathname[128], *path;
>>>
>>> switch (iomap->type) {
>>> case IOMAP_HOLE:
>>> @@ -389,7 +390,21 @@ iomap_dio_actor(struct inode *inode, loff_t pos, loff_t length,
>>> case IOMAP_INLINE:
>>> return iomap_dio_inline_actor(inode, pos, length, dio, iomap);
>>> default:
>>> - WARN_ON_ONCE(1);
>>
>> It seems like we should explicitly catch IOMAP_DELALLOC for this case, and leave the
>> default: as a WARN_ON that is not user-triggerable? i.e.
>>
>> case IOMAP_DELALLOC:
>> <all the fancy warnings>
>> return -EIO;
>> default:
>> WARN_ON_ONCE(1);
>> return -EIO;
>>
>>> + /*
>>> + * DIO is not serialised against mmap() access at all, and so
>>> + * if the page_mkwrite occurs between the writeback and the
>>> + * iomap_apply() call in the DIO path, then it will see the
>>> + * DELALLOC block that the page-mkwrite allocated.
>>> + */
>>> + path = file_path(dio->iocb->ki_filp, pathname,
>>> + sizeof(pathname));
>>> + if (IS_ERR(path))
>>> + path = "(unknown)";
>>> +
>>> + pr_warn_ratelimited("page_mkwrite() is racing with DIO read (iomap->type = %u).\n"
>>> + "File: %s PID: %d Comm: %.20s\n",
>>> + iomap->type, path, current->pid,
>>> + current->comm);
>>
>> This is very specific ...
>>
>> Do we know that mmap/page_mkwrite is (and will always be) the only way to reach this
>> point?
>>
>> It seems to me that this message won't be very useful for the admin; "pg_mkwrite" may
>> mean something to us, but doubtful for the general public. And "type = 1" won't mean
>> much to the reader, either.
>>
>> Maybe something like:
>>
>> "DIO encountered delayed allocation block, racing buffered+direct? File: %s Comm: %.20s\n"
>>
>> It just seems that a user-facing warning should be something the admin has a chance of
>> acting on without needing to file a bug for analysis by the developers.
>>
>> (though TBH "delayed allocation" probably doesn't mean much to the admin, either)
>
> /me suggests
>
> "Direct I/O collision with buffered write! File: %s..."?
Sure, that sounds good to me. Terser is better.
> I concede that we ought to leave the nastier WARN for the default
> case since there are no other IOMAP_ types and so any other code is
> a sign of a serious screwup.
*nod* thanks.
-Eric
On 8/30/20 8:45 PM, Qian Cai wrote:
> It is trivial to trigger a WARN_ON_ONCE(1) in iomap_dio_actor() by
> unprivileged users which would taint the kernel, or worse - panic if
> panic_on_warn or panic_on_taint is set. Hence, just convert it to
> pr_warn_ratelimited() to let users know their workloads are racing.
> Thanks Dave Chinner for the initial analysis of the racing reproducers.
>
> Signed-off-by: Qian Cai <[email protected]>
> ---
>
> v2: Record the path, pid and command as well.
>
> fs/iomap/direct-io.c | 17 ++++++++++++++++-
> 1 file changed, 16 insertions(+), 1 deletion(-)
>
> diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c
> index c1aafb2ab990..66a4502ef675 100644
> --- a/fs/iomap/direct-io.c
> +++ b/fs/iomap/direct-io.c
> @@ -374,6 +374,7 @@ iomap_dio_actor(struct inode *inode, loff_t pos, loff_t length,
> void *data, struct iomap *iomap, struct iomap *srcmap)
> {
> struct iomap_dio *dio = data;
> + char pathname[128], *path;
>
> switch (iomap->type) {
> case IOMAP_HOLE:
> @@ -389,7 +390,21 @@ iomap_dio_actor(struct inode *inode, loff_t pos, loff_t length,
> case IOMAP_INLINE:
> return iomap_dio_inline_actor(inode, pos, length, dio, iomap);
> default:
> - WARN_ON_ONCE(1);
It seems like we should explicitly catch IOMAP_DELALLOC for this case, and leave the
default: as a WARN_ON that is not user-triggerable? i.e.
case IOMAP_DELALLOC:
<all the fancy warnings>
return -EIO;
default:
WARN_ON_ONCE(1);
return -EIO;
> + /*
> + * DIO is not serialised against mmap() access at all, and so
> + * if the page_mkwrite occurs between the writeback and the
> + * iomap_apply() call in the DIO path, then it will see the
> + * DELALLOC block that the page-mkwrite allocated.
> + */
> + path = file_path(dio->iocb->ki_filp, pathname,
> + sizeof(pathname));
> + if (IS_ERR(path))
> + path = "(unknown)";
> +
> + pr_warn_ratelimited("page_mkwrite() is racing with DIO read (iomap->type = %u).\n"
> + "File: %s PID: %d Comm: %.20s\n",
> + iomap->type, path, current->pid,
> + current->comm);
This is very specific ...
Do we know that mmap/page_mkwrite is (and will always be) the only way to reach this
point?
It seems to me that this message won't be very useful for the admin; "pg_mkwrite" may
mean something to us, but doubtful for the general public. And "type = 1" won't mean
much to the reader, either.
Maybe something like:
"DIO encountered delayed allocation block, racing buffered+direct? File: %s Comm: %.20s\n"
It just seems that a user-facing warning should be something the admin has a chance of
acting on without needing to file a bug for analysis by the developers.
(though TBH "delayed allocation" probably doesn't mean much to the admin, either)
-Eric
> return -EIO;
> }
> }
>