From: Ira Weiny <[email protected]>
Changes from V2[1]:
Update this cover letter
Update commit messages
From Matthew Wilcox
Put functions in highmem.h rather than pagemap.h
Investigate 0-day build errors.
AFAICT the patches were applied to the wrong tree and caused
build errors.
There are many places in the kernel where kmap is used for a simple memory
operation like memcpy, memset, or memmove and then the page is unmapped.
This kmap/mem*/kunmap pattern is mixed between kmap and kmap_atomic uses. All
of them could use kmap_atomic() which is faster. However, kmap_atomic() is
being deprecated in favor of kmap_local_page().
Use kmap_local_page() in the existing page operations. Lift
memcpy_[to|from]_page to highmem.h. Remove memzero_page() and use zero_user()
instead. Add memcpy_page(), memmove_page(), and memset_page() to be used in
future patches. Finally, add BUG_ON()s to check for any miss use of the API
and prevent data corruption in the same way zero_user() does.
This is V3 to get into 5.11 so that we can start to convert all the various
subsystems in 5.12.[2]
These are based on tip/core/mm. I'm sending to Andrew and Thomas but I'm
expecting these to go through tip/core/mm via Thomas if that is ok with Andrew.
[1] https://lore.kernel.org/lkml/[email protected]/
[2] https://lore.kernel.org/lkml/[email protected]/
Ira Weiny (2):
mm/highmem: Remove deprecated kmap_atomic
mm/highmem: Lift memcpy_[to|from]_page to core
include/linux/highmem.h | 81 ++++++++++++++++++++++++++++++++++-------
lib/iov_iter.c | 26 ++-----------
2 files changed, 70 insertions(+), 37 deletions(-)
--
2.28.0.rc0.12.gb6a658bd00c9
From: Ira Weiny <[email protected]>
kmap_atomic() is being deprecated in favor of kmap_local_page().
Replace the uses of kmap_atomic() within the highmem code.
Signed-off-by: Ira Weiny <[email protected]>
---
include/linux/highmem.h | 28 ++++++++++++++--------------
1 file changed, 14 insertions(+), 14 deletions(-)
diff --git a/include/linux/highmem.h b/include/linux/highmem.h
index f597830f26b4..3bfe6a12d14d 100644
--- a/include/linux/highmem.h
+++ b/include/linux/highmem.h
@@ -146,9 +146,9 @@ static inline void invalidate_kernel_vmap_range(void *vaddr, int size)
#ifndef clear_user_highpage
static inline void clear_user_highpage(struct page *page, unsigned long vaddr)
{
- void *addr = kmap_atomic(page);
+ void *addr = kmap_local_page(page);
clear_user_page(addr, vaddr, page);
- kunmap_atomic(addr);
+ kunmap_local(addr);
}
#endif
@@ -199,16 +199,16 @@ alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma,
static inline void clear_highpage(struct page *page)
{
- void *kaddr = kmap_atomic(page);
+ void *kaddr = kmap_local_page(page);
clear_page(kaddr);
- kunmap_atomic(kaddr);
+ kunmap_local(kaddr);
}
static inline void zero_user_segments(struct page *page,
unsigned start1, unsigned end1,
unsigned start2, unsigned end2)
{
- void *kaddr = kmap_atomic(page);
+ void *kaddr = kmap_local_page(page);
BUG_ON(end1 > PAGE_SIZE || end2 > PAGE_SIZE);
@@ -218,7 +218,7 @@ static inline void zero_user_segments(struct page *page,
if (end2 > start2)
memset(kaddr + start2, 0, end2 - start2);
- kunmap_atomic(kaddr);
+ kunmap_local(kaddr);
flush_dcache_page(page);
}
@@ -241,11 +241,11 @@ static inline void copy_user_highpage(struct page *to, struct page *from,
{
char *vfrom, *vto;
- vfrom = kmap_atomic(from);
- vto = kmap_atomic(to);
+ vfrom = kmap_local_page(from);
+ vto = kmap_local_page(to);
copy_user_page(vto, vfrom, vaddr, to);
- kunmap_atomic(vto);
- kunmap_atomic(vfrom);
+ kunmap_local(vto);
+ kunmap_local(vfrom);
}
#endif
@@ -256,11 +256,11 @@ static inline void copy_highpage(struct page *to, struct page *from)
{
char *vfrom, *vto;
- vfrom = kmap_atomic(from);
- vto = kmap_atomic(to);
+ vfrom = kmap_local_page(from);
+ vto = kmap_local_page(to);
copy_page(vto, vfrom);
- kunmap_atomic(vto);
- kunmap_atomic(vfrom);
+ kunmap_local(vto);
+ kunmap_local(vfrom);
}
#endif
--
2.28.0.rc0.12.gb6a658bd00c9
From: Ira Weiny <[email protected]>
Working through a conversion to a call such as kmap_thread() revealed
many places where the pattern kmap/memcpy/kunmap occurred.
Eric Biggers, Matthew Wilcox, Christoph Hellwig, Dan Williams, and Al
Viro all suggested putting this code into helper functions. Al Viro
further pointed out that these functions already existed in the iov_iter
code.[1]
Various locations for the lifted functions were considered.
Headers like mm.h or string.h seem ok but don't really portray the
functionality well. pagemap.h made some sense but is for page cache
functionality.[2]
Another alternative would be to create a new header for the promoted
memcpy functions, but it masks the fact that these are designed to copy
to/from pages using the kernel direct mappings and complicates matters
with a new header.
Placing these functions in 'highmem.h' is suboptimal especially with the
changes being proposed in the functionality of kmap. From a caller
perspective including/using 'highmem.h' implies that the functions
defined in that header are only required when highmem is in use which is
increasingly not the case with modern processors. However, highmem.h is
where all the current functions like this reside (zero_user(),
clear_highpage(), clear_user_highpage(), copy_user_highpage(), and
copy_highpage()). So it makes the most sense even though it is
distasteful for some.[3]
Lift memcpy_to_page() and memcpy_from_page() to pagemap.h.
Remove memzero_page() in favor of zero_user() to zero a page.
Add a memcpy_page(), memmove_page, and memset_page() to cover more
kmap/mem*/kunmap. patterns.
Add BUG_ON bounds checks to ensure the newly lifted page memory
operations do not result in corrupted data in neighbor pages and to make
them consistent with zero_user().[4]
Finally use kmap_local_page() in all the new calls.
[1] https://lore.kernel.org/lkml/[email protected]/
https://lore.kernel.org/lkml/[email protected]/
[2] https://lore.kernel.org/lkml/[email protected]/
[3] https://lore.kernel.org/lkml/[email protected]/#t
https://lore.kernel.org/lkml/[email protected]/
[4] https://lore.kernel.org/lkml/[email protected]/
Cc: Dave Hansen <[email protected]>
Suggested-by: Matthew Wilcox <[email protected]>
Suggested-by: Christoph Hellwig <[email protected]>
Suggested-by: Dan Williams <[email protected]>
Suggested-by: Al Viro <[email protected]>
Suggested-by: Eric Biggers <[email protected]>
Signed-off-by: Ira Weiny <[email protected]>
---
Changes for V3:
From Matthew Wilcox
Move calls to highmem.h
Add BUG_ON()
Changes for V2:
From Thomas Gleixner
Change kmap_atomic() to kmap_local_page() after basing
on tip/core/mm
From Joonas Lahtinen
Reverse offset/val in memset_page()
---
include/linux/highmem.h | 53 +++++++++++++++++++++++++++++++++++++++++
lib/iov_iter.c | 26 +++-----------------
2 files changed, 56 insertions(+), 23 deletions(-)
diff --git a/include/linux/highmem.h b/include/linux/highmem.h
index 3bfe6a12d14d..75b2ca0b050b 100644
--- a/include/linux/highmem.h
+++ b/include/linux/highmem.h
@@ -265,4 +265,57 @@ static inline void copy_highpage(struct page *to, struct page *from)
#endif
+static inline void memcpy_page(struct page *dst_page, size_t dst_off,
+ struct page *src_page, size_t src_off,
+ size_t len)
+{
+ char *dst = kmap_local_page(dst_page);
+ char *src = kmap_local_page(src_page);
+
+ BUG_ON(dst_off + len > PAGE_SIZE || src_off + len > PAGE_SIZE);
+ memcpy(dst + dst_off, src + src_off, len);
+ kunmap_local(src);
+ kunmap_local(dst);
+}
+
+static inline void memmove_page(struct page *dst_page, size_t dst_off,
+ struct page *src_page, size_t src_off,
+ size_t len)
+{
+ char *dst = kmap_local_page(dst_page);
+ char *src = kmap_local_page(src_page);
+
+ BUG_ON(dst_off + len > PAGE_SIZE || src_off + len > PAGE_SIZE);
+ memmove(dst + dst_off, src + src_off, len);
+ kunmap_local(src);
+ kunmap_local(dst);
+}
+
+static inline void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len)
+{
+ char *from = kmap_local_page(page);
+
+ BUG_ON(offset + len > PAGE_SIZE);
+ memcpy(to, from + offset, len);
+ kunmap_local(from);
+}
+
+static inline void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len)
+{
+ char *to = kmap_local_page(page);
+
+ BUG_ON(offset + len > PAGE_SIZE);
+ memcpy(to + offset, from, len);
+ kunmap_local(to);
+}
+
+static inline void memset_page(struct page *page, size_t offset, int val, size_t len)
+{
+ char *addr = kmap_local_page(page);
+
+ BUG_ON(offset + len > PAGE_SIZE);
+ memset(addr + offset, val, len);
+ kunmap_local(addr);
+}
+
#endif /* _LINUX_HIGHMEM_H */
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index 1635111c5bd2..8ed1f846fcc3 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -5,6 +5,7 @@
#include <linux/fault-inject-usercopy.h>
#include <linux/uio.h>
#include <linux/pagemap.h>
+#include <linux/highmem.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/splice.h>
@@ -466,27 +467,6 @@ void iov_iter_init(struct iov_iter *i, unsigned int direction,
}
EXPORT_SYMBOL(iov_iter_init);
-static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len)
-{
- char *from = kmap_atomic(page);
- memcpy(to, from + offset, len);
- kunmap_atomic(from);
-}
-
-static void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len)
-{
- char *to = kmap_atomic(page);
- memcpy(to + offset, from, len);
- kunmap_atomic(to);
-}
-
-static void memzero_page(struct page *page, size_t offset, size_t len)
-{
- char *addr = kmap_atomic(page);
- memset(addr + offset, 0, len);
- kunmap_atomic(addr);
-}
-
static inline bool allocated(struct pipe_buffer *buf)
{
return buf->ops == &default_pipe_buf_ops;
@@ -964,7 +944,7 @@ static size_t pipe_zero(size_t bytes, struct iov_iter *i)
do {
size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
- memzero_page(pipe->bufs[i_head & p_mask].page, off, chunk);
+ zero_user(pipe->bufs[i_head & p_mask].page, off, chunk);
i->head = i_head;
i->iov_offset = off + chunk;
n -= chunk;
@@ -981,7 +961,7 @@ size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
return pipe_zero(bytes, i);
iterate_and_advance(i, bytes, v,
clear_user(v.iov_base, v.iov_len),
- memzero_page(v.bv_page, v.bv_offset, v.bv_len),
+ zero_user(v.bv_page, v.bv_offset, v.bv_len),
memset(v.iov_base, 0, v.iov_len)
)
--
2.28.0.rc0.12.gb6a658bd00c9