This series is based on (mostly) v5.12-rc8-mmots-2021-04-21-23-08. It tries to
fix some corner case bugs for uffd-wp on either thp or fork(). Then it
introduced a new test with pagemap/pageout.
Patch layout:
Patch 1: cleanup for THP, it'll slightly simplify the follow up patches
Patch 2-4: misc fixes for uffd-wp here and there; please refer to each patch
Patch 5: add pagemap support for uffd-wp
Patch 6: add pagemap/pageout test for uffd-wp
The last test introduced can also verify some of the fixes in previous patches,
as the test will fail without the fixes. However it's not easy to verify all
the changes in patch 2-4, but hopefully they can still be properly reviewed.
Note that if considering the ongoing uffd-wp shmem & hugetlbfs work, patch 5
will be incomplete as it's missing e.g. hugetlbfs part or the special swap pte
detection. However that's not needed in this series, and since that series is
still during review, this series does not depend on that one (the last test
only runs with anonymous memory, not file-backed). So this series can be
merged even before that series.
Please review, thanks.
Peter Xu (6):
mm/thp: Simplify copying of huge zero page pmd when fork
mm/userfaultfd: Fix uffd-wp special cases for fork()
mm/userfaultfd: Fix a few thp pmd missing uffd-wp bit
mm/userfaultfd: Fail uffd-wp registeration if not supported
mm/pagemap: Export uffd-wp protection information
userfaultfd/selftests: Add pagemap uffd-wp test
Documentation/admin-guide/mm/pagemap.rst | 2 +
fs/proc/task_mmu.c | 9 ++
fs/userfaultfd.c | 9 +-
include/linux/huge_mm.h | 2 +-
include/linux/swapops.h | 2 +
mm/huge_memory.c | 36 +++---
mm/memory.c | 25 ++--
tools/testing/selftests/vm/userfaultfd.c | 154 +++++++++++++++++++++++
8 files changed, 206 insertions(+), 33 deletions(-)
--
2.26.2
We should fail uffd-wp registration immediately if the arch does not even have
CONFIG_HAVE_ARCH_USERFAULTFD_WP defined. That'll block also relevant ioctls on
e.g. UFFDIO_WRITEPROTECT because that'll check against VM_UFFD_WP, which can
only be applied with a success registration.
Remove the WP feature bit too for those archs when handling UFFDIO_API ioctl.
Signed-off-by: Peter Xu <[email protected]>
---
fs/userfaultfd.c | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c
index 14f92285d04f8..5dd78238cc156 100644
--- a/fs/userfaultfd.c
+++ b/fs/userfaultfd.c
@@ -1304,8 +1304,12 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
vm_flags = 0;
if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MISSING)
vm_flags |= VM_UFFD_MISSING;
- if (uffdio_register.mode & UFFDIO_REGISTER_MODE_WP)
+ if (uffdio_register.mode & UFFDIO_REGISTER_MODE_WP) {
+#ifndef CONFIG_HAVE_ARCH_USERFAULTFD_WP
+ goto out;
+#endif
vm_flags |= VM_UFFD_WP;
+ }
if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR) {
#ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
goto out;
@@ -1942,6 +1946,9 @@ static int userfaultfd_api(struct userfaultfd_ctx *ctx,
uffdio_api.features = UFFD_API_FEATURES;
#ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
uffdio_api.features &= ~UFFD_FEATURE_MINOR_HUGETLBFS;
+#endif
+#ifndef CONFIG_HAVE_ARCH_USERFAULTFD_WP
+ uffdio_api.features &= ~UFFD_FEATURE_PAGEFAULT_FLAG_WP;
#endif
uffdio_api.ioctls = UFFD_API_IOCTLS;
ret = -EFAULT;
--
2.26.2
Add one anonymous specific test to start using pagemap. With pagemap support,
we can directly read the uffd-wp bit from pgtable without triggering any fault,
so it's easier to do sanity checks in unit tests.
Meanwhile this test also leverages the newly introduced MADV_PAGEOUT madvise
function to test swap ptes with uffd-wp bit set, and across fork()s.
Signed-off-by: Peter Xu <[email protected]>
---
tools/testing/selftests/vm/userfaultfd.c | 154 +++++++++++++++++++++++
1 file changed, 154 insertions(+)
diff --git a/tools/testing/selftests/vm/userfaultfd.c b/tools/testing/selftests/vm/userfaultfd.c
index 6339aeaeeff8b..93eae095b61e6 100644
--- a/tools/testing/selftests/vm/userfaultfd.c
+++ b/tools/testing/selftests/vm/userfaultfd.c
@@ -1170,6 +1170,144 @@ static int userfaultfd_minor_test(void)
return stats.missing_faults != 0 || stats.minor_faults != nr_pages;
}
+#define BIT_ULL(nr) (1ULL << (nr))
+#define PM_SOFT_DIRTY BIT_ULL(55)
+#define PM_MMAP_EXCLUSIVE BIT_ULL(56)
+#define PM_UFFD_WP BIT_ULL(57)
+#define PM_FILE BIT_ULL(61)
+#define PM_SWAP BIT_ULL(62)
+#define PM_PRESENT BIT_ULL(63)
+
+static int pagemap_open(void)
+{
+ int fd = open("/proc/self/pagemap", O_RDONLY);
+
+ if (fd < 0)
+ err("open pagemap");
+
+ return fd;
+}
+
+static uint64_t pagemap_read_vaddr(int fd, void *vaddr)
+{
+ uint64_t value;
+ int ret;
+
+ ret = pread(fd, &value, sizeof(uint64_t),
+ ((uint64_t)vaddr >> 12) * sizeof(uint64_t));
+ if (ret != sizeof(uint64_t))
+ err("pread() on pagemap failed");
+
+ return value;
+}
+
+/* This macro let __LINE__ works in err() */
+#define pagemap_check_wp(value, wp) do { \
+ if (!!(value & PM_UFFD_WP) != wp) \
+ err("pagemap uffd-wp bit error: 0x%"PRIx64, value); \
+ } while (0)
+
+static int pagemap_test_fork(bool present)
+{
+ pid_t child = fork();
+ uint64_t value;
+ int fd, result;
+
+ if (!child) {
+ /* Open the pagemap fd of the child itself */
+ fd = pagemap_open();
+ value = pagemap_read_vaddr(fd, area_dst);
+ /*
+ * After fork() uffd-wp bit should be gone as long as we're
+ * without UFFD_FEATURE_EVENT_FORK
+ */
+ pagemap_check_wp(value, false);
+ /* Succeed */
+ exit(0);
+ }
+ waitpid(child, &result, 0);
+ return result;
+}
+
+static void userfaultfd_pagemap_test(unsigned int test_pgsize)
+{
+ struct uffdio_register uffdio_register;
+ int pagemap_fd;
+ uint64_t value;
+
+ /* Pagemap tests uffd-wp only */
+ if (!test_uffdio_wp)
+ return;
+
+ /* Not enough memory to test this page size */
+ if (test_pgsize > nr_pages * page_size)
+ return;
+
+ printf("testing uffd-wp with pagemap (pgsize=%u): ", test_pgsize);
+ /* Flush so it doesn't flush twice in parent/child later */
+ fflush(stdout);
+
+ uffd_test_ops->release_pages(area_dst);
+
+ if (test_pgsize > page_size) {
+ /* This is a thp test */
+ if (madvise(area_dst, nr_pages * page_size, MADV_HUGEPAGE))
+ err("madvise(MADV_HUGEPAGE) failed");
+ } else if (test_pgsize == page_size) {
+ /* This is normal page test; force no thp */
+ if (madvise(area_dst, nr_pages * page_size, MADV_NOHUGEPAGE))
+ err("madvise(MADV_NOHUGEPAGE) failed");
+ }
+
+ if (userfaultfd_open(0))
+ err("userfaultfd_open");
+
+ uffdio_register.range.start = (unsigned long) area_dst;
+ uffdio_register.range.len = nr_pages * page_size;
+ uffdio_register.mode = UFFDIO_REGISTER_MODE_WP;
+ if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
+ err("register failed");
+
+ pagemap_fd = pagemap_open();
+
+ /* Touch the page */
+ *area_dst = 1;
+ wp_range(uffd, (uint64_t)area_dst, test_pgsize, true);
+ value = pagemap_read_vaddr(pagemap_fd, area_dst);
+ pagemap_check_wp(value, true);
+ /* Make sure uffd-wp bit dropped when fork */
+ if (pagemap_test_fork(true))
+ err("Detected stall uffd-wp bit in child");
+
+ /* Exclusive required or PAGEOUT won't work */
+ if (!(value & PM_MMAP_EXCLUSIVE))
+ err("multiple mapping detected: 0x%"PRIx64, value);
+
+ if (madvise(area_dst, test_pgsize, MADV_PAGEOUT))
+ err("madvise(MADV_PAGEOUT) failed");
+
+ /* Uffd-wp should persist even swapped out */
+ value = pagemap_read_vaddr(pagemap_fd, area_dst);
+ pagemap_check_wp(value, true);
+ /* Make sure uffd-wp bit dropped when fork */
+ if (pagemap_test_fork(false))
+ err("Detected stall uffd-wp bit in child");
+
+ /* Unprotect; this tests swap pte modifications */
+ wp_range(uffd, (uint64_t)area_dst, page_size, false);
+ value = pagemap_read_vaddr(pagemap_fd, area_dst);
+ pagemap_check_wp(value, false);
+
+ /* Fault in the page from disk */
+ *area_dst = 2;
+ value = pagemap_read_vaddr(pagemap_fd, area_dst);
+ pagemap_check_wp(value, false);
+
+ close(pagemap_fd);
+ close(uffd);
+ printf("done\n");
+}
+
static int userfaultfd_stress(void)
{
void *area;
@@ -1341,6 +1479,22 @@ static int userfaultfd_stress(void)
}
close(uffd);
+
+ if (test_type == TEST_ANON) {
+ /*
+ * shmem/hugetlb won't be able to run since they have different
+ * behavior on fork() (file-backed memory normally drops ptes
+ * directly when fork), meanwhile the pagemap test will verify
+ * pgtable entry of fork()ed child.
+ */
+ userfaultfd_pagemap_test(page_size);
+ /*
+ * Hard-code for x86_64 for now for 2M THP, as x86_64 is
+ * currently the only one that supports uffd-wp
+ */
+ userfaultfd_pagemap_test(page_size * 512);
+ }
+
return userfaultfd_zeropage_test() || userfaultfd_sig_test()
|| userfaultfd_events_test() || userfaultfd_minor_test();
}
--
2.26.2
On Wed, Apr 28, 2021 at 06:50:24PM -0400, Peter Xu wrote:
> This series is based on (mostly) v5.12-rc8-mmots-2021-04-21-23-08. It tries to
> fix some corner case bugs for uffd-wp on either thp or fork(). Then it
> introduced a new test with pagemap/pageout.
Sorry I forgot to mention this series is also based on the other selftest
cleanup series applied upon the rc8 tag:
https://lore.kernel.org/patchwork/cover/1412450/
--
Peter Xu