Hi,
This converts the iomap core and bio_release_pages() to
pin_user_pages_fast(), also referred to as FOLL_PIN here.
The conversion is temporarily guarded by
CONFIG_BLK_USE_PIN_USER_PAGES_FOR_DIO. In the future (not part of this
series), when we are certain that all filesystems have converted their
Direct IO paths to FOLL_PIN, then we can do the final step, which is to
get rid of CONFIG_BLK_USE_PIN_USER_PAGES_FOR_DIO and search-and-replace
the dio_w_*() functions with their final names (see bvec.h changes).
I'd like to get this part committed at some point, because it seems to
work well already. And this will help get the remaining items, below,
converted.
Status: although many filesystems have been converted, some remain to be
investigated. These include (you can recreate this list by grepping for
iov_iter_get_pages):
cephfs
cifs
9P
RDS
net/core: datagram.c, skmsg.c
net/tls
fs/splice.c
Testing: this passes some light LTP and xfstest runs and fio and a few
other things like that, on my local x86_64 test machine, both with and
without CONFIG_BLK_USE_PIN_USER_PAGES_FOR_DIO being set.
Conflicts: Logan, the iov_iter parts of this will conflict with your
[PATCH v9 2/8] iov_iter: introduce iov_iter_get_pages_[alloc_]flags(),
but I think it's easy to resolve.
John Hubbard (6):
mm/gup: introduce pin_user_page()
block: add dio_w_*() wrappers for pin, unpin user pages
iov_iter: new iov_iter_pin_pages*() routines
block, bio, fs: convert most filesystems to pin_user_pages_fast()
NFS: direct-io: convert to FOLL_PIN pages
fuse: convert direct IO paths to use FOLL_PIN
block/Kconfig | 24 ++++++++++++++
block/bio.c | 27 ++++++++--------
block/blk-map.c | 7 +++--
fs/direct-io.c | 40 ++++++++++++------------
fs/fuse/dev.c | 8 +++--
fs/fuse/file.c | 31 ++++++++++++-------
fs/fuse/fuse_i.h | 1 +
fs/iomap/direct-io.c | 2 +-
fs/nfs/direct.c | 19 ++++--------
include/linux/bvec.h | 40 ++++++++++++++++++++++++
include/linux/mm.h | 1 +
include/linux/uio.h | 4 +++
lib/iov_iter.c | 74 +++++++++++++++++++++++++++++++++++++++++---
mm/gup.c | 33 ++++++++++++++++++++
14 files changed, 244 insertions(+), 67 deletions(-)
base-commit: e022620b5d056e822e42eb9bc0f24fcb97389d86
--
2.37.2
Convert the fuse filesystem to use pin_user_pages_fast() and
unpin_user_page(), instead of get_user_pages_fast() and put_page().
Signed-off-by: John Hubbard <[email protected]>
---
fs/fuse/dev.c | 8 ++++++--
fs/fuse/file.c | 31 ++++++++++++++++++++-----------
fs/fuse/fuse_i.h | 1 +
3 files changed, 27 insertions(+), 13 deletions(-)
diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c
index 51897427a534..eb841fc82bb9 100644
--- a/fs/fuse/dev.c
+++ b/fs/fuse/dev.c
@@ -675,7 +675,10 @@ static void fuse_copy_finish(struct fuse_copy_state *cs)
flush_dcache_page(cs->pg);
set_page_dirty_lock(cs->pg);
}
- put_page(cs->pg);
+ if (cs->pipebufs)
+ put_page(cs->pg);
+ else
+ dio_w_unpin_user_page(cs->pg);
}
cs->pg = NULL;
}
@@ -730,7 +733,8 @@ static int fuse_copy_fill(struct fuse_copy_state *cs)
}
} else {
size_t off;
- err = iov_iter_get_pages2(cs->iter, &page, PAGE_SIZE, 1, &off);
+ err = dio_w_iov_iter_pin_pages(cs->iter, &page, PAGE_SIZE, 1,
+ &off);
if (err < 0)
return err;
BUG_ON(!err);
diff --git a/fs/fuse/file.c b/fs/fuse/file.c
index 1a3afd469e3a..a79aa4fea937 100644
--- a/fs/fuse/file.c
+++ b/fs/fuse/file.c
@@ -625,14 +625,19 @@ void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos,
}
static void fuse_release_user_pages(struct fuse_args_pages *ap,
- bool should_dirty)
+ bool should_dirty, bool is_kvec)
{
unsigned int i;
- for (i = 0; i < ap->num_pages; i++) {
- if (should_dirty)
- set_page_dirty_lock(ap->pages[i]);
- put_page(ap->pages[i]);
+ if (is_kvec) {
+ for (i = 0; i < ap->num_pages; i++) {
+ if (should_dirty)
+ set_page_dirty_lock(ap->pages[i]);
+ put_page(ap->pages[i]);
+ }
+ } else {
+ dio_w_unpin_user_pages_dirty_lock(ap->pages, ap->num_pages,
+ should_dirty);
}
}
@@ -733,7 +738,7 @@ static void fuse_aio_complete_req(struct fuse_mount *fm, struct fuse_args *args,
struct fuse_io_priv *io = ia->io;
ssize_t pos = -1;
- fuse_release_user_pages(&ia->ap, io->should_dirty);
+ fuse_release_user_pages(&ia->ap, io->should_dirty, io->is_kvec);
if (err) {
/* Nothing */
@@ -1414,10 +1419,10 @@ static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii,
while (nbytes < *nbytesp && ap->num_pages < max_pages) {
unsigned npages;
size_t start;
- ret = iov_iter_get_pages2(ii, &ap->pages[ap->num_pages],
- *nbytesp - nbytes,
- max_pages - ap->num_pages,
- &start);
+ ret = dio_w_iov_iter_pin_pages(ii, &ap->pages[ap->num_pages],
+ *nbytesp - nbytes,
+ max_pages - ap->num_pages,
+ &start);
if (ret < 0)
break;
@@ -1483,6 +1488,9 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter,
fl_owner_t owner = current->files;
size_t nbytes = min(count, nmax);
+ /* For use in fuse_release_user_pages(): */
+ io->is_kvec = iov_iter_is_kvec(iter);
+
err = fuse_get_user_pages(&ia->ap, iter, &nbytes, write,
max_pages);
if (err && !nbytes)
@@ -1498,7 +1506,8 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter,
}
if (!io->async || nres < 0) {
- fuse_release_user_pages(&ia->ap, io->should_dirty);
+ fuse_release_user_pages(&ia->ap, io->should_dirty,
+ io->is_kvec);
fuse_io_free(ia);
}
ia = NULL;
diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h
index 488b460e046f..1d927e499395 100644
--- a/fs/fuse/fuse_i.h
+++ b/fs/fuse/fuse_i.h
@@ -290,6 +290,7 @@ struct fuse_io_priv {
struct kiocb *iocb;
struct completion *done;
bool blocking;
+ bool is_kvec;
};
#define FUSE_IO_PRIV_SYNC(i) \
--
2.37.2