BTRFS need to get the last singlepage bvec, so introduce
this helper to make it happy.
Signed-off-by: Ming Lei <[email protected]>
---
include/linux/bvec.h | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/include/linux/bvec.h b/include/linux/bvec.h
index baf379d56106..e9a623067c52 100644
--- a/include/linux/bvec.h
+++ b/include/linux/bvec.h
@@ -206,4 +206,14 @@ static inline void bvec_iter_advance_mp(const struct bio_vec *bv,
#define bvec_for_each_sp_bvec(sp_bvl, mp_bvec, iter) \
__bvec_for_each_sp_bvec(sp_bvl, mp_bvec, iter, BVEC_ITER_ALL_INIT)
+/* get the last singlepage bvec from the multipage bvec */
+static inline void bvec_get_last_sp(struct bio_vec *mp_bv,
+ struct bio_vec *sp_bv)
+{
+ struct bvec_iter iter;
+
+ bvec_for_each_sp_bvec(*sp_bv, mp_bv, iter)
+ ;
+}
+
#endif /* __LINUX_BVEC_ITER_H */
--
2.7.4
In bio_check_pages_dirty(), bvec->bv_page is used as flag
for marking if the page has been dirtied & released, and if
no, it will be dirtied in deferred workqueue.
With multipage bvec, we can't do that any more, so change
the logic into checking all pages in one mp bvec, and only
release all these pages if all are dirtied, otherwise dirty
them all in deferred wrokqueue.
Signed-off-by: Ming Lei <[email protected]>
---
block/bio.c | 45 +++++++++++++++++++++++++++++++++++++--------
1 file changed, 37 insertions(+), 8 deletions(-)
diff --git a/block/bio.c b/block/bio.c
index a76ed8a780de..e65de53acb9f 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -1655,8 +1655,9 @@ void bio_set_pages_dirty(struct bio *bio)
{
struct bio_vec *bvec;
int i;
+ struct bvec_iter_all bia;
- bio_for_each_segment_all(bvec, bio, i) {
+ bio_for_each_segment_all_sp(bvec, bio, i, bia) {
struct page *page = bvec->bv_page;
if (page && !PageCompound(page))
@@ -1664,16 +1665,26 @@ void bio_set_pages_dirty(struct bio *bio)
}
}
+static inline void release_mp_bvec_pages(struct bio_vec *bvec)
+{
+ struct bio_vec bv;
+ struct bvec_iter iter;
+
+ bvec_for_each_sp_bvec(bv, bvec, iter)
+ put_page(bv.bv_page);
+}
+
static void bio_release_pages(struct bio *bio)
{
struct bio_vec *bvec;
int i;
- bio_for_each_segment_all(bvec, bio, i) {
+ /* iterate each mp bvec */
+ bio_for_each_segment_all_mp(bvec, bio, i) {
struct page *page = bvec->bv_page;
if (page)
- put_page(page);
+ release_mp_bvec_pages(bvec);
}
}
@@ -1717,20 +1728,38 @@ static void bio_dirty_fn(struct work_struct *work)
}
}
+static inline void check_mp_bvec_pages(struct bio_vec *bvec,
+ int *nr_dirty, int *nr_pages)
+{
+ struct bio_vec bv;
+ struct bvec_iter iter;
+
+ bvec_for_each_sp_bvec(bv, bvec, iter) {
+ struct page *page = bv.bv_page;
+
+ if (PageDirty(page) || PageCompound(page))
+ (*nr_dirty)++;
+ (*nr_pages)++;
+ }
+}
+
void bio_check_pages_dirty(struct bio *bio)
{
struct bio_vec *bvec;
int nr_clean_pages = 0;
int i;
- bio_for_each_segment_all(bvec, bio, i) {
- struct page *page = bvec->bv_page;
+ bio_for_each_segment_all_mp(bvec, bio, i) {
+ int nr_dirty = 0, nr_pages = 0;
+
+ check_mp_bvec_pages(bvec, &nr_dirty, &nr_pages);
- if (PageDirty(page) || PageCompound(page)) {
- put_page(page);
+ /* release all pages in the mp bvec if all are dirtied */
+ if (nr_dirty == nr_pages) {
+ release_mp_bvec_pages(bvec);
bvec->bv_page = NULL;
} else {
- nr_clean_pages++;
+ nr_clean_pages += nr_pages;
}
}
--
2.7.4
Signed-off-by: Ming Lei <[email protected]>
---
drivers/md/bcache/btree.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/drivers/md/bcache/btree.c b/drivers/md/bcache/btree.c
index fc35cfb4d0f1..61d40b1903f3 100644
--- a/drivers/md/bcache/btree.c
+++ b/drivers/md/bcache/btree.c
@@ -419,8 +419,9 @@ static void do_btree_node_write(struct btree *b)
int j;
struct bio_vec *bv;
void *base = (void *) ((unsigned long) i & ~(PAGE_SIZE - 1));
+ struct bvec_iter_all bia;
- bio_for_each_segment_all(bv, b->bio, j)
+ bio_for_each_segment_all_sp(bv, b->bio, j, bia)
memcpy(page_address(bv->bv_page),
base + j * PAGE_SIZE, PAGE_SIZE);
--
2.7.4
Signed-off-by: Ming Lei <[email protected]>
---
drivers/md/dm-crypt.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index 593cdf88bf5f..c6932fb85418 100644
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -1038,8 +1038,9 @@ static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone)
{
unsigned int i;
struct bio_vec *bv;
+ struct bvec_iter_all bia;
- bio_for_each_segment_all(bv, clone, i) {
+ bio_for_each_segment_all_sp(bv, clone, i, bia) {
BUG_ON(!bv->bv_page);
mempool_free(bv->bv_page, cc->page_pool);
}
--
2.7.4
The bio is always freed after running crypt_free_buffer_pages(),
so it isn't necessary to clear the bv->bv_page.
Signed-off-by: Ming Lei <[email protected]>
---
drivers/md/dm-crypt.c | 1 -
1 file changed, 1 deletion(-)
diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index 7c6c57216bf2..593cdf88bf5f 100644
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -1042,7 +1042,6 @@ static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone)
bio_for_each_segment_all(bv, clone, i) {
BUG_ON(!bv->bv_page);
mempool_free(bv->bv_page, cc->page_pool);
- bv->bv_page = NULL;
}
}
--
2.7.4
Signed-off-by: Ming Lei <[email protected]>
---
drivers/md/raid1.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c
index a1f3fbed9100..4818f40e7ce9 100644
--- a/drivers/md/raid1.c
+++ b/drivers/md/raid1.c
@@ -988,12 +988,13 @@ static void alloc_behind_pages(struct bio *bio, struct r1bio *r1_bio)
{
int i;
struct bio_vec *bvec;
+ struct bvec_iter_all bia;
struct bio_vec *bvecs = kzalloc(bio->bi_vcnt * sizeof(struct bio_vec),
GFP_NOIO);
if (unlikely(!bvecs))
return;
- bio_for_each_segment_all(bvec, bio, i) {
+ bio_for_each_segment_all_sp(bvec, bio, i, bia) {
bvecs[i] = *bvec;
bvecs[i].bv_page = alloc_page(GFP_NOIO);
if (unlikely(!bvecs[i].bv_page))
--
2.7.4
Signed-off-by: Ming Lei <[email protected]>
---
fs/mpage.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/fs/mpage.c b/fs/mpage.c
index 28af984a3d96..bd94b3dbf27e 100644
--- a/fs/mpage.c
+++ b/fs/mpage.c
@@ -46,9 +46,10 @@
static void mpage_end_io(struct bio *bio)
{
struct bio_vec *bv;
+ struct bvec_iter_all bia;
int i;
- bio_for_each_segment_all(bv, bio, i) {
+ bio_for_each_segment_all_sp(bv, bio, i, bia) {
struct page *page = bv->bv_page;
page_endio(page, op_is_write(bio_op(bio)), bio->bi_error);
}
--
2.7.4
Signed-off-by: Ming Lei <[email protected]>
---
fs/crypto/crypto.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/fs/crypto/crypto.c b/fs/crypto/crypto.c
index ac8e4f6a3773..4bb1d7eff6f4 100644
--- a/fs/crypto/crypto.c
+++ b/fs/crypto/crypto.c
@@ -453,8 +453,9 @@ static void completion_pages(struct work_struct *work)
struct bio *bio = ctx->r.bio;
struct bio_vec *bv;
int i;
+ struct bvec_iter_all bia;
- bio_for_each_segment_all(bv, bio, i) {
+ bio_for_each_segment_all_sp(bv, bio, i, bia) {
struct page *page = bv->bv_page;
int ret = fscrypt_decrypt_page(page->mapping->host, page,
PAGE_SIZE, 0, page->index);
--
2.7.4
Signed-off-by: Ming Lei <[email protected]>
---
fs/btrfs/check-integrity.c | 5 +++--
fs/btrfs/compression.c | 3 ++-
fs/btrfs/disk-io.c | 3 ++-
fs/btrfs/extent_io.c | 9 ++++++---
fs/btrfs/file-item.c | 6 ++++--
fs/btrfs/inode.c | 15 ++++++++++-----
fs/btrfs/raid56.c | 6 ++++--
7 files changed, 31 insertions(+), 16 deletions(-)
diff --git a/fs/btrfs/check-integrity.c b/fs/btrfs/check-integrity.c
index ab14c2e635ca..e3cc7b18fe8b 100644
--- a/fs/btrfs/check-integrity.c
+++ b/fs/btrfs/check-integrity.c
@@ -2828,6 +2828,7 @@ static void __btrfsic_submit_bio(struct bio *bio)
struct bio_vec *bvec;
int bio_is_patched;
char **mapped_datav;
+ struct bvec_iter_all bia;
dev_bytenr = 512 * bio->bi_iter.bi_sector;
bio_is_patched = 0;
@@ -2844,7 +2845,7 @@ static void __btrfsic_submit_bio(struct bio *bio)
goto leave;
cur_bytenr = dev_bytenr;
- bio_for_each_segment_all(bvec, bio, i) {
+ bio_for_each_segment_all_sp(bvec, bio, i, bia) {
BUG_ON(bvec->bv_len != PAGE_SIZE);
mapped_datav[i] = kmap(bvec->bv_page);
@@ -2858,7 +2859,7 @@ static void __btrfsic_submit_bio(struct bio *bio)
mapped_datav, bio->bi_vcnt,
bio, &bio_is_patched,
NULL, bio->bi_opf);
- bio_for_each_segment_all(bvec, bio, i)
+ bio_for_each_segment_all_sp(bvec, bio, i, bia)
kunmap(bvec->bv_page);
kfree(mapped_datav);
} else if (NULL != dev_state && (bio->bi_opf & REQ_PREFLUSH)) {
diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c
index 7f390849343b..adcf5568f010 100644
--- a/fs/btrfs/compression.c
+++ b/fs/btrfs/compression.c
@@ -198,12 +198,13 @@ static void end_compressed_bio_read(struct bio *bio)
} else {
int i;
struct bio_vec *bvec;
+ struct bvec_iter_all bia;
/*
* we have verified the checksum already, set page
* checked so the end_io handlers know about it
*/
- bio_for_each_segment_all(bvec, cb->orig_bio, i)
+ bio_for_each_segment_all_sp(bvec, cb->orig_bio, i, bia)
SetPageChecked(bvec->bv_page);
bio_endio(cb->orig_bio);
diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
index 18004169552c..c2f9a42cd200 100644
--- a/fs/btrfs/disk-io.c
+++ b/fs/btrfs/disk-io.c
@@ -965,8 +965,9 @@ static int btree_csum_one_bio(struct bio *bio)
struct bio_vec *bvec;
struct btrfs_root *root;
int i, ret = 0;
+ struct bvec_iter_all bia;
- bio_for_each_segment_all(bvec, bio, i) {
+ bio_for_each_segment_all_sp(bvec, bio, i, bia) {
root = BTRFS_I(bvec->bv_page->mapping->host)->root;
ret = csum_dirty_buffer(root->fs_info, bvec->bv_page);
if (ret)
diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index 4ac383a3a649..137f95063c85 100644
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -2471,8 +2471,9 @@ static void end_bio_extent_writepage(struct bio *bio)
u64 start;
u64 end;
int i;
+ struct bvec_iter_all bia;
- bio_for_each_segment_all(bvec, bio, i) {
+ bio_for_each_segment_all_sp(bvec, bio, i, bia) {
struct page *page = bvec->bv_page;
struct inode *inode = page->mapping->host;
struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
@@ -2541,8 +2542,9 @@ static void end_bio_extent_readpage(struct bio *bio)
int mirror;
int ret;
int i;
+ struct bvec_iter_all bia;
- bio_for_each_segment_all(bvec, bio, i) {
+ bio_for_each_segment_all_sp(bvec, bio, i, bia) {
struct page *page = bvec->bv_page;
struct inode *inode = page->mapping->host;
struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
@@ -3692,8 +3694,9 @@ static void end_bio_extent_buffer_writepage(struct bio *bio)
struct bio_vec *bvec;
struct extent_buffer *eb;
int i, done;
+ struct bvec_iter_all bia;
- bio_for_each_segment_all(bvec, bio, i) {
+ bio_for_each_segment_all_sp(bvec, bio, i, bia) {
struct page *page = bvec->bv_page;
eb = (struct extent_buffer *)page->private;
diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c
index e97e322c28f0..c281df0e6485 100644
--- a/fs/btrfs/file-item.c
+++ b/fs/btrfs/file-item.c
@@ -179,6 +179,7 @@ static int __btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio,
int nblocks;
int count = 0, i;
u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
+ struct bvec_iter_all bia;
path = btrfs_alloc_path();
if (!path)
@@ -223,7 +224,7 @@ static int __btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio,
if (dio)
offset = logical_offset;
- bio_for_each_segment_all(bvec, bio, i) {
+ bio_for_each_segment_all_sp(bvec, bio, i, bia) {
page_bytes_left = bvec->bv_len;
if (count)
goto next;
@@ -447,6 +448,7 @@ int btrfs_csum_one_bio(struct inode *inode, struct bio *bio,
unsigned long total_bytes = 0;
unsigned long this_sum_bytes = 0;
u64 offset;
+ struct bvec_iter_all bia;
WARN_ON(bio->bi_vcnt <= 0);
sums = kzalloc(btrfs_ordered_sum_size(fs_info, bio->bi_iter.bi_size),
@@ -465,7 +467,7 @@ int btrfs_csum_one_bio(struct inode *inode, struct bio *bio,
sums->bytenr = (u64)bio->bi_iter.bi_sector << 9;
index = 0;
- bio_for_each_segment_all(bvec, bio, j) {
+ bio_for_each_segment_all_sp(bvec, bio, j, bia) {
if (!contig)
offset = page_offset(bvec->bv_page) + bvec->bv_offset;
diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
index f2b281ad7af6..544e1cf9a552 100644
--- a/fs/btrfs/inode.c
+++ b/fs/btrfs/inode.c
@@ -7955,6 +7955,7 @@ static void btrfs_retry_endio_nocsum(struct bio *bio)
struct inode *inode;
struct bio_vec *bvec;
int i;
+ struct bvec_iter_all bia;
if (bio->bi_error)
goto end;
@@ -7964,7 +7965,7 @@ static void btrfs_retry_endio_nocsum(struct bio *bio)
ASSERT(bio->bi_io_vec->bv_len == btrfs_inode_sectorsize(inode));
done->uptodate = 1;
- bio_for_each_segment_all(bvec, bio, i)
+ bio_for_each_segment_all_sp(bvec, bio, i, bia)
clean_io_failure(done->inode, done->start, bvec->bv_page, 0);
end:
complete(&done->done);
@@ -7983,6 +7984,7 @@ static int __btrfs_correct_data_nocsum(struct inode *inode,
int nr_sectors;
int i;
int ret;
+ struct bvec_iter_all bia;
fs_info = BTRFS_I(inode)->root->fs_info;
sectorsize = fs_info->sectorsize;
@@ -7990,7 +7992,7 @@ static int __btrfs_correct_data_nocsum(struct inode *inode,
start = io_bio->logical;
done.inode = inode;
- bio_for_each_segment_all(bvec, &io_bio->bio, i) {
+ bio_for_each_segment_all_sp(bvec, &io_bio->bio, i, bia) {
nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec->bv_len);
pgoff = bvec->bv_offset;
@@ -8034,6 +8036,7 @@ static void btrfs_retry_endio(struct bio *bio)
int uptodate;
int ret;
int i;
+ struct bvec_iter_all bia;
if (bio->bi_error)
goto end;
@@ -8046,7 +8049,7 @@ static void btrfs_retry_endio(struct bio *bio)
inode = bio->bi_io_vec->bv_page->mapping->host;
ASSERT(bio->bi_io_vec->bv_len == btrfs_inode_sectorsize(inode));
- bio_for_each_segment_all(bvec, bio, i) {
+ bio_for_each_segment_all_sp(bvec, bio, i, bia) {
ret = __readpage_endio_check(done->inode, io_bio, i,
bvec->bv_page, bvec->bv_offset,
done->start, bvec->bv_len);
@@ -8077,6 +8080,7 @@ static int __btrfs_subio_endio_read(struct inode *inode,
int csum_pos;
int i;
int ret;
+ struct bvec_iter_all bia;
fs_info = BTRFS_I(inode)->root->fs_info;
sectorsize = fs_info->sectorsize;
@@ -8085,7 +8089,7 @@ static int __btrfs_subio_endio_read(struct inode *inode,
start = io_bio->logical;
done.inode = inode;
- bio_for_each_segment_all(bvec, &io_bio->bio, i) {
+ bio_for_each_segment_all_sp(bvec, &io_bio->bio, i, bia) {
nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec->bv_len);
pgoff = bvec->bv_offset;
@@ -8380,6 +8384,7 @@ static int btrfs_submit_direct_hook(struct btrfs_dio_private *dip,
int nr_sectors;
int ret;
int i, j;
+ struct bvec_iter_all bia;
map_length = orig_bio->bi_iter.bi_size;
ret = btrfs_map_block(fs_info, btrfs_op(orig_bio), start_sector << 9,
@@ -8409,7 +8414,7 @@ static int btrfs_submit_direct_hook(struct btrfs_dio_private *dip,
btrfs_io_bio(bio)->logical = file_offset;
atomic_inc(&dip->pending_bios);
- bio_for_each_segment_all(bvec, orig_bio, j) {
+ bio_for_each_segment_all_sp(bvec, orig_bio, j, bia) {
nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec->bv_len);
i = 0;
next_block:
diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c
index d2a9a1ee5361..b218c4084c26 100644
--- a/fs/btrfs/raid56.c
+++ b/fs/btrfs/raid56.c
@@ -1150,6 +1150,7 @@ static void index_rbio_pages(struct btrfs_raid_bio *rbio)
unsigned long stripe_offset;
unsigned long page_index;
int i;
+ struct bvec_iter_all bia;
spin_lock_irq(&rbio->bio_list_lock);
bio_list_for_each(bio, &rbio->bio_list) {
@@ -1157,7 +1158,7 @@ static void index_rbio_pages(struct btrfs_raid_bio *rbio)
stripe_offset = start - rbio->bbio->raid_map[0];
page_index = stripe_offset >> PAGE_SHIFT;
- bio_for_each_segment_all(bvec, bio, i)
+ bio_for_each_segment_all_sp(bvec, bio, i, bia)
rbio->bio_pages[page_index + i] = bvec->bv_page;
}
spin_unlock_irq(&rbio->bio_list_lock);
@@ -1434,8 +1435,9 @@ static void set_bio_pages_uptodate(struct bio *bio)
{
struct bio_vec *bvec;
int i;
+ struct bvec_iter_all bia;
- bio_for_each_segment_all(bvec, bio, i)
+ bio_for_each_segment_all_sp(bvec, bio, i, bia)
SetPageUptodate(bvec->bv_page);
}
--
2.7.4
Preparing for supporting multipage bvec.
Signed-off-by: Ming Lei <[email protected]>
---
fs/btrfs/compression.c | 5 ++++-
fs/btrfs/extent_io.c | 8 ++++++--
2 files changed, 10 insertions(+), 3 deletions(-)
diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c
index adcf5568f010..485a60923bba 100644
--- a/fs/btrfs/compression.c
+++ b/fs/btrfs/compression.c
@@ -448,8 +448,11 @@ int btrfs_submit_compressed_write(struct inode *inode, u64 start,
static u64 bio_end_offset(struct bio *bio)
{
struct bio_vec *last = &bio->bi_io_vec[bio->bi_vcnt - 1];
+ struct bio_vec uninitialized_var(bv);
- return page_offset(last->bv_page) + last->bv_len + last->bv_offset;
+ bvec_get_last_sp(last, &bv);
+
+ return page_offset(bv.bv_page) + bv.bv_len + bv.bv_offset;
}
static noinline int add_ra_bio_pages(struct inode *inode,
diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index 3c021f4dd8bf..4a1c81cee883 100644
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -2733,11 +2733,15 @@ static int __must_check submit_one_bio(struct bio *bio, int mirror_num,
{
int ret = 0;
struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
- struct page *page = bvec->bv_page;
struct extent_io_tree *tree = bio->bi_private;
+ struct bio_vec uninitialized_var(bv);
+ struct page *page;
u64 start;
- start = page_offset(page) + bvec->bv_offset;
+ bvec_get_last_sp(bvec, &bv);
+ page = bv.bv_page;
+
+ start = page_offset(page) + bv.bv_offset;
bio->bi_private = NULL;
bio_get(bio);
--
2.7.4
BTRFS uses bio->bi_vcnt to figure out page numbers, this
way becomes not correct once we start to support multipage
bvec.
So use bio_segments_all() instead.
Signed-off-by: Ming Lei <[email protected]>
---
fs/btrfs/check-integrity.c | 7 ++++---
fs/btrfs/extent_io.c | 9 ++++++---
fs/btrfs/extent_io.h | 1 +
fs/btrfs/inode.c | 2 +-
4 files changed, 12 insertions(+), 7 deletions(-)
diff --git a/fs/btrfs/check-integrity.c b/fs/btrfs/check-integrity.c
index e3cc7b18fe8b..9c184a61e098 100644
--- a/fs/btrfs/check-integrity.c
+++ b/fs/btrfs/check-integrity.c
@@ -2829,17 +2829,18 @@ static void __btrfsic_submit_bio(struct bio *bio)
int bio_is_patched;
char **mapped_datav;
struct bvec_iter_all bia;
+ unsigned num_pages = bio_segments_all(bio);
dev_bytenr = 512 * bio->bi_iter.bi_sector;
bio_is_patched = 0;
if (dev_state->state->print_mask &
BTRFSIC_PRINT_MASK_SUBMIT_BIO_BH)
pr_info("submit_bio(rw=%d,0x%x, bi_vcnt=%u, bi_sector=%llu (bytenr %llu), bi_bdev=%p)\n",
- bio_op(bio), bio->bi_opf, bio->bi_vcnt,
+ bio_op(bio), bio->bi_opf, num_pages,
(unsigned long long)bio->bi_iter.bi_sector,
dev_bytenr, bio->bi_bdev);
- mapped_datav = kmalloc_array(bio->bi_vcnt,
+ mapped_datav = kmalloc_array(num_pages,
sizeof(*mapped_datav), GFP_NOFS);
if (!mapped_datav)
goto leave;
@@ -2856,7 +2857,7 @@ static void __btrfsic_submit_bio(struct bio *bio)
cur_bytenr += bvec->bv_len;
}
btrfsic_process_written_block(dev_state, dev_bytenr,
- mapped_datav, bio->bi_vcnt,
+ mapped_datav, num_pages,
bio, &bio_is_patched,
NULL, bio->bi_opf);
bio_for_each_segment_all_sp(bvec, bio, i, bia)
diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index 137f95063c85..3c021f4dd8bf 100644
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -2273,6 +2273,7 @@ int btrfs_get_io_failure_record(struct inode *inode, u64 start, u64 end,
}
int btrfs_check_repairable(struct inode *inode, struct bio *failed_bio,
+ unsigned num_pages,
struct io_failure_record *failrec, int failed_mirror)
{
struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
@@ -2296,7 +2297,7 @@ int btrfs_check_repairable(struct inode *inode, struct bio *failed_bio,
* a) deliver good data to the caller
* b) correct the bad sectors on disk
*/
- if (failed_bio->bi_vcnt > 1) {
+ if (num_pages > 1) {
/*
* to fulfill b), we need to know the exact failing sectors, as
* we don't want to rewrite any more than the failed ones. thus,
@@ -2390,6 +2391,7 @@ static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset,
struct bio *bio;
int read_mode = 0;
int ret;
+ unsigned num_pages = bio_segments_all(failed_bio);
BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE);
@@ -2397,13 +2399,14 @@ static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset,
if (ret)
return ret;
- ret = btrfs_check_repairable(inode, failed_bio, failrec, failed_mirror);
+ ret = btrfs_check_repairable(inode, failed_bio, num_pages, failrec,
+ failed_mirror);
if (!ret) {
free_io_failure(inode, failrec);
return -EIO;
}
- if (failed_bio->bi_vcnt > 1)
+ if (num_pages > 1)
read_mode |= REQ_FAILFAST_DEV;
phy_offset >>= inode->i_sb->s_blocksize_bits;
diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h
index 17f9ce479ed7..8fce380bc8d7 100644
--- a/fs/btrfs/extent_io.h
+++ b/fs/btrfs/extent_io.h
@@ -483,6 +483,7 @@ void btrfs_free_io_failure_record(struct inode *inode, u64 start, u64 end);
int btrfs_get_io_failure_record(struct inode *inode, u64 start, u64 end,
struct io_failure_record **failrec_ret);
int btrfs_check_repairable(struct inode *inode, struct bio *failed_bio,
+ unsigned num_pages,
struct io_failure_record *failrec, int fail_mirror);
struct bio *btrfs_create_repair_bio(struct inode *inode, struct bio *failed_bio,
struct io_failure_record *failrec,
diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
index 544e1cf9a552..fa77cb210907 100644
--- a/fs/btrfs/inode.c
+++ b/fs/btrfs/inode.c
@@ -7914,7 +7914,7 @@ static int dio_read_error(struct inode *inode, struct bio *failed_bio,
return -EIO;
}
- if ((failed_bio->bi_vcnt > 1)
+ if ((bio_segments_all(failed_bio) > 1)
|| (failed_bio->bi_io_vec->bv_len
> btrfs_inode_sectorsize(inode)))
read_mode |= REQ_FAILFAST_DEV;
--
2.7.4
Signed-off-by: Ming Lei <[email protected]>
---
fs/btrfs/compression.c | 4 ++++
fs/btrfs/inode.c | 17 +++++++++++++++++
2 files changed, 21 insertions(+)
diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c
index 485a60923bba..54306dee392a 100644
--- a/fs/btrfs/compression.c
+++ b/fs/btrfs/compression.c
@@ -601,6 +601,10 @@ int btrfs_submit_compressed_read(struct inode *inode, struct bio *bio,
/* we need the actual starting offset of this extent in the file */
read_lock(&em_tree->lock);
+ /*
+ * It is still safe to retrieve the 1st page of the bio
+ * in this way after supporting multipage bvec.
+ */
em = lookup_extent_mapping(em_tree,
page_offset(bio->bi_io_vec->bv_page),
PAGE_SIZE);
diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
index fa77cb210907..d0714208495f 100644
--- a/fs/btrfs/inode.c
+++ b/fs/btrfs/inode.c
@@ -7914,6 +7914,11 @@ static int dio_read_error(struct inode *inode, struct bio *failed_bio,
return -EIO;
}
+ /*
+ * In this case it is still correct to retrieve length
+ * of the 1st bvec of the bio in this way after supporting
+ * multipage bvec.
+ */
if ((bio_segments_all(failed_bio) > 1)
|| (failed_bio->bi_io_vec->bv_len
> btrfs_inode_sectorsize(inode)))
@@ -7960,6 +7965,12 @@ static void btrfs_retry_endio_nocsum(struct bio *bio)
if (bio->bi_error)
goto end;
+ /*
+ * WARNING:
+ *
+ * With multipage bvec, the following way of direct access to
+ * bvec table is only safe if the bio includes single page.
+ */
ASSERT(bio->bi_vcnt == 1);
inode = bio->bi_io_vec->bv_page->mapping->host;
ASSERT(bio->bi_io_vec->bv_len == btrfs_inode_sectorsize(inode));
@@ -8045,6 +8056,12 @@ static void btrfs_retry_endio(struct bio *bio)
start = done->start;
+ /*
+ * WARNING:
+ *
+ * With multipage bvec, the following way of direct access to
+ * bvec table is only safe if the bio includes single page.
+ */
ASSERT(bio->bi_vcnt == 1);
inode = bio->bi_io_vec->bv_page->mapping->host;
ASSERT(bio->bi_io_vec->bv_len == btrfs_inode_sectorsize(inode));
--
2.7.4
This patch pulls the trigger for multipage bvecs.
Any request queue which doesn't set QUEUE_FLAG_NO_MP
should support to handle multipage bvecs.
Signed-off-by: Ming Lei <[email protected]>
---
block/bio.c | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/block/bio.c b/block/bio.c
index 9acc3db0f2df..a2ad9c577aff 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -843,6 +843,11 @@ int bio_add_page(struct bio *bio, struct page *page,
* a consecutive offset. Optimize this special case.
*/
if (bio->bi_vcnt > 0) {
+ struct request_queue *q = NULL;
+
+ if (bio->bi_bdev)
+ q = bdev_get_queue(bio->bi_bdev);
+
bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
if (page == bv->bv_page &&
@@ -850,6 +855,14 @@ int bio_add_page(struct bio *bio, struct page *page,
bv->bv_len += len;
goto done;
}
+
+ /* disable multipage bvec too if cluster isn't enabled */
+ if (q && !blk_queue_no_mp(q) && blk_queue_cluster(q) &&
+ (bvec_to_phys(bv) + bv->bv_len ==
+ page_to_phys(page) + offset)) {
+ bv->bv_len += len;
+ goto done;
+ }
}
if (bio->bi_vcnt >= bio->bi_max_vecs)
--
2.7.4
For making multipage bvec working well.
Signed-off-by: Ming Lei <[email protected]>
---
fs/buffer.c | 21 +++++++++++++--------
1 file changed, 13 insertions(+), 8 deletions(-)
diff --git a/fs/buffer.c b/fs/buffer.c
index 63d2f40c21fd..014892593f57 100644
--- a/fs/buffer.c
+++ b/fs/buffer.c
@@ -3063,8 +3063,7 @@ void guard_bio_eod(int op, struct bio *bio)
unsigned truncated_bytes;
/*
* It is safe to truncate the last bvec in the following way
- * even though multipage bvec is supported, but we need to
- * fix the parameters passed to zero_user().
+ * even though multipage bvec is supported.
*/
struct bio_vec *bvec = &bio->bi_io_vec[bio->bi_vcnt - 1];
@@ -3087,15 +3086,21 @@ void guard_bio_eod(int op, struct bio *bio)
/* Uhhuh. We've got a bio that straddles the device size! */
truncated_bytes = bio->bi_iter.bi_size - (maxsector << 9);
- /* Truncate the bio.. */
- bio->bi_iter.bi_size -= truncated_bytes;
- bvec->bv_len -= truncated_bytes;
-
/* ..and clear the end of the buffer for reads */
if (op == REQ_OP_READ) {
- zero_user(bvec->bv_page, bvec->bv_offset + bvec->bv_len,
- truncated_bytes);
+ struct bvec_iter start = BVEC_ITER_ALL_INIT;
+ struct bvec_iter iter;
+ struct bio_vec bv;
+
+ start.bi_bvec_done = bvec->bv_len - truncated_bytes;
+
+ __bvec_for_each_sp_bvec(bv, bvec, iter, start)
+ zero_user(bv.bv_page, bv.bv_offset, bv.bv_len);
}
+
+ /* Truncate the bio.. */
+ bio->bi_iter.bi_size -= truncated_bytes;
+ bvec->bv_len -= truncated_bytes;
}
static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh,
--
2.7.4
Signed-off-by: Ming Lei <[email protected]>
---
fs/iomap.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/fs/iomap.c b/fs/iomap.c
index 354a123f170e..11da77b24599 100644
--- a/fs/iomap.c
+++ b/fs/iomap.c
@@ -698,8 +698,9 @@ static void iomap_dio_bio_end_io(struct bio *bio)
} else {
struct bio_vec *bvec;
int i;
+ struct bvec_iter_all bia;
- bio_for_each_segment_all(bvec, bio, i)
+ bio_for_each_segment_all_sp(bvec, bio, i, bia)
put_page(bvec->bv_page);
bio_put(bio);
}
--
2.7.4
Signed-off-by: Ming Lei <[email protected]>
---
fs/block_dev.c | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/fs/block_dev.c b/fs/block_dev.c
index 6254cee8f8f3..644b94460348 100644
--- a/fs/block_dev.c
+++ b/fs/block_dev.c
@@ -209,6 +209,7 @@ __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,
ssize_t ret;
blk_qc_t qc;
int i;
+ struct bvec_iter_all bia;
if ((pos | iov_iter_alignment(iter)) &
(bdev_logical_block_size(bdev) - 1))
@@ -253,7 +254,7 @@ __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,
}
__set_current_state(TASK_RUNNING);
- bio_for_each_segment_all(bvec, &bio, i) {
+ bio_for_each_segment_all_sp(bvec, &bio, i, bia) {
if (should_dirty && !PageCompound(bvec->bv_page))
set_page_dirty_lock(bvec->bv_page);
put_page(bvec->bv_page);
@@ -315,8 +316,9 @@ static void blkdev_bio_end_io(struct bio *bio)
} else {
struct bio_vec *bvec;
int i;
+ struct bvec_iter_all bia;
- bio_for_each_segment_all(bvec, bio, i)
+ bio_for_each_segment_all_sp(bvec, bio, i, bia)
put_page(bvec->bv_page);
bio_put(bio);
}
--
2.7.4
Signed-off-by: Ming Lei <[email protected]>
---
fs/exofs/ore.c | 3 ++-
fs/exofs/ore_raid.c | 3 ++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/fs/exofs/ore.c b/fs/exofs/ore.c
index 8bb72807e70d..38a7d8bfdd4c 100644
--- a/fs/exofs/ore.c
+++ b/fs/exofs/ore.c
@@ -406,8 +406,9 @@ static void _clear_bio(struct bio *bio)
{
struct bio_vec *bv;
unsigned i;
+ struct bvec_iter_all bia;
- bio_for_each_segment_all(bv, bio, i) {
+ bio_for_each_segment_all_sp(bv, bio, i, bia) {
unsigned this_count = bv->bv_len;
if (likely(PAGE_SIZE == this_count))
diff --git a/fs/exofs/ore_raid.c b/fs/exofs/ore_raid.c
index 27cbdb697649..37c0a9aa2ec2 100644
--- a/fs/exofs/ore_raid.c
+++ b/fs/exofs/ore_raid.c
@@ -429,6 +429,7 @@ static void _mark_read4write_pages_uptodate(struct ore_io_state *ios, int ret)
{
struct bio_vec *bv;
unsigned i, d;
+ struct bvec_iter_all bia;
/* loop on all devices all pages */
for (d = 0; d < ios->numdevs; d++) {
@@ -437,7 +438,7 @@ static void _mark_read4write_pages_uptodate(struct ore_io_state *ios, int ret)
if (!bio)
continue;
- bio_for_each_segment_all(bv, bio, i) {
+ bio_for_each_segment_all_sp(bv, bio, i, bia) {
struct page *page = bv->bv_page;
SetPageUptodate(page);
--
2.7.4
Signed-off-by: Ming Lei <[email protected]>
---
fs/f2fs/data.c | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c
index 5d1a192e1c3c..d48b2057c2ee 100644
--- a/fs/f2fs/data.c
+++ b/fs/f2fs/data.c
@@ -53,6 +53,7 @@ static void f2fs_read_end_io(struct bio *bio)
{
struct bio_vec *bvec;
int i;
+ struct bvec_iter_all bia;
#ifdef CONFIG_F2FS_FAULT_INJECTION
/*
@@ -72,7 +73,7 @@ static void f2fs_read_end_io(struct bio *bio)
}
}
- bio_for_each_segment_all(bvec, bio, i) {
+ bio_for_each_segment_all_sp(bvec, bio, i, bia) {
struct page *page = bvec->bv_page;
if (!bio->bi_error) {
@@ -92,8 +93,9 @@ static void f2fs_write_end_io(struct bio *bio)
struct f2fs_sb_info *sbi = bio->bi_private;
struct bio_vec *bvec;
int i;
+ struct bvec_iter_all bia;
- bio_for_each_segment_all(bvec, bio, i) {
+ bio_for_each_segment_all_sp(bvec, bio, i, bia) {
struct page *page = bvec->bv_page;
enum count_type type = WB_DATA_TYPE(page);
@@ -206,6 +208,7 @@ static bool __has_merged_page(struct f2fs_bio_info *io, struct inode *inode,
struct bio_vec *bvec;
struct page *target;
int i;
+ struct bvec_iter_all bia;
if (!io->bio)
return false;
@@ -213,7 +216,7 @@ static bool __has_merged_page(struct f2fs_bio_info *io, struct inode *inode,
if (!inode && !page && !ino)
return true;
- bio_for_each_segment_all(bvec, io->bio, i) {
+ bio_for_each_segment_all_sp(bvec, io->bio, i, bia) {
if (bvec->bv_page->mapping)
target = bvec->bv_page;
--
2.7.4
Signed-off-by: Ming Lei <[email protected]>
---
fs/gfs2/lops.c | 3 ++-
fs/gfs2/meta_io.c | 3 ++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/fs/gfs2/lops.c b/fs/gfs2/lops.c
index b1f9144b42c7..ddbd1f772cdb 100644
--- a/fs/gfs2/lops.c
+++ b/fs/gfs2/lops.c
@@ -208,13 +208,14 @@ static void gfs2_end_log_write(struct bio *bio)
struct bio_vec *bvec;
struct page *page;
int i;
+ struct bvec_iter_all bia;
if (bio->bi_error) {
sdp->sd_log_error = bio->bi_error;
fs_err(sdp, "Error %d writing to log\n", bio->bi_error);
}
- bio_for_each_segment_all(bvec, bio, i) {
+ bio_for_each_segment_all_sp(bvec, bio, i, bia) {
page = bvec->bv_page;
if (page_has_buffers(page))
gfs2_end_log_write_bh(sdp, bvec, bio->bi_error);
diff --git a/fs/gfs2/meta_io.c b/fs/gfs2/meta_io.c
index 49db8ef13fdf..317cc8ed74ce 100644
--- a/fs/gfs2/meta_io.c
+++ b/fs/gfs2/meta_io.c
@@ -190,8 +190,9 @@ static void gfs2_meta_read_endio(struct bio *bio)
{
struct bio_vec *bvec;
int i;
+ struct bvec_iter_all bia;
- bio_for_each_segment_all(bvec, bio, i) {
+ bio_for_each_segment_all_sp(bvec, bio, i, bia) {
struct page *page = bvec->bv_page;
struct buffer_head *bh = page_buffers(page);
unsigned int len = bvec->bv_len;
--
2.7.4
Signed-off-by: Ming Lei <[email protected]>
---
fs/xfs/xfs_aops.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c
index 0f56fcd3a5d5..1acd6162aec1 100644
--- a/fs/xfs/xfs_aops.c
+++ b/fs/xfs/xfs_aops.c
@@ -139,6 +139,7 @@ xfs_destroy_ioend(
for (bio = &ioend->io_inline_bio; bio; bio = next) {
struct bio_vec *bvec;
int i;
+ struct bvec_iter_all bia;
/*
* For the last bio, bi_private points to the ioend, so we
@@ -150,7 +151,7 @@ xfs_destroy_ioend(
next = bio->bi_private;
/* walk each page on bio, ending page IO on them */
- bio_for_each_segment_all(bvec, bio, i)
+ bio_for_each_segment_all_sp(bvec, bio, i, bia)
xfs_finish_page_writeback(inode, bvec, error);
bio_put(bio);
--
2.7.4
Signed-off-by: Ming Lei <[email protected]>
---
fs/ext4/page-io.c | 3 ++-
fs/ext4/readpage.c | 3 ++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c
index d83b0f3c5fe9..34d24d8d63d9 100644
--- a/fs/ext4/page-io.c
+++ b/fs/ext4/page-io.c
@@ -63,8 +63,9 @@ static void ext4_finish_bio(struct bio *bio)
{
int i;
struct bio_vec *bvec;
+ struct bvec_iter_all bia;
- bio_for_each_segment_all(bvec, bio, i) {
+ bio_for_each_segment_all_sp(bvec, bio, i, bia) {
struct page *page = bvec->bv_page;
#ifdef CONFIG_EXT4_FS_ENCRYPTION
struct page *data_page = NULL;
diff --git a/fs/ext4/readpage.c b/fs/ext4/readpage.c
index a81b829d56de..6d50b2ca532a 100644
--- a/fs/ext4/readpage.c
+++ b/fs/ext4/readpage.c
@@ -71,6 +71,7 @@ static void mpage_end_io(struct bio *bio)
{
struct bio_vec *bv;
int i;
+ struct bvec_iter_all bia;
if (ext4_bio_encrypted(bio)) {
if (bio->bi_error) {
@@ -80,7 +81,7 @@ static void mpage_end_io(struct bio *bio)
return;
}
}
- bio_for_each_segment_all(bv, bio, i) {
+ bio_for_each_segment_all_sp(bv, bio, i, bia) {
struct page *page = bv->bv_page;
if (!bio->bi_error) {
--
2.7.4
Signed-off-by: Ming Lei <[email protected]>
---
fs/direct-io.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/fs/direct-io.c b/fs/direct-io.c
index aeae8c063451..e2ecd9fbbffb 100644
--- a/fs/direct-io.c
+++ b/fs/direct-io.c
@@ -487,7 +487,9 @@ static int dio_bio_complete(struct dio *dio, struct bio *bio)
err = bio->bi_error;
bio_check_pages_dirty(bio); /* transfers ownership */
} else {
- bio_for_each_segment_all(bvec, bio, i) {
+ struct bvec_iter_all bia;
+
+ bio_for_each_segment_all_sp(bvec, bio, i, bia) {
struct page *page = bvec->bv_page;
if (dio->op == REQ_OP_READ && !PageCompound(page) &&
--
2.7.4
Signed-off-by: Ming Lei <[email protected]>
---
block/bio.c | 17 +++++++++++------
block/blk-zoned.c | 5 +++--
block/bounce.c | 6 ++++--
3 files changed, 18 insertions(+), 10 deletions(-)
diff --git a/block/bio.c b/block/bio.c
index e65de53acb9f..9acc3db0f2df 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -992,7 +992,7 @@ int bio_alloc_pages(struct bio *bio, gfp_t gfp_mask)
int i;
struct bio_vec *bv;
- bio_for_each_segment_all(bv, bio, i) {
+ bio_for_each_segment_all_mp(bv, bio, i) {
bv->bv_page = alloc_page(gfp_mask);
if (!bv->bv_page) {
while (--bv >= bio->bi_io_vec)
@@ -1093,8 +1093,9 @@ static int bio_copy_from_iter(struct bio *bio, struct iov_iter iter)
{
int i;
struct bio_vec *bvec;
+ struct bvec_iter_all bia;
- bio_for_each_segment_all(bvec, bio, i) {
+ bio_for_each_segment_all_sp(bvec, bio, i, bia) {
ssize_t ret;
ret = copy_page_from_iter(bvec->bv_page,
@@ -1124,8 +1125,9 @@ static int bio_copy_to_iter(struct bio *bio, struct iov_iter iter)
{
int i;
struct bio_vec *bvec;
+ struct bvec_iter_all bia;
- bio_for_each_segment_all(bvec, bio, i) {
+ bio_for_each_segment_all_sp(bvec, bio, i, bia) {
ssize_t ret;
ret = copy_page_to_iter(bvec->bv_page,
@@ -1147,8 +1149,9 @@ void bio_free_pages(struct bio *bio)
{
struct bio_vec *bvec;
int i;
+ struct bvec_iter_all bia;
- bio_for_each_segment_all(bvec, bio, i)
+ bio_for_each_segment_all_sp(bvec, bio, i, bia)
__free_page(bvec->bv_page);
}
EXPORT_SYMBOL(bio_free_pages);
@@ -1448,11 +1451,12 @@ static void __bio_unmap_user(struct bio *bio)
{
struct bio_vec *bvec;
int i;
+ struct bvec_iter_all bia;
/*
* make sure we dirty pages we wrote to
*/
- bio_for_each_segment_all(bvec, bio, i) {
+ bio_for_each_segment_all_sp(bvec, bio, i, bia) {
if (bio_data_dir(bio) == READ)
set_page_dirty_lock(bvec->bv_page);
@@ -1544,8 +1548,9 @@ static void bio_copy_kern_endio_read(struct bio *bio)
char *p = bio->bi_private;
struct bio_vec *bvec;
int i;
+ struct bvec_iter_all bia;
- bio_for_each_segment_all(bvec, bio, i) {
+ bio_for_each_segment_all_sp(bvec, bio, i, bia) {
memcpy(p, page_address(bvec->bv_page), bvec->bv_len);
p += bvec->bv_len;
}
diff --git a/block/blk-zoned.c b/block/blk-zoned.c
index 472211fa183a..68cf3054296c 100644
--- a/block/blk-zoned.c
+++ b/block/blk-zoned.c
@@ -81,6 +81,7 @@ int blkdev_report_zones(struct block_device *bdev,
unsigned int ofst;
void *addr;
int ret;
+ struct bvec_iter_all bia;
if (!q)
return -ENXIO;
@@ -148,7 +149,7 @@ int blkdev_report_zones(struct block_device *bdev,
n = 0;
nz = 0;
nr_rep = 0;
- bio_for_each_segment_all(bv, bio, i) {
+ bio_for_each_segment_all_sp(bv, bio, i, bia) {
if (!bv->bv_page)
break;
@@ -181,7 +182,7 @@ int blkdev_report_zones(struct block_device *bdev,
*nr_zones = nz;
out:
- bio_for_each_segment_all(bv, bio, i)
+ bio_for_each_segment_all_sp(bv, bio, i, bia)
__free_page(bv->bv_page);
bio_put(bio);
diff --git a/block/bounce.c b/block/bounce.c
index 08841ed4cdae..871bb12be0c4 100644
--- a/block/bounce.c
+++ b/block/bounce.c
@@ -135,11 +135,12 @@ static void bounce_end_io(struct bio *bio, mempool_t *pool)
struct bio_vec *bvec, orig_vec;
int i;
struct bvec_iter orig_iter = bio_orig->bi_iter;
+ struct bvec_iter_all bia;
/*
* free up bounce indirect pages used
*/
- bio_for_each_segment_all(bvec, bio, i) {
+ bio_for_each_segment_all_sp(bvec, bio, i, bia) {
orig_vec = bio_iter_iovec(bio_orig, orig_iter);
if (bvec->bv_page == orig_vec.bv_page)
@@ -209,13 +210,14 @@ static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig,
int rw = bio_data_dir(*bio_orig);
struct bio_vec *to;
unsigned i;
+ struct bvec_iter_all bia;
if (!need_bounce(q, *bio_orig))
return;
bio = bio_clone_bioset_sp(*bio_orig, GFP_NOIO, fs_bio_set);
- bio_for_each_segment_all(to, bio, i) {
+ bio_for_each_segment_all_sp(to, bio, i, bia) {
struct page *page = to->bv_page;
if (page_to_pfn(page) <= queue_bounce_pfn(q))
--
2.7.4