If enabling bio polling queue task, we will split and chain
the bios if needed, then fill the request to the lossless ring
of polling queue task.
Signed-off-by: Ping Gan <[email protected]>
---
drivers/nvme/target/io-cmd-bdev.c | 243 ++++++++++++++++++++++++++----
1 file changed, 214 insertions(+), 29 deletions(-)
diff --git a/drivers/nvme/target/io-cmd-bdev.c b/drivers/nvme/target/io-cmd-bdev.c
index 468833675cc9..6f7d04ae6cb7 100644
--- a/drivers/nvme/target/io-cmd-bdev.c
+++ b/drivers/nvme/target/io-cmd-bdev.c
@@ -184,6 +184,16 @@ static void nvmet_bio_done(struct bio *bio)
nvmet_req_bio_put(req, bio);
}
+static void nvmet_pqt_bio_done(struct bio *bio)
+{
+ struct nvmet_pqt_bio_req *req_done = bio->bi_private;
+
+ nvmet_req_complete(req_done->req, blk_to_nvme_status(req_done->req,
+ bio->bi_status));
+ nvmet_req_bio_put(req_done->req, bio);
+ req_done->io_completed = 1;
+}
+
#ifdef CONFIG_BLK_DEV_INTEGRITY
static int nvmet_bdev_alloc_bip(struct nvmet_req *req, struct bio *bio,
struct sg_mapping_iter *miter)
@@ -237,6 +247,38 @@ static int nvmet_bdev_alloc_bip(struct nvmet_req *req, struct bio *bio,
}
#endif /* CONFIG_BLK_DEV_INTEGRITY */
+#ifdef CONFIG_NVME_MULTIPATH
+extern struct block_device *nvme_mpath_get_bdev(struct block_device *bdev);
+extern const struct block_device_operations nvme_ns_head_ops;
+#endif
+
+static inline int nvmet_chain_par_bio(struct nvmet_req *req, struct bio **bio,
+ struct sg_mapping_iter *prot_miter, struct block_device *bdev,
+ sector_t sector, struct bio_list *blist)
+{
+ struct bio *parent, *child;
+ unsigned int vec_cnt;
+ int rc;
+
+ parent = *bio;
+ vec_cnt = queue_max_segments(bdev->bd_disk->queue);
+ if (req->metadata_len) {
+ rc = nvmet_bdev_alloc_bip(req, parent,
+ prot_miter);
+ if (unlikely(rc))
+ return rc;
+ }
+ child = bio_alloc(bdev, vec_cnt, parent->bi_opf, GFP_KERNEL);
+ child->bi_iter.bi_sector = sector;
+ *bio = child;
+ bio_chain(*bio, parent);
+ parent->bi_opf |= REQ_POLLED;
+ parent->bi_opf |= REQ_NOWAIT;
+ parent->bi_opf |= REQ_NOMERGE;
+ bio_list_add(blist, parent);
+ return 0;
+}
+
static void nvmet_bdev_execute_rw(struct nvmet_req *req)
{
unsigned int sg_cnt = req->sg_cnt;
@@ -247,8 +289,13 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req)
blk_opf_t opf;
int i, rc;
struct sg_mapping_iter prot_miter;
- unsigned int iter_flags;
+ unsigned int iter_flags, max_sectors;
+ unsigned short vec_cnt, max_segments;
unsigned int total_len = nvmet_rw_data_len(req) + req->metadata_len;
+ bool pqt_enabled = nvmet_pqt_enabled();
+ unsigned int sg_len;
+ struct nvmet_pqt_bio_req *req_done = NULL;
+ struct block_device *bdev = req->ns->bdev;
if (!nvmet_check_transfer_len(req, total_len))
return;
@@ -268,6 +315,24 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req)
iter_flags = SG_MITER_FROM_SG;
}
+#ifdef CONFIG_NVME_MULTIPATH
+ if (pqt_enabled && bdev->bd_disk->fops == &nvme_ns_head_ops) {
+ bdev = nvme_mpath_get_bdev(bdev);
+ if (!bdev) {
+ nvmet_req_complete(req, 0);
+ return;
+ }
+ opf |= REQ_DRV;
+ }
+#endif
+ if (pqt_enabled) {
+ req_done = kmalloc(sizeof(struct nvmet_pqt_bio_req), GFP_KERNEL);
+ if (!req_done) {
+ nvmet_req_complete(req, 0);
+ return;
+ }
+ }
+
if (is_pci_p2pdma_page(sg_page(req->sg)))
opf |= REQ_NOMERGE;
@@ -278,54 +343,174 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req)
bio_init(bio, req->ns->bdev, req->inline_bvec,
ARRAY_SIZE(req->inline_bvec), opf);
} else {
- bio = bio_alloc(req->ns->bdev, bio_max_segs(sg_cnt), opf,
+ vec_cnt = bio_max_segs(sg_cnt);
+ if (pqt_enabled)
+ vec_cnt = queue_max_segments(bdev->bd_disk->queue);
+ bio = bio_alloc(bdev, vec_cnt, opf,
GFP_KERNEL);
}
bio->bi_iter.bi_sector = sector;
- bio->bi_private = req;
- bio->bi_end_io = nvmet_bio_done;
+ if (!pqt_enabled) {
+ bio->bi_private = req;
+ bio->bi_end_io = nvmet_bio_done;
+ } else {
+ req_done->req = req;
+ bio->bi_private = req_done;
+ bio->bi_end_io = nvmet_pqt_bio_done;
+ }
- blk_start_plug(&plug);
+ if (!pqt_enabled)
+ blk_start_plug(&plug);
if (req->metadata_len)
sg_miter_start(&prot_miter, req->metadata_sg,
req->metadata_sg_cnt, iter_flags);
- for_each_sg(req->sg, sg, req->sg_cnt, i) {
- while (bio_add_page(bio, sg_page(sg), sg->length, sg->offset)
- != sg->length) {
- struct bio *prev = bio;
-
- if (req->metadata_len) {
- rc = nvmet_bdev_alloc_bip(req, bio,
- &prot_miter);
- if (unlikely(rc)) {
- bio_io_error(bio);
- return;
+ if (!pqt_enabled) {
+ for_each_sg(req->sg, sg, req->sg_cnt, i) {
+ while (bio_add_page(bio, sg_page(sg), sg->length, sg->offset)
+ != sg->length) {
+ struct bio *prev = bio;
+
+ if (req->metadata_len) {
+ rc = nvmet_bdev_alloc_bip(req, bio,
+ &prot_miter);
+ if (unlikely(rc)) {
+ bio_io_error(bio);
+ return;
+ }
}
- }
- bio = bio_alloc(req->ns->bdev, bio_max_segs(sg_cnt),
- opf, GFP_KERNEL);
- bio->bi_iter.bi_sector = sector;
+ bio = bio_alloc(req->ns->bdev, bio_max_segs(sg_cnt),
+ opf, GFP_KERNEL);
+ bio->bi_iter.bi_sector = sector;
- bio_chain(bio, prev);
- submit_bio(prev);
- }
+ bio_chain(bio, prev);
+ submit_bio(prev);
+ }
- sector += sg->length >> 9;
- sg_cnt--;
+ sector += sg->length >> 9;
+ sg_cnt--;
+ }
+ } else {
+ bio_list_init(&req_done->blist);
+ if (!test_bit(QUEUE_FLAG_POLL, &bdev->bd_disk->queue->queue_flags))
+ goto err_bio;
+ max_sectors = bdev->bd_disk->queue->limits.max_sectors;
+ max_sectors <<= 9;
+ max_segments = queue_max_segments(bdev->bd_disk->queue);
+ sg_len = 0;
+ unsigned int offset, len, vec_len, i;
+ bool sg_start_pg = true, need_chain_bio = false;
+ struct page *sglist_page, *max_sector_align;
+ sector_t temp_sector;
+
+ /*
+ * for bio's polling mode we will split bio to
+ * avoid low level's bio splitting when submit.
+ */
+ for_each_sg(req->sg, sg, req->sg_cnt, i) {
+ temp_sector = sector;
+ offset = (sg->offset % PAGE_SIZE);
+ if (offset + sg->length > PAGE_SIZE) { // need to split
+ len = sg->length;
+ i = 0;
+ sglist_page = virt_to_page(page_to_virt(sg_page(sg)) + offset);
+ if (offset != 0)
+ sg_start_pg = false;
+ while (len > PAGE_SIZE) {
+ max_sector_align = virt_to_page(page_to_virt(sglist_page) +
+ (PAGE_SIZE*i));
+ vec_len = sg_start_pg?PAGE_SIZE:(PAGE_SIZE - offset);
+ if (bio->bi_vcnt == max_segments - 1 ||
+ sg_len + vec_len > max_sectors)
+ need_chain_bio = true;
+ else {
+ __bio_add_page(bio, max_sector_align,
+ vec_len, sg_start_pg?0:offset);
+ temp_sector += vec_len >> 9;
+ sg_len += vec_len;
+ }
+ if (need_chain_bio) {
+ rc = nvmet_chain_par_bio(req, &bio, &prot_miter,
+ bdev, temp_sector, &req_done->blist);
+ if (unlikely(rc))
+ goto err_bio;
+ __bio_add_page(bio, max_sector_align, vec_len,
+ sg_start_pg?0:(PAGE_SIZE - offset));
+ temp_sector += vec_len >> 9;
+ sg_len = vec_len;
+ need_chain_bio = false;
+ }
+ if (!sg_start_pg) {
+ len -= (PAGE_SIZE - offset);
+ sg_start_pg = true;
+ } else {
+ len -= PAGE_SIZE;
+ }
+ i++;
+ }
+ if (len > 0) {
+ max_sector_align = virt_to_page(page_to_virt(sglist_page) +
+ (i * PAGE_SIZE));
+ if (bio->bi_vcnt == max_segments - 1 ||
+ sg_len + len > max_sectors) {
+ rc = nvmet_chain_par_bio(req, &bio, &prot_miter,
+ bdev, temp_sector, &req_done->blist);
+ if (unlikely(rc))
+ goto err_bio;
+ sg_len = len;
+ } else {
+ sg_len += len;
+ }
+ __bio_add_page(bio, max_sector_align, len, 0);
+ temp_sector += len >> 9;
+ }
+ } else {
+ if (bio->bi_vcnt == max_segments - 1 ||
+ sg_len + sg->length > max_sectors) {
+ rc = nvmet_chain_par_bio(req, &bio, &prot_miter,
+ bdev, temp_sector, &req_done->blist);
+ if (unlikely(rc))
+ goto err_bio;
+ sg_len = sg->length;
+ } else {
+ sg_len += sg->length;
+ }
+ __bio_add_page(bio, sg_page(sg), sg->length, sg->offset);
+ }
+ sector += sg->length >> 9;
+ sg_cnt--;
+ }
}
if (req->metadata_len) {
rc = nvmet_bdev_alloc_bip(req, bio, &prot_miter);
if (unlikely(rc)) {
- bio_io_error(bio);
- return;
+ goto err_bio;
}
}
- submit_bio(bio);
- blk_finish_plug(&plug);
+ if (pqt_enabled) {
+ bio->bi_opf |= REQ_POLLED;
+ bio->bi_opf |= REQ_NOWAIT;
+ bio->bi_opf |= REQ_NOMERGE;
+ bio_list_add(&req_done->blist, bio);
+ req_done->io_completed = 0;
+ rc = nvmet_pqt_ring_enqueue(req_done);
+ if (rc < 0)
+ goto err_bio;
+ nvmet_wakeup_pq_thread();
+ } else {
+ submit_bio(bio);
+ }
+ if (!pqt_enabled)
+ blk_finish_plug(&plug);
+ return;
+err_bio:
+ bio_io_error(bio);
+ if (pqt_enabled)
+ kfree(req_done);
+ return;
}
static void nvmet_bdev_execute_flush(struct nvmet_req *req)
--
2.26.2